Pad validator withdrawals with a reward accrual buffer

src/commands/start/startup_check.py

@@ -26,7 +26,14 @@
     wait_for_keystores_dir,
 )
 from src.common.wallet import wallet
-from src.config.settings import WITHDRAWALS_INTERVAL, settings
+from src.config.settings import (
+    MAX_WITHDRAWAL_BUFFER_GWEI,
+    MIN_WITHDRAWAL_BUFFER_GWEI,
+    MISSING_ASSETS_THRESHOLD_GWEI,
+    ORACLE_MISSING_ASSETS_THRESHOLD_GWEI,
+    WITHDRAWALS_INTERVAL,
+    settings,
+)
 from src.validators.keystores.local import LocalKeystore
 
 logger = logging.getLogger(__name__)
@@ -36,6 +43,20 @@
 async def startup_checks() -> None:
     validate_settings()
 
+    if not 1 <= MISSING_ASSETS_THRESHOLD_GWEI <= ORACLE_MISSING_ASSETS_THRESHOLD_GWEI:
+        raise ValueError(
+            f'MISSING_ASSETS_THRESHOLD_GWEI setting should be between 1 and '
+            f'{ORACLE_MISSING_ASSETS_THRESHOLD_GWEI} Gwei. Above the oracle threshold the '
+            'operator would skip exit queue shortfalls that the oracle covers with a full '
+            'validator exit.'
+        )
+
+    if not 0 <= MIN_WITHDRAWAL_BUFFER_GWEI <= MAX_WITHDRAWAL_BUFFER_GWEI:
+        raise ValueError(
+            f'MIN_WITHDRAWAL_BUFFER_GWEI setting should be between 0 and '
+            f'{MAX_WITHDRAWAL_BUFFER_GWEI} Gwei.'
+        )
+
     logger.info('Checking for newer operator version...')
     await check_operator_version()
 

src/common/contracts.py

@@ -14,6 +14,7 @@
     function_signature_to_4byte_selector,
     remove_0x_prefix,
 )
+from sw_utils import OsTokenConverter
 from web3 import AsyncWeb3, Web3
 from web3.contract import AsyncContract
 from web3.contract.async_contract import (
@@ -347,6 +348,32 @@ async def validators_manager(self) -> ChecksumAddress:
     async def get_exit_queue_index(self, position_ticket: int) -> int:
         return await self.contract.functions.getExitQueueIndex(position_ticket).call()
 
+    async def get_queued_exit_assets(
+        self, harvest_params: HarvestParams | None, block_number: BlockNumber
+    ) -> Wei:
+        """
+        Value of the queued exit shares, the part of the exit queue that keeps accruing rewards.
+        Legacy asset-denominated exits are fixed and excluded.
+        """
+        calls: list[HexStr] = []
+        if harvest_params is not None:
+            calls.append(self.get_update_state_call(harvest_params))
+        calls.append(self.encode_abi(fn_name='getExitQueueData', args=[]))
+        calls.append(self.encode_abi(fn_name='totalAssets', args=[]))
+        calls.append(self.encode_abi(fn_name='totalShares', args=[]))
+
+        multicall_response = await self.contract.functions.multicall(calls).call(
+            block_identifier=block_number
+        )
+        queued_shares, *_ = eth_abi.decode(
+            ['uint128', 'uint128', 'uint128', 'uint128', 'uint256'], multicall_response[-3]
+        )
+        converter = OsTokenConverter(
+            total_assets=Wei(Web3.to_int(multicall_response[-2])),
+            total_shares=Wei(Web3.to_int(multicall_response[-1])),
+        )
+        return converter.to_assets(Wei(queued_shares))
+
     async def get_validator_withdrawal_submitted_events(
         self,
         from_block: BlockNumber,

src/common/tests/test_contracts.py

@@ -0,0 +1,67 @@
+from unittest import mock
+
+from eth_abi import abi as eth_abi
+from hexbytes import HexBytes
+from web3 import AsyncWeb3, Web3
+from web3.types import BlockNumber, Wei
+
+from src.common.contracts import VaultContract
+from src.common.typings import HarvestParams
+
+VAULT_ADDRESS = Web3.to_checksum_address('0x' + '11' * 20)
+
+
+class TestGetQueuedExitAssets:
+    async def test_without_harvest_params(self):
+        vault = _vault_contract()
+        exit_queue_data = eth_abi.encode(
+            ['uint128', 'uint128', 'uint128', 'uint128', 'uint256'], [10, 1, 2, 3, 4]
+        )
+        response = [exit_queue_data, (25).to_bytes(32, 'big'), (100).to_bytes(32, 'big')]
+
+        with mock.patch.object(vault.contract.functions, 'multicall') as multicall_mock:
+            multicall_mock.return_value.call = mock.AsyncMock(return_value=response)
+            result = await vault.get_queued_exit_assets(None, BlockNumber(123))
+
+        assert result == Wei(2)  # 10 * 25 // 100
+        multicall_mock.return_value.call.assert_awaited_once_with(block_identifier=BlockNumber(123))
+        assert len(multicall_mock.call_args.args[0]) == 3
+
+    async def test_with_harvest_params(self):
+        vault = _vault_contract()
+        harvest_params = HarvestParams(
+            rewards_root=HexBytes(b'\x00' * 32),
+            reward=Wei(0),
+            unlocked_mev_reward=Wei(0),
+            proof=[],
+        )
+        exit_queue_data = eth_abi.encode(
+            ['uint128', 'uint128', 'uint128', 'uint128', 'uint256'], [10, 1, 2, 3, 4]
+        )
+        response = [b'', exit_queue_data, (25).to_bytes(32, 'big'), (100).to_bytes(32, 'big')]
+
+        with mock.patch.object(vault.contract.functions, 'multicall') as multicall_mock:
+            multicall_mock.return_value.call = mock.AsyncMock(return_value=response)
+            result = await vault.get_queued_exit_assets(harvest_params, BlockNumber(123))
+
+        assert result == Wei(2)  # 10 * 25 // 100
+        assert len(multicall_mock.call_args.args[0]) == 4
+
+    async def test_zero_total_shares(self):
+        vault = _vault_contract()
+        exit_queue_data = eth_abi.encode(
+            ['uint128', 'uint128', 'uint128', 'uint128', 'uint256'], [10, 1, 2, 3, 4]
+        )
+        response = [exit_queue_data, (0).to_bytes(32, 'big'), (0).to_bytes(32, 'big')]
+
+        with mock.patch.object(vault.contract.functions, 'multicall') as multicall_mock:
+            multicall_mock.return_value.call = mock.AsyncMock(return_value=response)
+            result = await vault.get_queued_exit_assets(None, BlockNumber(123))
+
+        assert result == Wei(0)
+
+
+def _vault_contract() -> VaultContract:
+    # A bare AsyncWeb3() avoids hitting the not-set-up ``execution_client`` singleton;
+    # eth.contract() only builds a local contract object, no network call is made.
+    return VaultContract(address=VAULT_ADDRESS, execution_client=AsyncWeb3())

src/config/settings.py

@@ -589,7 +589,40 @@ def network_config(self) -> NetworkConfig:
     group='Withdrawals',
     description='Minimum time between withdrawal processing runs, in seconds.',
 )
-MIN_WITHDRAWAL_AMOUNT_GWEI: Gwei = Gwei(1)
+# Withdrawal buffer is 0.1% of the exit queue, see calculate_withdrawal_buffer.
+WITHDRAWAL_BUFFER_BPS = 10
+# Excess buffer only lands as withdrawable assets and is re-staked, so cap it at 1 ETH.
+MAX_WITHDRAWAL_BUFFER_GWEI = Gwei(1_000_000_000)  # 1 ETH
+
+# The oracle exits validators once the shortfall reaches this value (MISSING_ASSETS_THRESHOLD
+# in v3-oracle), so the operator must not skip shortfalls at or above it.
+ORACLE_MISSING_ASSETS_THRESHOLD_GWEI = Gwei(10_000_000)  # 0.01 ETH
+MISSING_ASSETS_THRESHOLD_GWEI: Gwei = Gwei(
+    decouple_config(
+        'MISSING_ASSETS_THRESHOLD_GWEI',
+        default=1,
+        cast=int,
+        group='Withdrawals',
+        description=(
+            'Minimum exit queue shortfall in Gwei that triggers a validator withdrawal. '
+            'Must not exceed 10000000 Gwei (0.01 ETH, or 0.01 mGNO on Gnosis), the oracle '
+            'threshold for exiting validators.'
+        ),
+    )
+)
+MIN_WITHDRAWAL_BUFFER_GWEI: Gwei = Gwei(
+    decouple_config(
+        'MIN_WITHDRAWAL_BUFFER_GWEI',
+        default=10_000,
+        cast=int,
+        group='Withdrawals',
+        description=(
+            'Minimum buffer in Gwei added on top of the exit queue shortfall in a '
+            'validator withdrawal request. Must not exceed 1000000000 Gwei '
+            '(1 ETH, or 1 mGNO on Gnosis).'
+        ),
+    )
+)
 
 # telemetry
 TELEMETRY_INTERVAL: int = decouple_config(

src/withdrawals/assets.py

@@ -10,11 +10,17 @@
 from web3 import Web3
 from web3.types import Gwei, Wei
 
-from src.common.contracts import validators_checker_contract
+from src.common.contracts import VaultContract, validators_checker_contract
 from src.common.harvest import get_harvest_params
 from src.common.typings import ExitQueueMissingAssetsParams, PendingPartialWithdrawal
-from src.config.settings import settings
+from src.config.settings import (
+    MAX_WITHDRAWAL_BUFFER_GWEI,
+    MIN_WITHDRAWAL_BUFFER_GWEI,
+    WITHDRAWAL_BUFFER_BPS,
+    settings,
+)
 from src.validators.typings import ConsensusValidator, ValidatorConsolidationData
+from src.withdrawals.typings import ExitQueueAssets
 
 EXITING_STATUSES = [
     ValidatorStatus.ACTIVE_EXITING,
@@ -32,10 +38,10 @@ async def get_queued_assets(
     pending_partial_withdrawals: list[PendingPartialWithdrawal],
     chain_head: ChainHead,
     redemption_assets: Wei,
-) -> Gwei:
+) -> ExitQueueAssets:
     """
-    Get exit queue missing assets.
-    For Gno networks return value in mGNO-Gwei.
+    Get the exit queue shortfall and the value of the queued exit shares.
+    For Gno networks both are in mGNO-Gwei.
     """
     harvest_params = await get_harvest_params(settings.vault, chain_head.block_number)
 
@@ -66,9 +72,10 @@ async def get_queued_assets(
     if settings.network in GNO_NETWORKS:
         # apply mGNO -> GNO exchange rate
         withdrawing_assets = convert_to_gno(withdrawing_assets)
+
     # Missing assets express how much assets are needed to cover the exit requests
     # until the exit queue cumulative ticket is reached
-    queued_assets = await validators_checker_contract.get_exit_queue_missing_assets(
+    missing_assets = await validators_checker_contract.get_exit_queue_missing_assets(
         exit_queue_missing_assets_params=ExitQueueMissingAssetsParams(
             vault=settings.vault,
             withdrawing_assets=withdrawing_assets,
@@ -78,12 +85,35 @@ async def get_queued_assets(
         harvest_params=harvest_params,
         block_number=chain_head.block_number,
     )
+    # Queued shares value for the withdrawal buffer. Not needed when nothing is missing.
+    total_assets = Wei(0)
+    if missing_assets > 0:
+        total_assets = await VaultContract(settings.vault).get_queued_exit_assets(
+            harvest_params, chain_head.block_number
+        )
 
     if settings.network in GNO_NETWORKS:
         # apply GNO -> mGNO exchange rate
-        queued_assets = convert_to_mgno(queued_assets)
+        missing_assets = convert_to_mgno(missing_assets)
+        total_assets = convert_to_mgno(total_assets)
+
+    return ExitQueueAssets(
+        missing=Gwei(int(Web3.from_wei(missing_assets, 'gwei'))),
+        total=Gwei(int(Web3.from_wei(total_assets, 'gwei'))),
+    )
+
 
-    return Gwei(int(Web3.from_wei(queued_assets, 'gwei')))
+def calculate_withdrawal_buffer(total_queue_assets: Gwei) -> Gwei:
+    """
+    Queued assets keep accruing rewards while the withdrawal is pending on the consensus layer
+    (about 27h for a partial withdrawal, up to ~11 days for a full exit), so requesting the
+    exact shortfall leaves a new tiny one after every reward update. 0.1% of the queue covers
+    about 18 days of rewards at 2% APR; the floor covers queues too small for the ratio.
+    The cap keeps the buffer from growing with very large queues, the excess would only be
+    re-staked.
+    """
+    buffer = max(total_queue_assets * WITHDRAWAL_BUFFER_BPS // 10_000, MIN_WITHDRAWAL_BUFFER_GWEI)
+    return Gwei(min(buffer, MAX_WITHDRAWAL_BUFFER_GWEI))
 
 
 def _calculate_validators_exits_amount(