Add modern Zcash transaction support and Zebra RPC integration

src/electrumx/lib/coins.py

@@ -50,7 +50,8 @@
 import electrumx.server.daemon as daemon
 from electrumx.server.session import (ElectrumX, DashElectrumX,
                                       SmartCashElectrumX, AuxPoWElectrumX,
-                                      NameIndexElectrumX, NameIndexAuxPoWElectrumX)
+                                      NameIndexElectrumX, NameIndexAuxPoWElectrumX,
+                                      ZcashElectrumX)
 
 
 @dataclass(slots=True)
@@ -1397,12 +1398,48 @@ class Zcash(EquihashMixin, Coin):
     GENESIS_HASH = ('00040fe8ec8471911baa1db1266ea15d'
                     'd06b4a8a5c453883c000b031973dce08')
     DESERIALIZER = lib_tx.DeserializerZcash
+    DAEMON = daemon.ZcashZebraDaemon
+    SESSIONCLS = ZcashElectrumX
     TX_COUNT = 329196
     TX_COUNT_HEIGHT = 68379
     TX_PER_BLOCK = 5
     RPC_PORT = 8232
     REORG_LIMIT = 800
 
+    @classmethod
+    def max_fetch_blocks(cls, height: int) -> int:
+        # Previous conservative fixed policy, retained for comparison:
+        # return 10
+        #
+        # Zebra rejects JSON-RPC batch responses larger than 50 MiB.  The
+        # raw block is hex-encoded in that response, so the response size is
+        # roughly 2 * raw block bytes.  These values are rounded down from a
+        # full mainnet scan, targeting 35,000,000 bytes including 128 bytes
+        # of JSON overhead per response (well below Zebra's 52,428,800-byte
+        # limit).  The current and future range stays at 8: its future block
+        # sizes cannot be proven from historical measurements.
+        if height < 347_500:       # pre-Overwinter
+            return 8
+        if height < 419_200:       # Overwinter
+            return 10
+        if height < 653_600:       # Sapling
+            return 20
+        if height < 903_000:       # Blossom
+            return 600
+        if height < 1_046_400:     # Heartwood
+            return 200
+        if height < 1_687_104:     # Canopy
+            return 10
+        if height < 2_726_400:     # NU5
+            return 8
+        if height < 3_146_400:     # NU6
+            return 140
+        if height < 3_364_600:     # NU6.1
+            return 100
+        if height < 3_428_143:     # NU6.2
+            return 100
+        return 8                   # NU6.3 and future upgrades
+
 
 class ZcashTestnet(Zcash):
     SHORTNAME = "TAZ"

src/electrumx/lib/tx.py

@@ -28,7 +28,7 @@
 '''Transaction-related classes and functions.'''
 
 from dataclasses import dataclass
-from hashlib import blake2s
+from hashlib import blake2b, blake2s
 from typing import Sequence, Optional, Tuple
 
 from electrumx.lib.hash import sha256, double_sha256, hash_to_hex_str
@@ -50,6 +50,10 @@ class SkipTxDeserialize(Exception):
     '''
 
 
+class ZcashDeserializeError(ValueError):
+    '''Raised when a Zcash transaction has an invalid wire encoding.'''
+
+
 # note: slotted dataclasses are a bit faster than namedtuples
 # note: frozen dataclasses are much slower than non-frozen
 #       ref https://rednafi.com/python/statically-enforcing-frozen-dataclasses/
@@ -76,6 +80,18 @@ def serialize(self):
         ))
 
 
+@dataclass(kw_only=True, slots=True)
+class TxZcash(Tx):
+    '''Transparent projection and public metadata for a Zcash v5 or v6 transaction.'''
+    version_group_id: int
+    consensus_branch_id: int
+    expiry_height: int
+    fee_adjustment: int
+
+    def serialize(self):
+        raise ValueError('serializing a Zcash transparent projection is unsupported')
+
+
 @dataclass(kw_only=True, slots=True)
 class TxInput:
     '''Class representing a transaction input.'''
@@ -503,17 +519,240 @@ class DeserializerEquihashSegWit(DeserializerSegWit, DeserializerEquihash):
 
 
 class DeserializerZcash(DeserializerEquihash):
+    _V5_VERSION_GROUP_ID = 0x26A7270A
+    _V6_VERSION_GROUP_ID = 0xD884B698
+
+    @staticmethod
+    def _hash(personalization, data=b''):
+        return blake2b(data, digest_size=32, person=personalization).digest()
+
+    def _zcash_read(self, size):
+        if size < 0 or self.cursor + size > self._binary_length:
+            raise ZcashDeserializeError('truncated Zcash transaction')
+        return self._read_nbytes(size)
+
+    def _zcash_varint(self):
+        first = self._zcash_read(1)[0]
+        if first < 253:
+            return first
+        size, minimum = ((2, 253) if first == 253 else
+                         (4, 0x10000) if first == 254 else
+                         (8, 0x100000000))
+        value = int.from_bytes(self._zcash_read(size), 'little')
+        if value < minimum:
+            raise ZcashDeserializeError('non-canonical CompactSize')
+        return value
+
+    def _zcash_varbytes(self):
+        return self._zcash_read(self._zcash_varint())
+
+    def _zcash_transparent(self):
+        inputs = []
+        prevouts = []
+        sequences = []
+        scripts = []
+        input_count = self._zcash_varint()
+        if input_count > (self._binary_length - self.cursor) // 41:
+            raise ZcashDeserializeError('too many transparent inputs')
+        for _ in range(input_count):
+            prevout = self._zcash_read(36)
+            script_start = self.cursor
+            script = self._zcash_varbytes()
+            scripts.append(self.binary[script_start:self.cursor])
+            sequence = self._zcash_read(4)
+            prevouts.append(prevout)
+            sequences.append(sequence)
+            inputs.append(TxInput(
+                prev_txid_rev=prevout[:32],
+                prev_idx=int.from_bytes(prevout[32:], 'little'),
+                script=script,
+                sequence=int.from_bytes(sequence, 'little'),
+            ))
+
+        outputs = []
+        output_encodings = []
+        output_count = self._zcash_varint()
+        if output_count > (self._binary_length - self.cursor) // 9:
+            raise ZcashDeserializeError('too many transparent outputs')
+        for _ in range(output_count):
+            output_start = self.cursor
+            value = int.from_bytes(self._zcash_read(8), 'little', signed=True)
+            script = self._zcash_varbytes()
+            outputs.append(TxOutput(value=value, pk_script=script))
+            output_encodings.append(self.binary[output_start:self.cursor])
+
+        if inputs or outputs:
+            transparent_digest = self._hash(
+                b'ZTxIdTranspaHash',
+                self._hash(b'ZTxIdPrevoutHash', b''.join(prevouts)) +
+                self._hash(b'ZTxIdSequencHash', b''.join(sequences)) +
+                self._hash(b'ZTxIdOutputsHash', b''.join(output_encodings)),
+            )
+        else:
+            transparent_digest = self._hash(b'ZTxIdTranspaHash')
+        return inputs, outputs, scripts, transparent_digest
+
+    def _zcash_sapling(self, version):
+        spend_count = self._zcash_varint()
+        if spend_count > (self._binary_length - self.cursor) // 96:
+            raise ZcashDeserializeError('too many Sapling spends')
+        spends = [self._zcash_read(96) for _ in range(spend_count)]
+        output_count = self._zcash_varint()
+        if output_count > (self._binary_length - self.cursor) // 756:
+            raise ZcashDeserializeError('too many Sapling outputs')
+        outputs = [self._zcash_read(756) for _ in range(output_count)]
+        if not (spends or outputs):
+            return self._hash(b'ZTxIdSaplingHash'), 0
+
+        value_balance_bytes = self._zcash_read(8)
+        value_balance = int.from_bytes(value_balance_bytes, 'little', signed=True)
+        anchor = self._zcash_read(32) if spends else b''
+        self._zcash_read(192 * spend_count)
+        self._zcash_read(64 * spend_count)
+        self._zcash_read(192 * output_count)
+        self._zcash_read(64)
+
+        if spends:
+            noncompact_person = (b'ZTxIdSSpendNHash' if version == 5
+                                 else b'ZTxIdSSpendNH_v6')
+            compact = self._hash(b'ZTxIdSSpendCHash',
+                                 b''.join(spend[32:64] for spend in spends))
+            noncompact = self._hash(
+                noncompact_person,
+                b''.join(spend[:32] + (anchor if version == 5 else b'') + spend[64:]
+                         for spend in spends),
+            )
+            spend_digest = self._hash(b'ZTxIdSSpendsHash', compact + noncompact)
+        else:
+            spend_digest = self._hash(b'ZTxIdSSpendsHash')
+
+        if outputs:
+            compact = self._hash(
+                b'ZTxIdSOutC__Hash',
+                b''.join(output[32:148] for output in outputs),
+            )
+            memos = self._hash(
+                b'ZTxIdSOutM__Hash',
+                b''.join(output[148:660] for output in outputs),
+            )
+            noncompact = self._hash(
+                b'ZTxIdSOutN__Hash',
+                b''.join(output[:32] + output[660:] for output in outputs),
+            )
+            output_digest = self._hash(b'ZTxIdSOutputHash', compact + memos + noncompact)
+        else:
+            output_digest = self._hash(b'ZTxIdSOutputHash')
+        return self._hash(b'ZTxIdSaplingHash',
+                          spend_digest + output_digest + value_balance_bytes), value_balance
+
+    def _zcash_action_bundle(self, version, ironwood=False):
+        action_count = self._zcash_varint()
+        if action_count > (self._binary_length - self.cursor) // 820:
+            raise ZcashDeserializeError('too many Orchard-protocol actions')
+        if ironwood:
+            top_person = b'ZTxIdIronwd_H_v6'
+            compact_person = b'ZTxIdIrnActCH_v6'
+            memo_person = b'ZTxIdIrnActMH_v6'
+            noncompact_person = b'ZTxIdIrnActNH_v6'
+        elif version == 5:
+            top_person = b'ZTxIdOrchardHash'
+            compact_person = b'ZTxIdOrcActCHash'
+            memo_person = b'ZTxIdOrcActMHash'
+            noncompact_person = b'ZTxIdOrcActNHash'
+        else:
+            top_person = b'ZTxIdOrchardH_v6'
+            compact_person = b'ZTxIdOrcActCHash'
+            memo_person = b'ZTxIdOrcActMHash'
+            noncompact_person = b'ZTxIdOrcActNHash'
+
+        if not action_count:
+            return self._hash(top_person), 0
+        actions = [self._zcash_read(820) for _ in range(action_count)]
+        flags_and_balance = self._zcash_read(9)
+        flags = flags_and_balance[0]
+        if flags & 0xf8 or (not ironwood and flags & 0x04):
+            raise ZcashDeserializeError('invalid Orchard-protocol flags')
+        value_balance = int.from_bytes(flags_and_balance[1:], 'little', signed=True)
+        anchor = self._zcash_read(32)
+        self._zcash_varbytes()
+        self._zcash_read(64 * action_count)
+        self._zcash_read(64)
+
+        compact = self._hash(
+            compact_person,
+            b''.join(action[32:64] + action[96:212] for action in actions),
+        )
+        memos = self._hash(
+            memo_person,
+            b''.join(action[212:724] for action in actions),
+        )
+        noncompact = self._hash(
+            noncompact_person,
+            b''.join(action[:32] + action[64:96] + action[724:] for action in actions),
+        )
+        data = compact + memos + noncompact + flags_and_balance
+        if version == 5:
+            data += anchor
+        return self._hash(top_person, data), value_balance
+
+    def _read_v5_or_v6(self, version, header):
+        version_group_id = int.from_bytes(self._zcash_read(4), 'little')
+        expected_group_id = (self._V5_VERSION_GROUP_ID if version == 5
+                             else self._V6_VERSION_GROUP_ID)
+        if version_group_id != expected_group_id:
+            raise ZcashDeserializeError('unknown Zcash transaction format')
+        branch_id_bytes = self._zcash_read(4)
+        consensus_branch_id = int.from_bytes(branch_id_bytes, 'little')
+        locktime = int.from_bytes(self._zcash_read(4), 'little')
+        expiry_height = int.from_bytes(self._zcash_read(4), 'little')
+        # This is the NU6.3 v6 layout.  ZIP-233, which Zebra does not currently
+        # enable, inserts an 8-byte zip233Amount here.  Its value must be committed
+        # to header_digest and subtracted from fee_adjustment.  Do not silently parse
+        # ZIP-233 transactions using this layout; add an activation-aware branch when
+        # that consensus change is deployed.
+        header_digest = self._hash(b'ZTxIdHeadersHash',
+                                   header + version_group_id.to_bytes(4, 'little') +
+                                   branch_id_bytes + locktime.to_bytes(4, 'little') +
+                                   expiry_height.to_bytes(4, 'little'))
+        inputs, outputs, _scripts, transparent_digest = self._zcash_transparent()
+        sapling_digest, sapling_balance = self._zcash_sapling(version)
+        orchard_digest, orchard_balance = self._zcash_action_bundle(version)
+        digests = header_digest + transparent_digest + sapling_digest + orchard_digest
+        ironwood_balance = 0
+        if version == 6:
+            ironwood_digest, ironwood_balance = self._zcash_action_bundle(version, ironwood=True)
+            digests += ironwood_digest
+        root_person = b'ZcashTxHash_' + branch_id_bytes
+        txid = self._hash(root_person, digests)
+        return TxZcash(
+            version=version,
+            inputs=inputs,
+            outputs=outputs,
+            locktime=locktime,
+            txid_rev=txid,
+            wtxid_rev=txid,
+            version_group_id=version_group_id,
+            consensus_branch_id=consensus_branch_id,
+            expiry_height=expiry_height,
+            fee_adjustment=sapling_balance + orchard_balance + ironwood_balance,
+        )
+
     def read_tx(self):
         orig_start = self.cursor
         start = self.cursor
-        header = self._read_le_uint32()
+        header = int.from_bytes(self._zcash_read(4), 'little')
         overwintered = ((header >> 31) == 1)
         if overwintered:
             version = header & 0x7fffffff
-            self.cursor += 4  # versionGroupId
+            self._zcash_read(4)  # versionGroupId
         else:
             version = header
 
+        if version in (5, 6):
+            self.cursor = orig_start
+            header_bytes = self._zcash_read(4)
+            return self._read_v5_or_v6(version, header_bytes)
+
         is_overwinter_v3 = version == 3
         is_sapling_v4 = version == 4