Restructure to separate out consensus critical code

All consensus critical is now under bitcoin.core

Also improved the README

Author: petertodd

README

@@ -1,10 +1,44 @@
+python-bitcoinlib
+-----------------
 
-This python library provides an easy interface to the bitcoin
-data structures and protocol.
+This Python2/3 library provides an easy interface to the bitcoin data
+structures and protocol.
+
+
+Structure
+---------
+
+Everything consensus critical is found in the modules under bitcoin.core:
+
+bitcoin.core            - Basic datastructures, CTransaction, CBlock etc.
+bitcoin.core.bignum     - Bignum handling
+bitcoin.core.coredef    - Core definitions, MAX_MONEY, etc.
+bitcoin.core.key        - ECC pubkeys
+bitcoin.core.script     - Scripts and opcodes
+bitcoin.core.scripteval - Script evaluation/verification
+bitcoin.core.serialize  - Serialization
+
+In the future the bitcoin.core may use the Satoshi sourcecode directly as a
+libary. Non-consensus critical modules include the following:
+
+bitcoin.base58     - Base58 encoding
+bitcoin.bloom      - Bloom filters (incomplete)
+bitcoin.hash       - Non-cryptographic hashing
+bitcoin.net        - Network communication (broken currently)
+bitcoin.messages   - Network messages (broken currently)
+bitcoin.rpc        - Bitcoin Core RPC interface support
+bitcoin.wallet     - Wallet code, currently just CBitcoinAddress
+
+Effort has been made to follow the Satoshi source relatively closely, for
+instance Python code and classes that duplicate the functionality of
+corresponding Satoshi C++ code uses the same naming conventions: CTransaction,
+CBlockHeader, nValue etc. Otherwise Python naming conventions are followed.
 
 
 Unit tests
 ----------
 
+Under bitcoin/tests using test data from Bitcoin Core. To run them:
+
 python -m unittest discover
 python3 -m unittest discover

bitcoin/base58.py

@@ -18,11 +18,11 @@
         bchr = lambda x: bytes([x])
         bord = lambda x: x
 
-from bitcoin.serialize import Hash
+import binascii
 
-b58_digits = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz'
+import bitcoin.core
 
-from binascii import hexlify, unhexlify
+b58_digits = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz'
 
 class Base58Error(Exception):
     pass
@@ -34,7 +34,7 @@ def encode(b):
     """Encode bytes to a base58-encoded string"""
 
     # Convert big-endian bytes to integer
-    n = int('0x0' + hexlify(b).decode('utf8'), 16)
+    n = int('0x0' + binascii.hexlify(b).decode('utf8'), 16)
 
     # Divide that integer into bas58
     res = []
@@ -73,7 +73,7 @@ def decode(s):
     h = '%x' % n
     if len(h) % 2:
         h = '0' + h
-    res = unhexlify(h.encode('utf8'))
+    res = binascii.unhexlify(h.encode('utf8'))
 
     # Add padding back.
     pad = 0
@@ -90,7 +90,7 @@ class CBase58Data(bytes):
     def __new__(cls, s):
         k = decode(s)
         addrbyte, data, check0 = k[0:1], k[1:-4], k[-4:]
-        check1 = Hash(addrbyte + data)[:4]
+        check1 = bitcoin.core.Hash(addrbyte + data)[:4]
         if check0 != check1:
             raise Base58ChecksumError('Checksum mismatch: expected %r, calculated %r' % (check0, check1))
         return cls.from_bytes(data, ord(addrbyte))
@@ -106,7 +106,7 @@ def to_bytes(self):
 
     def __str__(self):
         vs = bchr(self.nVersion) + self
-        check = Hash(vs)[0:4]
+        check = bitcoin.core.Hash(vs)[0:4]
         return encode(vs + check)
 
     def __repr__(self):

bitcoin/bloom.py

@@ -11,12 +11,12 @@
 import struct
 import sys
 import math
-from bitcoin.serialize import *
-from bitcoin.coredefs import *
-from bitcoin.core import *
-from bitcoin.hash import MurmurHash3
 
-class CBloomFilter(Serializable):
+import bitcoin.core
+import bitcoin.core.serialize
+import bitcoin.hash
+
+class CBloomFilter(bitcoin.core.serialize.Serializable):
     # 20,000 items with fp rate < 0.1% or 10,000 items and <0.0001%
     MAX_BLOOM_FILTER_SIZE = 36000
     MAX_HASH_FUNCS = 50
@@ -51,15 +51,15 @@ def __init__(self, nElements, nFPRate, nTweak, nFlags):
         self.nFlags = nFlags
 
     def bloom_hash(self, nHashNum, vDataToHash):
-        return MurmurHash3(((nHashNum * 0xFBA4C795) + self.nTweak) & 0xFFFFFFFF, vDataToHash) % (len(self.vData) * 8)
+        return bitcoin.hash.MurmurHash3(((nHashNum * 0xFBA4C795) + self.nTweak) & 0xFFFFFFFF, vDataToHash) % (len(self.vData) * 8)
 
     __bit_mask = bytearray([0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80])
     def insert(self, elem):
         """Insert an element in the filter.
 
         elem may be a COutPoint or bytes
         """
-        if isinstance(elem, COutPoint):
+        if isinstance(elem, bitcoin.core.COutPoint):
             elem = elem.serialize()
 
         if len(self.vData) == 1 and self.vData[0] == 0xff:
@@ -75,7 +75,7 @@ def contains(self, elem):
 
         elem may be a COutPoint or bytes
         """
-        if isinstance(elem, COutPoint):
+        if isinstance(elem, bitcoin.core.COutPoint):
             elem = elem.serialize()
 
         if len(self.vData) == 1 and self.vData[0] == 0xff:
@@ -97,7 +97,7 @@ def IsRelevantAndUpdate(tx, tx_hash):
     __struct = struct.Struct(b'<IIB')
     @classmethod
     def stream_deserialize(cls, f):
-        vData = BytesSerializer.stream_deserialize(f)
+        vData = bitcoin.core.serialize.BytesSerializer.stream_deserialize(f)
         (nHashFuncs,
          nTweak,
          nFlags) = self.__struct.unpack(_ser_read(f, self.__struct.size))
@@ -110,8 +110,8 @@ def stream_deserialize(cls, f):
 
     def stream_serialize(self, f):
         if sys.version > '3':
-            BytesSerializer.stream_serialize(self.vData, f)
+            bitcoin.core.serialize.BytesSerializer.stream_serialize(self.vData, f)
         else:
             # 2.7 has problems with f.write(bytearray())
-            BytesSerializer.stream_serialize(bytes(self.vData), f)
+            bitcoin.core.serialize.BytesSerializer.stream_serialize(bytes(self.vData), f)
         f.write(self.__struct.pack(self.nHashFuncs, self.nTweak, self.nFlags))

bitcoin/core.py bitcoin/core/__init__.py

@@ -12,11 +12,11 @@
 import socket
 import binascii
 import hashlib
-import bitcoin.base58 as base58
-import bitcoin.script as script
 
-from bitcoin.serialize import *
-from bitcoin.coredefs import *
+from .script import CScript
+
+from .serialize import *
+from .coredefs import *
 
 def x(h):
     """Convert a hex string to bytes"""
@@ -64,85 +64,6 @@ def str_money_value(value):
         r += '0'
     return r
 
-class CBitcoinAddress(base58.CBase58Data):
-    """A Bitcoin address"""
-    PUBKEY_ADDRESS = 0
-    SCRIPT_ADDRESS = 5
-    PUBKEY_ADDRESS_TEST = 111
-    SCRIPT_ADDRESS_TEST = 196
-
-    def to_scriptPubKey(self):
-        """Convert an address to a scriptPubKey"""
-        if self.nVersion in (self.PUBKEY_ADDRESS, self.PUBKEY_ADDRESS_TEST):
-            return script.CScript([script.OP_DUP, script.OP_HASH160, self, script.OP_EQUALVERIFY, script.OP_CHECKSIG])
-
-        elif self.nVersion in (self.SCRIPT_ADDRESS, self.SCRIPT_ADDRESS_TEST):
-            return script.CScript([script.OP_HASH160, self, script.OP_EQUAL])
-
-        else:
-            raise ValueError("CBitcoinAddress: Don't know how to convert version %d to a scriptPubKey" % self.nVersion)
-
-
-class CAddress(object):
-    def __init__(self, protover=PROTO_VERSION):
-        self.protover = protover
-        self.nTime = 0
-        self.nServices = 1
-        self.pchReserved = b"\x00" * 10 + b"\xff" * 2
-        self.ip = "0.0.0.0"
-        self.port = 0
-    def deserialize(self, f):
-        if self.protover >= CADDR_TIME_VERSION:
-            self.nTime = struct.unpack(b"<I", ser_read(f,4))[0]
-        self.nServices = struct.unpack(b"<Q", ser_read(f,8))[0]
-        self.pchReserved = ser_read(f,12)
-        self.ip = socket.inet_ntoa(ser_read(f,4))
-        self.port = struct.unpack(b">H", ser_read(f,2))[0]
-    def serialize(self):
-        r = b""
-        if self.protover >= CADDR_TIME_VERSION:
-            r += struct.pack(b"<I", self.nTime)
-        r += struct.pack(b"<Q", self.nServices)
-        r += self.pchReserved
-        r += socket.inet_aton(self.ip)
-        r += struct.pack(b">H", self.port)
-        return r
-    def __repr__(self):
-        return "CAddress(nTime=%d nServices=%i ip=%s port=%i)" % (self.nTime, self.nServices, self.ip, self.port)
-
-class CInv(object):
-    typemap = {
-        0: "Error",
-        1: "TX",
-        2: "Block"}
-    def __init__(self):
-        self.type = 0
-        self.hash = 0
-    def deserialize(self, f):
-        self.type = struct.unpack(b"<i", ser_read(f,4))[0]
-        self.hash = ser_read(f,32)
-    def serialize(self):
-        r = b""
-        r += struct.pack(b"<i", self.type)
-        r += self.hash
-        return r
-    def __repr__(self):
-        return "CInv(type=%s hash=%064x)" % (self.typemap[self.type], self.hash)
-
-class CBlockLocator(object):
-    def __init__(self):
-        self.nVersion = PROTO_VERSION
-        self.vHave = []
-    def deserialize(self, f):
-        self.nVersion = struct.unpack(b"<i", ser_read(f,4))[0]
-        self.vHave = deser_uint256_vector(f)
-    def serialize(self):
-        r = b""
-        r += struct.pack(b"<i", self.nVersion)
-        r += ser_uint256_vector(self.vHave)
-        return r
-    def __repr__(self):
-        return "CBlockLocator(nVersion=%i vHave=%s)" % (self.nVersion, repr(self.vHave))
 
 class COutPoint(Serializable):
     """The combination of a transaction hash and an index n into its vout"""
@@ -184,7 +105,7 @@ class CTxIn(Serializable):
     """
     __slots__ = ['prevout', 'scriptSig', 'nSequence']
 
-    def __init__(self, prevout=None, scriptSig=script.CScript(), nSequence = 0xffffffff):
+    def __init__(self, prevout=None, scriptSig=CScript(), nSequence = 0xffffffff):
         if prevout is None:
             prevout = COutPoint()
         self.prevout = prevout
@@ -377,69 +298,6 @@ def calc_merkle_root(self):
             hashes.append(Hash(tx.serialize()))
         return CBlock.calc_merkle_root_from_hashes(hashes)
 
-class CUnsignedAlert(object):
-    def __init__(self):
-        self.nVersion = 1
-        self.nRelayUntil = 0
-        self.nExpiration = 0
-        self.nID = 0
-        self.nCancel = 0
-        self.setCancel = []
-        self.nMinVer = 0
-        self.nMaxVer = 0
-        self.setSubVer = []
-        self.nPriority = 0
-        self.strComment = b""
-        self.strStatusBar = b""
-        self.strReserved = b""
-    def deserialize(self, f):
-        self.nVersion = struct.unpack(b"<i", ser_read(f,4))[0]
-        self.nRelayUntil = struct.unpack(b"<q", ser_read(f,8))[0]
-        self.nExpiration = struct.unpack(b"<q", ser_read(f,8))[0]
-        self.nID = struct.unpack(b"<i", ser_read(f,4))[0]
-        self.nCancel = struct.unpack(b"<i", ser_read(f,4))[0]
-        self.setCancel = deser_int_vector(f)
-        self.nMinVer = struct.unpack(b"<i", ser_read(f,4))[0]
-        self.nMaxVer = struct.unpack(b"<i", ser_read(f,4))[0]
-        self.setSubVer = deser_string_vector(f)
-        self.nPriority = struct.unpack(b"<i", ser_read(f,4))[0]
-        self.strComment = deser_string(f)
-        self.strStatusBar = deser_string(f)
-        self.strReserved = deser_string(f)
-    def serialize(self):
-        r = b""
-        r += struct.pack(b"<i", self.nVersion)
-        r += struct.pack(b"<q", self.nRelayUntil)
-        r += struct.pack(b"<q", self.nExpiration)
-        r += struct.pack(b"<i", self.nID)
-        r += struct.pack(b"<i", self.nCancel)
-        r += ser_int_vector(self.setCancel)
-        r += struct.pack(b"<i", self.nMinVer)
-        r += struct.pack(b"<i", self.nMaxVer)
-        r += ser_string_vector(self.setSubVer)
-        r += struct.pack(b"<i", self.nPriority)
-        r += ser_string(self.strComment)
-        r += ser_string(self.strStatusBar)
-        r += ser_string(self.strReserved)
-        return r
-    def __repr__(self):
-        return "CUnsignedAlert(nVersion %d, nRelayUntil %d, nExpiration %d, nID %d, nCancel %d, nMinVer %d, nMaxVer %d, nPriority %d, strComment %s, strStatusBar %s, strReserved %s)" % (self.nVersion, self.nRelayUntil, self.nExpiration, self.nID, self.nCancel, self.nMinVer, self.nMaxVer, self.nPriority, self.strComment, self.strStatusBar, self.strReserved)
-
-class CAlert(object):
-    def __init__(self):
-        self.vchMsg = b""
-        self.vchSig = b""
-    def deserialize(self, f):
-        self.vchMsg = deser_string(f)
-        self.vchSig = deser_string(f)
-    def serialize(self):
-        r = b""
-        r += ser_string(self.vchMsg)
-        r += ser_string(self.vchSig)
-        return r
-    def __repr__(self):
-        return "CAlert(vchMsg.sz %d, vchSig.sz %d)" % (len(self.vchMsg), len(self.vchSig))
-
 
 class CheckTransactionError(ValueError):
     pass

bitcoin/bignum.py bitcoin/core/bignum.py

@@ -6,6 +6,8 @@
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 #
 
+"""Bignum routines"""
+
 from __future__ import absolute_import, division, print_function, unicode_literals
 
 import struct

bitcoin/coredefs.py bitcoin/core/coredefs.py

@@ -15,14 +15,14 @@
 NID_secp256k1 = 714 # from openssl/obj_mac.h
 
 # Thx to Sam Devlin for the ctypes magic 64-bit fix.
-def check_result (val, func, args):
+def _check_result (val, func, args):
     if val == 0:
         raise ValueError
     else:
         return ctypes.c_void_p (val)
 
 ssl.EC_KEY_new_by_curve_name.restype = ctypes.c_void_p
-ssl.EC_KEY_new_by_curve_name.errcheck = check_result
+ssl.EC_KEY_new_by_curve_name.errcheck = _check_result
 
 class CKey:
     POINT_CONVERSION_COMPRESSED = 2