netcraft
/
litecoin
mirror of https://github.com/litecoin-project/litecoin
6
0
Fork 0
Code Issues Projects Releases Wiki Activity

Make linearize scripts Python 3-compatible.

Browse Source
pull/308/head
Doug 8 years ago committed by Douglas Roark
parent d5aa19813c
commit 3c8f63ba7c
3 changed files with 47 additions and 26 deletions
Show Stats Download Patch File Download Diff File

13
contrib/linearize/README.md
Unescape View File

@ -1,5 +1,6 @@
# Linearize
Construct a linear, no-fork, best version of the Bitcoin blockchain.
Construct a linear, no-fork, best version of the Bitcoin blockchain. The scripts
run using Python 3 but are compatible with Python 2.
## Step 1: Download hash list
@ -9,7 +10,8 @@ Required configuration file settings for linearize-hashes:
* RPC: `rpcuser`, `rpcpassword`
Optional config file setting for linearize-hashes:
* RPC: `host`, `port` (Default: `127.0.0.1:8332`)
* RPC: `host` (Default: `127.0.0.1`)
* RPC: `port` (Default: `8332`)
* Blockchain: `min_height`, `max_height`
* `rev_hash_bytes`: If true, the written block hash list will be
byte-reversed. (In other words, the hash returned by getblockhash will have its
@ -17,6 +19,9 @@ bytes reversed.) False by default. Intended for generation of
standalone hash lists but safe to use with linearize-data.py, which will output
the same data no matter which byte format is chosen.
The `linearize-hashes` script requires a connection, local or remote, to a
JSON-RPC server. Running `bitcoind` or `bitcoin-qt -server` will be sufficient.
## Step 2: Copy local block data
$ ./linearize-data.py linearize.cfg
@ -24,13 +29,13 @@ the same data no matter which byte format is chosen.
Required configuration file settings:
* `output_file`: The file that will contain the final blockchain.
or
* `output`: Output directory for linearized blocks/blkNNNNN.dat output.
* `output`: Output directory for linearized `blocks/blkNNNNN.dat` output.
Optional config file setting for linearize-data:
* `file_timestamp`: Set each file's last-modified time to that of the most
recent block in that file.
* `genesis`: The hash of the genesis block in the blockchain.
* `input: bitcoind blocks/ directory containing blkNNNNN.dat
* `input`: bitcoind blocks/ directory containing blkNNNNN.dat
* `hashlist`: text file containing list of block hashes created by
linearize-hashes.py.
* `max_out_sz`: Maximum size for files created by the `output_file` option.

28
contrib/linearize/linearize-data.py
Unescape View File

@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python3
#
# linearize-data.py: Construct a linear, no-fork version of the chain.
#
@ -8,29 +8,33 @@
#
from __future__ import print_function, division
try: # Python 3
import http.client as httplib
except ImportError: # Python 2
import httplib
import json
import struct
import re
import os
import os.path
import base64
import httplib
import sys
import hashlib
import datetime
import time
from collections import namedtuple
from binascii import hexlify, unhexlify
settings = {}
##### Switch endian-ness #####
def hex_switchEndian(s):
""" Switches the endianness of a hex string (in pairs of hex chars) """
pairList = [s[i]+s[i+1] for i in range(0,len(s),2)]
return ''.join(pairList[::-1])
pairList = [s[i:i+2].encode() for i in range(0, len(s), 2)]
return b''.join(pairList[::-1]).decode()
def uint32(x):
return x & 0xffffffffL
return x & 0xffffffff
def bytereverse(x):
return uint32(( ((x) << 24) | (((x) << 8) & 0x00ff0000) |
@ -41,14 +45,14 @@ def bufreverse(in_buf):
for i in range(0, len(in_buf), 4):
word = struct.unpack('@I', in_buf[i:i+4])[0]
out_words.append(struct.pack('@I', bytereverse(word)))
return ''.join(out_words)
return b''.join(out_words)
def wordreverse(in_buf):
out_words = []
for i in range(0, len(in_buf), 4):
out_words.append(in_buf[i:i+4])
out_words.reverse()
return ''.join(out_words)
return b''.join(out_words)
def calc_hdr_hash(blk_hdr):
hash1 = hashlib.sha256()
@ -65,7 +69,7 @@ def calc_hash_str(blk_hdr):
hash = calc_hdr_hash(blk_hdr)
hash = bufreverse(hash)
hash = wordreverse(hash)
hash_str = hash.encode('hex')
hash_str = hexlify(hash).decode('utf-8')
return hash_str
def get_blk_dt(blk_hdr):
@ -217,7 +221,7 @@ class BlockDataCopier:
inMagic = inhdr[:4]
if (inMagic != self.settings['netmagic']):
print("Invalid magic: " + inMagic.encode('hex'))
print("Invalid magic: " + hexlify(inMagic).decode('utf-8'))
return
inLenLE = inhdr[4:]
su = struct.unpack("<I", inLenLE)
@ -294,14 +298,14 @@ if __name__ == '__main__':
if 'split_timestamp' not in settings:
settings['split_timestamp'] = 0
if 'max_out_sz' not in settings:
settings['max_out_sz'] = 1000L * 1000 * 1000
settings['max_out_sz'] = 1000 * 1000 * 1000
if 'out_of_order_cache_sz' not in settings:
settings['out_of_order_cache_sz'] = 100 * 1000 * 1000
settings['max_out_sz'] = long(settings['max_out_sz'])
settings['max_out_sz'] = int(settings['max_out_sz'])
settings['split_timestamp'] = int(settings['split_timestamp'])
settings['file_timestamp'] = int(settings['file_timestamp'])
settings['netmagic'] = settings['netmagic'].decode('hex')
settings['netmagic'] = unhexlify(settings['netmagic'].encode('utf-8'))
settings['out_of_order_cache_sz'] = int(settings['out_of_order_cache_sz'])
if 'output_file' not in settings and 'output' not in settings:

32
contrib/linearize/linearize-hashes.py
Unescape View File

@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python3
#
# linearize-hashes.py: List blocks in a linear, no-fork version of the chain.
#
@ -8,11 +8,14 @@
#
from __future__ import print_function
try: # Python 3
import http.client as httplib
except ImportError: # Python 2
import httplib
import json
import struct
import re
import base64
import httplib
import sys
settings = {}
@ -20,26 +23,32 @@ settings = {}
##### Switch endian-ness #####
def hex_switchEndian(s):
""" Switches the endianness of a hex string (in pairs of hex chars) """
pairList = [s[i]+s[i+1] for i in range(0,len(s),2)]
return ''.join(pairList[::-1])
pairList = [s[i:i+2].encode() for i in range(0, len(s), 2)]
return b''.join(pairList[::-1]).decode()
class BitcoinRPC:
def __init__(self, host, port, username, password):
authpair = "%s:%s" % (username, password)
self.authhdr = "Basic %s" % (base64.b64encode(authpair))
self.conn = httplib.HTTPConnection(host, port, False, 30)
authpair = authpair.encode('utf-8')
self.authhdr = b"Basic " + base64.b64encode(authpair)
self.conn = httplib.HTTPConnection(host, port=port, timeout=30)
def execute(self, obj):
self.conn.request('POST', '/', json.dumps(obj),
{ 'Authorization' : self.authhdr,
'Content-type' : 'application/json' })
try:
self.conn.request('POST', '/', json.dumps(obj),
{ 'Authorization' : self.authhdr,
'Content-type' : 'application/json' })
except ConnectionRefusedError:
print('RPC connection refused. Check RPC settings and the server status.',
file=sys.stderr)
return None
resp = self.conn.getresponse()
if resp is None:
print("JSON-RPC: no response", file=sys.stderr)
return None
body = resp.read()
body = resp.read().decode('utf-8')
resp_obj = json.loads(body)
return resp_obj
@ -70,6 +79,9 @@ def get_block_hashes(settings, max_blocks_per_call=10000):
batch.append(rpc.build_request(x, 'getblockhash', [height + x]))
reply = rpc.execute(batch)
if reply is None:
print('Cannot continue. Program will halt.')
return None
for x,resp_obj in enumerate(reply):
if rpc.response_is_error(resp_obj):

Write Preview
Loading…
Cancel
Save