diff --git a/.env b/.env index ebf2b0d7..56a4f57d 100644 --- a/.env +++ b/.env @@ -3,15 +3,15 @@ CHAIN_ID=714 KEYPASS="0123456789" INIT_HOLDER="0x04d63aBCd2b9b1baa327f2Dda0f873F197ccd186" # INIT_HOLDER_PRV="59ba8068eb256d520179e903f43dacf6d8d57d72bd306e1bd603fdb8c8da10e8" -RPC_URL="http://127.0.0.1:8545" +RPC_URL="http://bsc-node-0:8545" GENESIS_COMMIT="34618f607f8356cf147dde6a69fae150bd53d5bf" # fermi commit -PASSED_FORK_DELAY=40 -LAST_FORK_MORE_DELAY=10 +PASSED_FORK_DELAY=-100000000 +LAST_FORK_MORE_DELAY=0 FullImmutabilityThreshold=2048 MinBlocksForBlobRequests=576 DefaultExtraReserveForBlobRequests=32 BreatheBlockInterval=1200 -useLatestBscClient=false +useLatestBscClient=true EnableSentryNode=false EnableFullNode=false RegisterNodeID=false diff --git a/.gitignore b/.gitignore index ab57b06f..27fe633a 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,6 @@ .idea/ lib/ +bsc/ +docker-compose.cluster.yml +.env.cluster \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..43649e73 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,81 @@ +# Multi-stage Dockerfile for BSC Node +# Stage 1: Build Environment with all required tools +# We use Python 3.12 as the base to fulfill the requirement, then add Go 1.24 +FROM python:3.12-bookworm AS build-env + +# Arguments for versions +ARG NODE_VERSION=16.x +ARG GO_VERSION=1.24.0 + +# Install Go 1.24 (Detect architecture to support both x86_64 and arm64) +RUN ARCH=$(uname -m) && \ + if [ "$ARCH" = "x86_64" ]; then GO_ARCH="amd64"; else GO_ARCH="arm64"; fi && \ + curl -fsSL https://go.dev/dl/go${GO_VERSION}.linux-${GO_ARCH}.tar.gz | tar -C /usr/local -xz +ENV PATH="/usr/local/go/bin:$PATH" + +# Install basic dependencies +RUN apt-get update && apt-get install -y \ + curl \ + git \ + make \ + jq \ + build-essential \ + software-properties-common \ + && rm -rf /var/lib/apt/lists/* + +# Install Node.js (README mentions v16.15.0) +RUN curl -fsSL https://deb.nodesource.com/setup_${NODE_VERSION} | bash - && \ + apt-get install -y nodejs && \ + npm install -g npm@6.14.6 + +# Install Poetry using its official installer +# (Python 3.12 is already the system default in this image) +RUN curl -sSL https://install.python-poetry.org | python3 - +ENV PATH="/root/.local/bin:$PATH" + +# Install Foundry (Ethereum development toolkit) +# This is needed because BSC's system contracts (in the genesis/ folder) use 'forge' to compile. +RUN curl -L https://foundry.paradigm.xyz | bash && \ + /root/.foundry/bin/foundryup +ENV PATH="/root/.foundry/bin:$PATH" +ENV PATH="/node_deploy/bin:$PATH" + +# Setting up the workspace +WORKDIR /node_deploy + +# Copy and install Python requirements +# This avoids installing them manually every time you restart the container. +COPY requirements.txt . +RUN pip3 install --no-cache-dir -r requirements.txt + +# Build and install the validator tool +# By installing to /usr/local/bin, we ensure the binary is available even if the +# /node_deploy directory is shadowed by a volume mount from the host. +COPY create-validator/ ./create-validator/ +RUN cd create-validator && go build -o /usr/local/bin/create-validator + +# Copy entrypoint script +COPY entrypoint.sh /usr/local/bin/entrypoint.sh +RUN chmod +x /usr/local/bin/entrypoint.sh + +# Tool verification command +# This ensures that when the image is built, all required tools are present and functional. +RUN go version && \ + node -v && \ + npm -v && \ + python3 --version && \ + poetry --version && \ + forge --version && \ + jq --version && \ + create-validator --help + +# Add init check to .bashrc to warn users when they login +RUN echo 'if [ -f /tmp/cluster_initializing ]; then' >> /root/.bashrc && \ + echo ' echo "----------------------------------------------------"' >> /root/.bashrc && \ + echo ' echo " WARNING: Cluster is still initializing. Please wait! "' >> /root/.bashrc && \ + echo ' echo "----------------------------------------------------"' >> /root/.bashrc && \ + echo ' echo ""' >> /root/.bashrc && \ + echo 'fi' >> /root/.bashrc + +# Default command: show help or just stay open +CMD ["bash"] diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..f8314593 --- /dev/null +++ b/Makefile @@ -0,0 +1,45 @@ +# Makefile for BSC Local Cluster + +.PHONY: cluster-up cluster-down cluster-logs cluster-clean cluster-restart + +# Default Image to use for bootstrapping +TOOLBOX_IMAGE ?= bsc-toolbox:latest +COMPOSE_FILE := docker-compose.cluster.yml + +# Auto initialize and bring up the cluster +cluster-up: check-deps + @echo "[Phase 1] Initializing blockchain data & configs using isolated Toolbox environment..." + docker run --rm -v "$(CURDIR):/node_deploy" -w /node_deploy $(TOOLBOX_IMAGE) bash docker_cluster.sh prepare + @echo "" + @echo "[Phase 2] Data prepared! Starting BSC cluster via Docker Compose..." + docker compose -f "$(COMPOSE_FILE)" up -d + @echo "[Phase 3] Waiting for RPC... then Registering Validators" + @bash docker_cluster.sh wait-rpc + @docker run --network bsc_cluster_network --rm -v "$(CURDIR):/node_deploy" -w /node_deploy $(TOOLBOX_IMAGE) bash docker_cluster.sh register + @echo "BSC Local Cluster successfully started in background! Run 'make cluster-logs' to view live logs." + +# Safely stop and remove all containers +cluster-down: + @echo "Stopping and removing all BSC containers..." + if [ -f "$(COMPOSE_FILE)" ]; then docker compose -f "$(COMPOSE_FILE)" down; fi + +# View real-time logs for all cluster nodes +cluster-logs: + if [ -f "$(COMPOSE_FILE)" ]; then docker compose -f "$(COMPOSE_FILE)" logs -f; fi + +# Completely wipe cluster data and auto-generated configs (DANGEROUS) +cluster-clean: cluster-down + @echo "Wiping all local cluster data and temporary configurations (.local/, YAML, .env)..." + rm -rf "$(CURDIR)/.local" + rm -f "$(CURDIR)/.env.cluster" "$(COMPOSE_FILE)" + @echo "Workspace is completely clean." + +# Fast restart without wiping data or rebuilding config +cluster-restart: cluster-down + @echo "Restarting all BSC containers with existing config (Phase 2 only)..." + if [ -f "$(COMPOSE_FILE)" ]; then docker compose -f "$(COMPOSE_FILE)" up -d; fi + @echo "BSC Local Cluster successfully restarted." + +# Ensure host has curl for wait-rpc (Phase 3) +check-deps: + @command -v curl >/dev/null 2>&1 || (echo "ERROR: 'curl' not found on host. It is required for Phase 3!" && exit 1) diff --git a/README.md b/README.md index cf348fcc..d136e72c 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,10 @@ ## Installation Before proceeding to the next steps, please ensure that the following packages and softwares are well installed in your local machine: + +> [!TIP] +> **Don't want to install these manually?** You can skip to the [Docker Version](#docker-version-recommended) section at the bottom of this file. + - nodejs: v16.15.0 - npm: 6.14.6 - go: 1.24+ @@ -87,4 +91,83 @@ go build cd txblob go build ./txblob -``` \ No newline at end of file +``` + +## Docker Version (Recommended) + +To run a fully containerized, isolated local BSC cluster without installing dependencies on your host machine, use the provided `Makefile` which handles the 2-phase orchestration automatically. + +### Architecture Workflow + +```mermaid +sequenceDiagram + participant User + participant Makefile + participant Toolbox as Toolbox (Docker) + participant HostFS as Host FileSystem + participant Compose as Docker Compose + participant Docker as Docker Engine + + User->>Makefile: make cluster-up + + Note over Makefile,Toolbox: Phase 1: Initialization (prepare) + Makefile->>Toolbox: Start disposable 'bsc-toolbox' container and execute script + + Toolbox->>HostFS: Compile and save 'geth' binary + Toolbox->>HostFS: Generate genesis, keystores, config.toml (.local/) + Toolbox->>HostFS: Generate .env.cluster (cluster params, node count) + Toolbox->>HostFS: Generate docker-compose.cluster.yml (based on env) + + Toolbox-->>Makefile: Exit (container removed) + + Note over Makefile,Compose: Phase 2: Start Cluster (up) + Makefile->>Compose: docker compose -f docker-compose.cluster.yml up -d + + Compose->>HostFS: Read docker-compose.cluster.yml + Compose->>HostFS: Load .env.cluster (env injection) + + Compose->>Docker: Create & start N bsc-node-X containers + + Docker->>HostFS: Mount volumes (./ -> /node_deploy) + + Docker->>Docker: Run node_entrypoint.sh inside each container + Docker->>HostFS: Containers read config.toml / genesis / keystore + + Docker-->>User: Cluster running (N nodes) + + Note over Makefile,Toolbox: Phase 3: Validator Registration (register) + Makefile->>Makefile: Wait for RPC (localhost:8545 ready) + Makefile->>Toolbox: Start new Toolbox container inside 'bsc_cluster_network' + Toolbox->>HostFS: Load .env (get RPC_URL) + Toolbox->>Docker: Send 'Register' Transactions (via RPC to Node 0) + Toolbox-->>Makefile: Exit (registration tasks submitted) + + Note over User,Docker: Local BSC Cluster active with registered validators +``` + +### Quick Commands + +- **`make cluster-up`**: One-click start. It runs the initialization phase, starts the isolated nodes via Docker Compose, and then automatically handles validator registration on StakeHub. +- **`make cluster-down`**: Safely stop all running nodes. +- **`make cluster-logs`**: Stream aggregated, color-coded logs from all running nodes. +- **`make cluster-restart`**: Fast restart the cluster (nodes only). Use this if you manually modified `.local/nodeX/config.toml` and want to apply changes without wiping the blockchain data. +- **`make cluster-clean`**: **WARNING**. Wipes all generated data (`.local/`), genesis files, and temporary yaml configs. Use this to reset the chain back to block zero. + +### Node Ports Mapping + +Each node runs identically on port `8545` internally. Host mapping is structured sequentially: + +| Node | RPC (HTTP/WS) | Metrics (Prometheus) | pprof (Debug) | P2P (TCP/UDP) | +| :--- | :--- | :--- | :--- | :--- | +| **Node 0** | 8545 | 6060 | 7060 | 30311 | +| **Node 1** | 8547 | 6062 | 7062 | 30312 | +| **Node 2** | 8549 | 6064 | 7064 | 30313 | +| **Node 3** | 8551 | 6066 | 7066 | 30314 | + +For example, to check the block height of Node 1: +`curl -H "Content-Type: application/json" -X POST --data '{"jsonrpc":"2.0","method":"eth_blockNumber","params":[],"id":1}' http://127.0.0.1:8547` + +### Logging + +By default, nodes output all their logs directly to the Docker logging driver (STDOUT). You can view them using: +`docker logs -f bsc-node-0` \ No newline at end of file diff --git a/bsc_cluster.sh b/bsc_cluster.sh old mode 100644 new mode 100755 index c0c0fe91..8ccb6eba --- a/bsc_cluster.sh +++ b/bsc_cluster.sh @@ -16,13 +16,15 @@ gcmode="full" sleepBeforeStart=15 sleepAfterStart=10 -# stop geth client +# 1. Stop any previously running geth instances function exit_previous() { ValIdx=$1 - ps -ef | grep geth$ValIdx | grep config |awk '{print $2}' | xargs kill + # if no geth exist just skip it + ps -ef | grep geth$ValIdx | grep config |awk '{print $2}' | xargs -r kill sleep ${sleepBeforeStart} } +# 2. Cleanup .local and copy initial keys for validators function create_validator() { rm -rf ${workspace}/.local mkdir -p ${workspace}/.local @@ -33,6 +35,7 @@ function create_validator() { done } +# 3. Build bsc geth client from source if configured function prepare_bsc_client() { if [ ${useLatestBscClient} = true ]; then if [ ! -f "${workspace}/bsc/Makefile" ]; then @@ -42,13 +45,20 @@ function prepare_bsc_client() { cd ${workspace}/bsc && git pull && make geth && mv -f ${workspace}/bsc/build/bin/geth ${workspace}/bin/ fi } -# reset genesis, but keep edited genesis-template.json + +# 4. Reset genesis submodule and install dependencies (Poetry, NPM, Forge) +# This prepares the environment for generating the genesis block function reset_genesis() { if [ ! -f "${workspace}/genesis/genesis-template.json" ]; then cd ${workspace} && git submodule update --init --recursive genesis cd ${workspace}/genesis && git reset --hard ${GENESIS_COMMIT} fi cd ${workspace}/genesis + + # 1. Update the 'genesis' submodule safely + # Backup user templates, force-update the repository to a specific + # compatible version ($GENESIS_COMMIT) to prevent breaking changes, + # and then restore the user templates. cp genesis-template.json genesis-template.json.bk cp scripts/init_holders.template scripts/init_holders.template.bk git stash @@ -57,8 +67,11 @@ function reset_genesis() { mv genesis-template.json.bk genesis-template.json mv scripts/init_holders.template.bk scripts/init_holders.template + # 2. Install project-specific dependencies poetry install --no-root npm install + + # 3. Clean up and reinstall Foundry framework components (Standard Library & Tests) rm -rf lib/forge-std forge install --no-git foundry-rs/forge-std@v1.7.3 cd lib/forge-std/lib @@ -66,13 +79,17 @@ function reset_genesis() { git clone https://github.com/dapphub/ds-test } +# 5. Generate validator configurations, hardfork times, and the final genesis.json function prepare_config() { rm -f ${workspace}/genesis/validators.conf passedHardforkTime=$(expr $(date +%s) + ${PASSED_FORK_DELAY}) echo "passedHardforkTime "${passedHardforkTime} > ${workspace}/.local/hardforkTime.txt initHolders=${INIT_HOLDER} + + # 1. Collect Validator Information & Setup Node Directories for ((i = 0; i < size; i++)); do + # read their randomly generated cryptographic key files (Consensus addresses and BLS vote keys) for f in ${workspace}/.local/validator${i}/keystore/*; do cons_addr="0x$(cat ${f} | jq -r .address)" initHolders=${initHolders}","${cons_addr} @@ -84,11 +101,15 @@ function prepare_config() { cp ${workspace}/keys/password.txt ./ cp ${workspace}/.local/hardforkTime.txt ./ bbcfee_addrs=${fee_addr} + # It assigns a massive, hardcoded amount of "voting power" (0x000001d1a94a2000) so the node has the authority to forge blocks. powers="0x000001d1a94a2000" #2000000000000 mv ${workspace}/.local/bls${i}/bls ./ && rm -rf ${workspace}/.local/bls${i} vote_addr=0x$(cat ./bls/keystore/*json | jq .pubkey | sed 's/"//g') + # it writes all these unique peices: the consensus address, fee collection address, voting power, and BLS vote address echo "${cons_addr},${bbcfee_addrs},${fee_addr},${powers},${vote_addr}" >> ${workspace}/genesis/validators.conf if [ ${EnableSentryNode} = true ]; then + # Sentry nodes act as a protective firewall/proxy for Validators, + # hiding the Validator's real IP address from public P2P network attacks. mkdir -p ${workspace}/.local/sentry${i} fi done @@ -97,11 +118,15 @@ function prepare_config() { fi rm -f ${workspace}/.local/hardforkTime.txt + # 2. Hack / Patch the System Smart Contracts cd ${workspace}/genesis/ git checkout HEAD contracts + # "hack" the source code of the core BSCValidatorSet.sol smart contract right before compiling. + # They lower the turnLength and explicitly tell the system to ignore validator punishments/updates for the first 2,000 blocks to ensure your local network starts smoothly without validators instantly getting jailed. sed -i -e 's/alreadyInit = true;/turnLength = 16;alreadyInit = true;/' ${workspace}/genesis/contracts/BSCValidatorSet.sol sed -i -e 's/public onlyCoinbase onlyZeroGasPrice {/public onlyCoinbase onlyZeroGasPrice {if (block.number < 2000) return;/' ${workspace}/genesis/contracts/BSCValidatorSet.sol + # 3. Generate the Final Genesis Block poetry run python -m scripts.generate generate-validators poetry run python -m scripts.generate generate-init-holders "${initHolders}" poetry run python -m scripts.generate dev \ @@ -124,8 +149,10 @@ function prepare_config() { cp genesis-dev.json genesis.json } +# 6. Initialize the geth network for each node using the generated genesis.json function initNetwork() { cd ${workspace} + # 1. Assigning P2P Identities for ((i = 0; i < size; i++)); do mkdir ${workspace}/.local/node${i}/geth cp ${workspace}/keys/validator-nodekey${i} ${workspace}/.local/node${i}/geth/nodekey @@ -140,6 +167,7 @@ function initNetwork() { cp ${workspace}/keys/fullnode-nodekey0 ${workspace}/.local/fullnode0/geth/nodekey fi + # 2. Preparing Network Arguments init_extra_args="" if [ ${EnableSentryNode} = true ]; then init_extra_args="--init.sentrynode-size ${size} --init.sentrynode-ports 30411" @@ -161,23 +189,23 @@ function initNetwork() { init_extra_args="${init_extra_args} --init.evn-validator-whitelist" fi fi + + # 3. Generating Network Configs (config.toml) ${workspace}/bin/geth init-network --init.dir ${workspace}/.local --init.size=${size} --config ${workspace}/config.toml ${init_extra_args} ${workspace}/genesis/genesis.json rm -f ${workspace}/*bsc.log* + + # 4. Initializing the Blockchain Database (geth init) for ((i = 0; i < size; i++)); do sed -i -e '/""/d' ${workspace}/.local/node${i}/config.toml # init genesis initLog=${workspace}/.local/node${i}/init.log - if [ $i -eq 0 ] ; then - ${workspace}/bin/geth --datadir ${workspace}/.local/node${i} init --state.scheme ${stateScheme} --db.engine ${dbEngine} ${workspace}/genesis/genesis.json > "${initLog}" 2>&1 - else - ${workspace}/bin/geth --datadir ${workspace}/.local/node${i} init --state.scheme path --db.engine pebble ${workspace}/genesis/genesis.json > "${initLog}" 2>&1 - fi + ${workspace}/bin/geth --datadir ${workspace}/.local/node${i} init --state.scheme ${stateScheme} --db.engine ${dbEngine} ${workspace}/genesis/genesis.json > "${initLog}" 2>&1 rm -f ${workspace}/.local/node${i}/*bsc.log* if [ ${EnableSentryNode} = true ]; then sed -i -e '/""/d' ${workspace}/.local/sentry${i}/config.toml initLog=${workspace}/.local/sentry${i}/init.log - ${workspace}/bin/geth --datadir ${workspace}/.local/sentry${i} init --state.scheme path --db.engine pebble ${workspace}/genesis/genesis.json > "${initLog}" 2>&1 + ${workspace}/bin/geth --datadir ${workspace}/.local/sentry${i} init --state.scheme ${stateScheme} --db.engine ${dbEngine} ${workspace}/genesis/genesis.json > "${initLog}" 2>&1 rm -f ${workspace}/.local/sentry${i}/*bsc.log* fi done @@ -185,7 +213,7 @@ function initNetwork() { sed -i -e '/""/d' ${workspace}/.local/fullnode0/config.toml sed -i -e 's/EnableEVNFeatures = true/EnableEVNFeatures = false/g' ${workspace}/.local/fullnode0/config.toml initLog=${workspace}/.local/fullnode0/init.log - ${workspace}/bin/geth --datadir ${workspace}/.local/fullnode0 init --state.scheme path --db.engine pebble ${workspace}/genesis/genesis.json > "${initLog}" 2>&1 + ${workspace}/bin/geth --datadir ${workspace}/.local/fullnode0 init --state.scheme ${stateScheme} --db.engine ${dbEngine} ${workspace}/genesis/genesis.json > "${initLog}" 2>&1 rm -f ${workspace}/.local/fullnode0/*bsc.log* fi } @@ -206,11 +234,12 @@ function start_node() { nohup ${geth_bin} --config ${datadir}/config.toml \ --datadir ${datadir} \ --nodekey ${datadir}/geth/nodekey \ + --cache 512 \ --rpc.allow-unprotected-txs --allow-insecure-unlock \ --ws --ws.addr 0.0.0.0 --ws.port ${ws_port} \ --http --http.addr 0.0.0.0 --http.port ${http_port} --http.corsdomain "*" \ - --metrics --metrics.addr localhost --metrics.port ${metrics_port} \ - --pprof --pprof.addr localhost --pprof.port ${pprof_port} \ + --metrics --metrics.addr 0.0.0.0 --metrics.port ${metrics_port} \ + --pprof --pprof.addr 0.0.0.0 --pprof.port ${pprof_port} \ --gcmode ${gcmode} --syncmode full --monitor.maliciousvote \ --rialtohash ${rialtoHash} \ --override.passedforktime ${PassedForkTime} \ @@ -228,11 +257,13 @@ function start_node() { >> ${datadir}/bsc-node.log 2>&1 & } +# 7. Start the nodes (Validators, Sentry, Full) in the background function native_start() { PassedForkTime=`cat ${workspace}/.local/node0/hardforkTime.txt|grep passedHardforkTime|awk -F" " '{print $NF}'` LastHardforkTime=$(expr ${PassedForkTime} + ${LAST_FORK_MORE_DELAY}) rialtoHash=`cat ${workspace}/.local/node0/init.log|grep "database=chaindata"|awk -F"=" '{print $NF}'|awk -F'"' '{print $1}'` + # Starting Validator Nodes for ((i=0; i9-validator +# fast-finality panic at consensus/parlia/snapshot.go:411 +# (https://github.com/bnb-chain/bsc/tree/skip-execution). +function prepare_bsc_client() { + if [ ${useLatestBscClient} = true ]; then + local BRANCH="${BSC_GETH_BRANCH:-master}" + if [ ! -f "${workspace}/bsc/Makefile" ]; then + cd ${workspace} + git clone --branch "${BRANCH}" https://github.com/bnb-chain/bsc.git + else + cd ${workspace}/bsc && git fetch origin && git checkout "${BRANCH}" && git pull origin "${BRANCH}" + fi + cd ${workspace}/bsc && make geth && cp -f ${workspace}/bsc/build/bin/geth ${workspace}/bin/ + fi +} + +# 3. Reset genesis submodule and install dependencies (Poetry, NPM, Forge) +# This prepares the environment for generating the genesis block +function reset_genesis() { + if [ ! -f "${workspace}/genesis/genesis-template.json" ]; then + cd ${workspace} && git submodule update --init --recursive genesis + cd ${workspace}/genesis && git reset --hard ${GENESIS_COMMIT} + fi + cd ${workspace}/genesis + + # 1. Update the 'genesis' submodule safely + # Backup user templates, force-update the repository to a specific + # compatible version ($GENESIS_COMMIT) to prevent breaking changes, + # and then restore the user templates. + cp genesis-template.json genesis-template.json.bk + cp scripts/init_holders.template scripts/init_holders.template.bk + git stash + cd ${workspace} && git submodule update --remote --recursive genesis && cd ${workspace}/genesis + git reset --hard ${GENESIS_COMMIT} + mv genesis-template.json.bk genesis-template.json + mv scripts/init_holders.template.bk scripts/init_holders.template + + # 2. Install project-specific dependencies + poetry install --no-root + npm install + + # 3. Clean up and reinstall Foundry framework components (Standard Library & Tests) + # Direct git clone instead of `forge install --no-git` because the latter + # fails inside a parent git repo with "not a git repository: ../../.git/modules/lib/ds-test" + # when forge tries to recursively init forge-std's ds-test submodule. + rm -rf lib/forge-std + git clone --depth 1 --branch v1.7.3 https://github.com/foundry-rs/forge-std.git lib/forge-std + cd lib/forge-std/lib + rm -rf ds-test + git clone https://github.com/dapphub/ds-test +} + +# Patches genesis to disable fast-finality (Plato/Luban) for private chains with N>4 validators. +# +# Why: BSC's Plato hardfork (BLS fast finality) activates at block 7 in the default +# genesis-template. With >4 validators across a WAN, race conditions at chain start +# cause DoubleSign forks at block 2, and the fast-finality reorg path +# (consensus/parlia/snapshot.go:411) panics on misshapen attestation state. +# +# Empirically: 4 validators → works. 21 validators across 3 GCP regions → consistently +# panics at block 8-9. See https://github.com/bnb-chain/bsc/blob/master/consensus/parlia/snapshot.go +# Mainnet didn't bootstrap with Plato either; Plato activated years into a running chain. +# +# This function pushes Plato/Luban (and dependent Berlin/London/Hertz) past chain +# lifetime, plus disables time-based forks that depend on Plato. The chain runs +# vanilla Parlia (probabilistic finality, ~12-block depth). +# +# Toggle: set DISABLE_FAST_FINALITY=false in .env to keep upstream behavior (default 4-validator setup). +function patch_for_private_chain() { + if [ "${DISABLE_FAST_FINALITY:-true}" != "true" ]; then + echo "patch_for_private_chain: DISABLE_FAST_FINALITY=false, skipping" + return 0 + fi + + local TPL=${workspace}/genesis/genesis-template.json + sed -i 's/"lubanBlock": 6,/"lubanBlock": 100000000,/' $TPL + sed -i 's/"platoBlock": 7,/"platoBlock": 100000000,/' $TPL + sed -i 's/"berlinBlock": 8,/"berlinBlock": 100000001,/' $TPL + sed -i 's/"londonBlock": 8,/"londonBlock": 100000001,/' $TPL + sed -i 's/"hertzBlock": 8,/"hertzBlock": 100000002,/' $TPL + sed -i 's/"hertzfixBlock": 8,/"hertzfixBlock": 100000002,/' $TPL + sed -i 's/"bohrTime": 0,/"bohrTime": null,/' $TPL + sed -i 's/"pascalTime": 0,/"pascalTime": null,/' $TPL + sed -i 's/"pragueTime": 0,/"pragueTime": null,/' $TPL + sed -i 's/"lorentzTime": 0,/"lorentzTime": null,/' $TPL + sed -i 's/"maxwellTime": 0,/"maxwellTime": null,/' $TPL + sed -i 's/"fermiTime": 0,/"fermiTime": null,/' $TPL + sed -i 's/"haberFixTime": 0,/"haberFixTime": null,/' $TPL + grep -q '"lubanBlock": 100000000' $TPL || { echo "patch_for_private_chain: luban patch failed" >&2; exit 1; } + echo "patch_for_private_chain: pushed Plato/Luban to block 100000000, disabled time-based post-Plato forks" + + # Sort validators ascending by consensusAddr in extraData. + # Parlia's snapshot.validators() returns ascending-sorted; if extraData order + # differs, in-turn calculation goes to wrong validator → "unauthorized validator" at block 1. + local VT=${workspace}/genesis/scripts/validators.template + if ! grep -q "// SORT_PATCHED" $VT; then + sed -i 's|function extraDataSerialize(validators) {|function extraDataSerialize(validators) {\n // SORT_PATCHED\n validators = validators.slice().sort((a,b) => a.consensusAddr.toLowerCase().localeCompare(b.consensusAddr.toLowerCase()));|' $VT + echo "patch_for_private_chain: validators.template now sorts by consensusAddr" + fi +} + +# 4. Generate validator configurations, hardfork times, and the final genesis.json +function prepare_config() { + rm -f ${workspace}/genesis/validators.conf + + passedHardforkTime=$(expr $(date +%s) + ${PASSED_FORK_DELAY}) + echo "passedHardforkTime "${passedHardforkTime} > ${workspace}/.local/hardforkTime.txt + initHolders=${INIT_HOLDER} + + # 1. Collect Validator Information & Setup Node Directories + for ((i = 0; i < size; i++)); do + # read their randomly generated cryptographic key files (Consensus addresses and BLS vote keys) + for f in ${workspace}/.local/validator${i}/keystore/*; do + cons_addr="0x$(cat ${f} | jq -r .address)" + initHolders=${initHolders}","${cons_addr} + fee_addr=${cons_addr} + done + + targetDir=${workspace}/.local/node${i} + mkdir -p ${targetDir} && cd ${targetDir} + cp ${workspace}/keys/password.txt ./ + cp ${workspace}/.local/hardforkTime.txt ./ + bbcfee_addrs=${fee_addr} + # It assigns a massive, hardcoded amount of "voting power" (0x000001d1a94a2000) so the node has the authority to forge blocks. + powers="0x000001d1a94a2000" #2000000000000 + mv ${workspace}/.local/bls${i}/bls ./ && rm -rf ${workspace}/.local/bls${i} + vote_addr=0x$(cat ./bls/keystore/*json | jq .pubkey | sed 's/"//g') + # it writes all these unique peices: the consensus address, fee collection address, voting power, and BLS vote address + echo "${cons_addr},${bbcfee_addrs},${fee_addr},${powers},${vote_addr}" >> ${workspace}/genesis/validators.conf + if [ ${EnableSentryNode} = true ]; then + # Sentry nodes act as a protective firewall/proxy for Validators, + # hiding the Validator's real IP address from public P2P network attacks. + mkdir -p ${workspace}/.local/sentry${i} + fi + done + if [ ${EnableFullNode} = true ]; then + mkdir -p ${workspace}/.local/fullnode0 + fi + rm -f ${workspace}/.local/hardforkTime.txt + + # 2. Hack / Patch the System Smart Contracts + cd ${workspace}/genesis/ + git checkout HEAD contracts + # "hack" the source code of the core BSCValidatorSet.sol smart contract right before compiling. + # They lower the turnLength and explicitly tell the system to ignore validator punishments/updates for the first 2,000 blocks to ensure your local network starts smoothly without validators instantly getting jailed. + sed -i -e 's/alreadyInit = true;/turnLength = 16;alreadyInit = true;/' ${workspace}/genesis/contracts/BSCValidatorSet.sol + sed -i -e 's/public onlyCoinbase onlyZeroGasPrice {/public onlyCoinbase onlyZeroGasPrice {if (block.number < 2000) return;/' ${workspace}/genesis/contracts/BSCValidatorSet.sol + + # 3. Generate the Final Genesis Block + poetry run python -m scripts.generate generate-validators + poetry run python -m scripts.generate generate-init-holders "${initHolders}" + poetry run python -m scripts.generate dev \ + --dev-chain-id "${CHAIN_ID}" \ + --init-burn-ratio "1000" \ + --init-felony-slash-scope "60" \ + --breathe-block-interval "10 minutes" \ + --block-interval "3 seconds" \ + --stake-hub-protector "${INIT_HOLDER}" \ + --unbond-period "2 minutes" \ + --downtime-jail-time "2 minutes" \ + --felony-jail-time "3 minutes" \ + --misdemeanor-threshold "50" \ + --felony-threshold "150" \ + --init-voting-period "2 minutes / BLOCK_INTERVAL" \ + --init-min-period-after-quorum "uint64(1 minutes / BLOCK_INTERVAL)" \ + --governor-protector "${INIT_HOLDER}" \ + --init-minimal-delay "1 minutes" \ + --token-recover-portal-protector "${INIT_HOLDER}" + cp genesis-dev.json genesis.json +} + +# 5. Initialize the geth network for each node using the generated genesis.json +function initNetwork() { + cd ${workspace} + # 1. Assigning P2P Identities + for ((i = 0; i < size; i++)); do + mkdir ${workspace}/.local/node${i}/geth + cp ${workspace}/keys/validator-nodekey${i} ${workspace}/.local/node${i}/geth/nodekey + mv ${workspace}/.local/validator${i}/keystore ${workspace}/.local/node${i}/ && rm -rf ${workspace}/.local/validator${i} + if [ ${EnableSentryNode} = true ]; then + mkdir ${workspace}/.local/sentry${i}/geth + cp ${workspace}/keys/sentry-nodekey${i} ${workspace}/.local/sentry${i}/geth/nodekey + fi + done + if [ ${EnableFullNode} = true ]; then + mkdir ${workspace}/.local/fullnode0/geth + cp ${workspace}/keys/fullnode-nodekey0 ${workspace}/.local/fullnode0/geth/nodekey + fi + + # 2. Preparing Network Arguments + init_extra_args="" + if [ ${EnableSentryNode} = true ]; then + init_extra_args="--init.sentrynode-size ${size} --init.sentrynode-ports 30411" + fi + if [ ${EnableFullNode} = true ]; then + init_extra_args="${init_extra_args} --init.fullnode-size 1 --init.fullnode-ports 30511" + fi + if [ "${RegisterNodeID}" = true ]; then + if [ "${EnableSentryNode}" = true ]; then + init_extra_args="${init_extra_args} --init.evn-sentry-register" + else + init_extra_args="${init_extra_args} --init.evn-validator-register" + fi + fi + if [ "${EnableEVNWhitelist}" = true ]; then + if [ "${EnableSentryNode}" = true ]; then + init_extra_args="${init_extra_args} --init.evn-sentry-whitelist" + else + init_extra_args="${init_extra_args} --init.evn-validator-whitelist" + fi + fi + + # 3. Generating Network Configs (config.toml) + ${workspace}/bin/geth init-network --init.dir ${workspace}/.local --init.size=${size} --config ${workspace}/config.toml ${init_extra_args} ${workspace}/genesis/genesis.json + rm -f ${workspace}/*bsc.log* + + # 4. Initializing the Blockchain Database (geth init) + for ((i = 0; i < size; i++)); do + sed -i -e '/""/d' ${workspace}/.local/node${i}/config.toml + # init genesis + initLog=${workspace}/.local/node${i}/init.log + ${workspace}/bin/geth --datadir ${workspace}/.local/node${i} init --state.scheme ${stateScheme} --db.engine ${dbEngine} ${workspace}/genesis/genesis.json > "${initLog}" 2>&1 + rm -f ${workspace}/.local/node${i}/*bsc.log* + + if [ ${EnableSentryNode} = true ]; then + sed -i -e '/""/d' ${workspace}/.local/sentry${i}/config.toml + initLog=${workspace}/.local/sentry${i}/init.log + ${workspace}/bin/geth --datadir ${workspace}/.local/sentry${i} init --state.scheme ${stateScheme} --db.engine ${dbEngine} ${workspace}/genesis/genesis.json > "${initLog}" 2>&1 + rm -f ${workspace}/.local/sentry${i}/*bsc.log* + fi + done + if [ ${EnableFullNode} = true ]; then + sed -i -e '/""/d' ${workspace}/.local/fullnode0/config.toml + sed -i -e 's/EnableEVNFeatures = true/EnableEVNFeatures = false/g' ${workspace}/.local/fullnode0/config.toml + initLog=${workspace}/.local/fullnode0/init.log + ${workspace}/bin/geth --datadir ${workspace}/.local/fullnode0 init --state.scheme ${stateScheme} --db.engine ${dbEngine} ${workspace}/genesis/genesis.json > "${initLog}" 2>&1 + rm -f ${workspace}/.local/fullnode0/*bsc.log* + fi +} + +# 6. Patch P2P network to use Docker DNS instead of localhost +function patch_p2p_network() { + # Replace 127.0.0.1 IPs in config.toml with docker-compose service names + for ((i=0; i ${workspace}/.env.cluster +RIALTO_HASH=${rialtoHash} +PASSED_FORK_TIME=${PassedForkTime} +LAST_HARDFORK_TIME=${LastHardforkTime} +FULL_IMMUTABILITY_THRESHOLD=${FullImmutabilityThreshold} +BREATHE_BLOCK_INTERVAL=${BreatheBlockInterval} +MIN_FOR_BLOB_REQUESTS=${MinBlocksForBlobRequests} +DEFAULT_EXTRA_RESERVE=${DefaultExtraReserveForBlobRequests} +EOF + + COMPOSE_FILE=${workspace}/docker-compose.cluster.yml + echo "Generating ${COMPOSE_FILE}..." + echo "services:" > $COMPOSE_FILE + + # Validators + for ((i=0; i> $COMPOSE_FILE + bsc-node-${i}: + image: bsc-toolbox:latest + container_name: bsc-node-${i} + env_file: .env.cluster + environment: + - NODE_TYPE=node + - NODE_INDEX=${i} + volumes: + - .:/node_deploy + ports: + - "${base_rpc}:8545" # RPC & WS + - "${base_metrics}:6060" # Metrics + - "${base_pprof}:7060" # Pprof + - "${p2p_port}:${p2p_port}" # P2P TCP + - "${p2p_port}:${p2p_port}/udp" # P2P UDP + command: ["/node_deploy/node_entrypoint.sh"] + +EOF + done + + # Sentry + if [ ${EnableSentryNode} = true ]; then + for ((i=0; i> $COMPOSE_FILE + bsc-sentry-${i}: + image: bsc-toolbox:latest + container_name: bsc-sentry-${i} + env_file: .env.cluster + environment: + - NODE_TYPE=sentry + - NODE_INDEX=${i} + volumes: + - .:/node_deploy + ports: + - "${base_rpc}:8545" + - "${base_metrics}:6060" + - "${base_pprof}:7060" + - "${p2p_port}:${p2p_port}" + - "${p2p_port}:${p2p_port}/udp" + command: ["/node_deploy/node_entrypoint.sh"] + +EOF + done + fi + + # Full Node + if [ ${EnableFullNode} = true ]; then + base_rpc=8645 + base_metrics=6160 + base_pprof=7160 + p2p_port=30511 + cat <> $COMPOSE_FILE + bsc-fullnode-0: + image: bsc-toolbox:latest + container_name: bsc-fullnode-0 + env_file: .env.cluster + environment: + - NODE_TYPE=full + - NODE_INDEX=0 + volumes: + - .:/node_deploy + ports: + - "${base_rpc}:8545" + - "${base_metrics}:6060" + - "${base_pprof}:7060" + - "${p2p_port}:${p2p_port}" + - "${p2p_port}:${p2p_port}/udp" + command: ["/node_deploy/node_entrypoint.sh"] + +EOF + fi + + cat <> $COMPOSE_FILE + +networks: + default: + name: bsc_cluster_network +EOF + + echo "Generated \${COMPOSE_FILE} successfully!" +} + +# 8. Use create-validator tool to register validator nodes on StakeHub +function register_stakehub(){ + # wait feynman enable + for ((i = 0; i < size; i++));do + create-validator --consensus-key-dir ${workspace}/keys/validator${i} --vote-key-dir ${workspace}/keys/bls${i} \ + --password-path ${workspace}/keys/password.txt --amount 20001 --validator-desc Val${i} --rpc-url ${RPC_URL} + done +} + +# Command dispatcher +CMD=$1 +case ${CMD} in +prepare) + echo "Preparing Docker cluster configs..." + create_validator # Step 1: Prepare keys and .local/ + prepare_bsc_client # Step 2: Build geth if necessary + reset_genesis # Step 3: Setup genesis deps (Forge, Poetry, etc) + patch_for_private_chain # Step 3b: Disable fast-finality for >4-validator chains (no-op if DISABLE_FAST_FINALITY=false) + prepare_config # Step 4: Generate genesis.json and node configs + initNetwork # Step 5: Initialize Geth data directories + patch_p2p_network # Step 6: Patch 127.0.0.1 in configs to docker DNS + generate_compose # Step 7: Generate .env.cluster and docker-compose + echo "Preparation complete!" + ;; +register) + register_stakehub + ;; + wait-rpc) + echo "Waiting for RPC to be available at http://localhost:8545..." + MAX_WAIT=300 # 5 minutes + ELAPSED=0 + until curl -s -X POST -H "Content-Type: application/json" --data '{"jsonrpc":"2.0","method":"eth_blockNumber","params":[],"id":1}' http://localhost:8545 > /dev/null 2>&1; do + printf "." + sleep 2 + ELAPSED=$((ELAPSED + 2)) + if [ "$ELAPSED" -ge "$MAX_WAIT" ]; then + echo "" + echo "ERROR: RPC not available after ${MAX_WAIT}s. Aborting." + exit 1 + fi + done + echo "RPC is ready!" + ;; +*) + echo "Usage: docker_cluster.sh | prepare | register" + ;; +esac diff --git a/entrypoint.sh b/entrypoint.sh new file mode 100644 index 00000000..59d271ea --- /dev/null +++ b/entrypoint.sh @@ -0,0 +1,27 @@ +#!/bin/bash +set -e + +# Change to the application directory +cd /node_deploy + +# Create a lock file to indicate initialization is in progress +touch /tmp/cluster_initializing + +# If the container is starting for the first time or if we want to ensure +# a fresh start, we run the reset script. +echo "====================================================" +echo "Initializing BSC cluster... Please wait." +echo "use docker logs -f bsc-toolbox check for progress" +echo "====================================================" +bash ./bsc_cluster.sh reset + +# Remove the lock file and signal completion +rm /tmp/cluster_initializing +echo "" +echo "====================================================" +echo ">>> BSC CLUSTER INITIALIZATION COMPLETE! <<<" +echo "====================================================" +echo "" + +# Execute the passed command (e.g., /bin/bash) +exec "$@" diff --git a/node_entrypoint.sh b/node_entrypoint.sh new file mode 100755 index 00000000..563968f8 --- /dev/null +++ b/node_entrypoint.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +set -e + +# Node Entrypoint for Docker Compose running BSC geth + +# Ensure environment variables exist +if [ -z "$NODE_TYPE" ] || [ -z "$NODE_INDEX" ]; then + echo "ERROR: NODE_TYPE and NODE_INDEX must be provided" + exit 1 +fi + +if [ -z "$RIALTO_HASH" ]; then + echo "ERROR: RIALTO_HASH not set by env" + exit 1 +fi + +workspace="/node_deploy" +geth_bin="${workspace}/bin/geth" + +echo "Starting BSC $NODE_TYPE node $NODE_INDEX..." + +function start_node() { + local datadir=$1 + local extra_args="" + + # Only Validator nodes require mining and unlocking parameters + if [ "$NODE_TYPE" = "node" ]; then + cons_addr="0x$(jq -r .address ${datadir}/keystore/*)" + extra_args="--mine --vote --unlock ${cons_addr} --miner.etherbase ${cons_addr} --password ${datadir}/password.txt --blspassword ${datadir}/password.txt" + fi + + # Execute geth (replaces the bash shell process with geth) + exec ${geth_bin} --config ${datadir}/config.toml \ + --datadir ${datadir} \ + --nodekey ${datadir}/geth/nodekey \ + --cache 512 \ + --rpc.allow-unprotected-txs --allow-insecure-unlock \ + --ws --ws.addr 0.0.0.0 --ws.port 8545 \ + --http --http.addr 0.0.0.0 --http.port 8545 --http.corsdomain "*" \ + --metrics --metrics.addr 0.0.0.0 --metrics.port 6060 \ + --pprof --pprof.addr 0.0.0.0 --pprof.port 7060 \ + --gcmode full --syncmode full --monitor.maliciousvote \ + --rialtohash ${RIALTO_HASH} \ + --override.immutabilitythreshold ${FULL_IMMUTABILITY_THRESHOLD} \ + --override.breatheblockinterval ${BREATHE_BLOCK_INTERVAL} \ + --override.minforblobrequest ${MIN_FOR_BLOB_REQUESTS} \ + --override.defaultextrareserve ${DEFAULT_EXTRA_RESERVE} \ + $extra_args +} + +if [ "$NODE_TYPE" = "node" ]; then + start_node "${workspace}/.local/node${NODE_INDEX}" +elif [ "$NODE_TYPE" = "sentry" ]; then + start_node "${workspace}/.local/sentry${NODE_INDEX}" +elif [ "$NODE_TYPE" = "full" ]; then + start_node "${workspace}/.local/fullnode${NODE_INDEX}" +else + echo "Unknown NODE_TYPE: $NODE_TYPE" + exit 1 +fi