diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..a41b48b --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,4 @@ + \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..7f5f963 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,108 @@ +name: CI + +on: + push: + branches: [master] + pull_request: + branches: [master] + workflow_dispatch: # https://docs.github.com/en/actions/how-tos/manage-workflow-runs/manually-run-a-workflow + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + # if a PR gets force pushed, cancel the current run and run the new one + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + unittests: + name: unittests + runs-on: ubuntu-24.04 + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Install system packages + run: | + sudo apt-get -qq update + sudo apt-get install -yq libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev librocksdb-dev libleveldb-dev libboost-all-dev libsodium-dev + + - uses: actions/setup-python@v6 + with: + python-version: '3.14' + cache: 'pip' + cache-dependency-path: 'pyproject.toml' + + - name: Log Python version and Git tag + run: | + python3 --version + git tag + + - name: Install Python deps + run: | + # test framework -related deps + pip install pycodestyle + pip install pytest-asyncio + pip install pytest-cov + pip install Sphinx + # e-x itself + # - install both leveldb and rocksdb bindings + pip install ".[leveldb,rocksdb]" + # hashes and other altcoin-specific stuff + pip install tribushashm + pip install blake256 + pip install dash_hash + pip install git+https://github.com/bitcoinplusorg/x13-hash + pip install xevan_hash + pip install quark_hash + pip install groestlcoin_hash + pip install x16r_hash + pip install pycryptodomex + pip install git+https://github.com/Electra-project/nist5_hash + pip install x16rv2_hash + + - name: Run pytest + run: pytest --cov=electrumx + + pycodestyle: + name: pycodestyle + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v6 + + - uses: actions/setup-python@v6 + with: + python-version: '3.14' + cache: 'pip' + + - name: Install pycodestyle + run: pip install pycodestyle + + - name: Run pycodestyle + run: | + pycodestyle \ + --max-line-length=140 \ + src + + build_docs: + # note: this build task is not related to read-the-docs; it is only here to sanity-check the docs can be built. + # For the read-the-docs build, see .readthedocs.yml + # and the webhook set up in https://github.com/spesmilo/electrumx/settings/hooks + name: build_docs + runs-on: ubuntu-24.04 + env: + LC_ALL: "C" + steps: + - uses: actions/checkout@v6 + + - uses: actions/setup-python@v6 + with: + python-version: '3.14' + cache: 'pip' + cache-dependency-path: 'docs/requirements-docs.txt' + + - name: Install docs deps + run: pip install -r docs/requirements-docs.txt + + - name: Build docs + run: | + cd docs + make html diff --git a/.gitignore b/.gitignore index add658f..439786e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,15 +1,20 @@ **/__pycache__/ -.cache/ -.pytest_cache/ +/.cache/ +/.pytest_cache/ */*~ *.#* *# *~ -docs/_build -/build -/dist -/electrumx.egg-info -.vscode/ -.mypy_cache/ -.idea/ +/docs/_build/ +/build/ +/dist/ +/venv/ +/.vscode/ +/.mypy_cache/ +/.idea/ + +/src/electrumx.egg-info/ +/src/e_x.egg-info/ +/electrumx.egg-info/ +/e_x.egg-info/ /certs diff --git a/.readthedocs.yml b/.readthedocs.yml new file mode 100644 index 0000000..8f682ff --- /dev/null +++ b/.readthedocs.yml @@ -0,0 +1,13 @@ +version: 2 + +build: + os: ubuntu-24.04 + tools: + python: "3.13" + +sphinx: + configuration: docs/conf.py + +python: + install: + - requirements: docs/requirements-docs.txt diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 9f4b3d5..0000000 --- a/.travis.yml +++ /dev/null @@ -1,53 +0,0 @@ -sudo: required -dist: trusty -language: python -before_install: - - sudo add-apt-repository -y ppa:streetcrypto7/rocksdb - - sudo add-apt-repository -y ppa:streetcrypto7/leveldb - - sudo apt-get -qq update || true - - sudo apt-get install -yq libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev librocksdb-dev libleveldb-dev -python: - - "3.7-dev" - - "nightly" -# command to install dependencies -install: - - pip install aiohttp - - pip install aiorpcX - - pip install ecdsa - - pip install plyvel - - pip install pycodestyle - - pip install pylru - - pip install python-rocksdb - - pip install pytest-asyncio - - pip install pytest-cov - - pip install Sphinx -# hashes install - - pip install tribushashm - - pip install blake256 - - pip install scrypt - - pip install x11_hash - - pip install git+https://github.com/bitcoinplusorg/x13-hash - - pip install xevan_hash - - pip install quark_hash - - pip install groestlcoin_hash - - pip install neoscrypt - - pip install x16r_hash - - pip install pycryptodomex - - pip install git+https://github.com/Electra-project/nist5_hash - - pip install git+https://github.com/RitoProject/x21s_hash - - pip install x16rv2_hash - - pip install git+https://github.com/VerusCoin/verushashpy - - pip install bell-yespower - - pip install cpupower - - pip install kawpow -# command to run tests -script: - - pytest --cov=electrumx - - pycodestyle --max-line-length=100 electrumx/server/*.py electrumx/lib/*.py *.py - - sh -c "cd docs && make html" -# Dont report coverage from nightly -after_success: - - if [[ $(python3 -V 2>&1) == *"Python 3.7"* ]]; then - pip install python-coveralls; - coveralls; - fi diff --git a/LICENCE b/LICENCE index eac7aaf..ee875d8 100644 --- a/LICENCE +++ b/LICENCE @@ -1,4 +1,5 @@ -Copyright (c) 2016-2017, Neil Booth +Copyright (c) 2016-2020, Neil Booth +Copyright (c) 2020-2026, The Electrum developers All rights reserved. diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..711d43c --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,5 @@ +# note: files listed in this file: +# - will be added to the sdist +# - but won't be added to the wheel (unless by other means) + +graft tests/ diff --git a/README.md b/README.md new file mode 100644 index 0000000..1fae1e1 --- /dev/null +++ b/README.md @@ -0,0 +1,31 @@ +# ElectrumX - python electrum server + +``` +Licence: MIT +Original Author: Neil Booth +Current Maintainers: The Electrum developers +Language: Python (>= 3.10) +``` + +[![Latest PyPI package](https://badge.fury.io/py/e_x.svg)](https://pypi.org/project/e-x/) +[![Build Status](https://github.com/spesmilo/electrumx/actions/workflows/ci.yml/badge.svg?branch=master)](https://github.com/spesmilo/electrumx/actions/workflows/ci.yml) + +This project is a fork of [kyuupichan/electrumx](https://github.com/kyuupichan/electrumx). +The original author dropped support for Bitcoin, which we intend to keep. + +ElectrumX allows users to run their own Electrum server. It connects to your +full node and indexes the blockchain, allowing efficient querying of the history of +arbitrary addresses. The server can be exposed publicly, and joined to the public network +of servers via peer discovery. As of May 2020, a significant chunk of the public +Electrum server network runs ElectrumX. + +### Documentation + +See [readthedocs](https://electrumx-spesmilo.readthedocs.io). + +### Releases + +Running from the latest git tag is always a safe bet, for which there are also releases on PyPI: +``` +$ pip install e-x +``` diff --git a/README.rst b/README.rst deleted file mode 100644 index 1d86b50..0000000 --- a/README.rst +++ /dev/null @@ -1,22 +0,0 @@ -.. image:: https://travis-ci.org/kyuupichan/electrumx.svg?branch=master - :target: https://travis-ci.org/kyuupichan/electrumx -.. image:: https://coveralls.io/repos/github/kyuupichan/electrumx/badge.svg - :target: https://coveralls.io/github/kyuupichan/electrumx - -=============================================== -ElectrumX - Reimplementation of electrum-server -=============================================== - -For a future network with bigger blocks. - - :Licence: MIT - :Language: Python (>= 3.7) - :Author: Neil Booth - -Documentation -============= - -See `readthedocs `_. - - -**Neil Booth** kyuupichan@gmail.com https://github.com/kyuupichan diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..bfe9081 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,3 @@ +# Security Policy + +See https://github.com/spesmilo/electrum/blob/master/SECURITY.md diff --git a/contrib/Dockerfile b/contrib/Dockerfile index 7742dbb..7c321ba 100644 --- a/contrib/Dockerfile +++ b/contrib/Dockerfile @@ -1,35 +1,52 @@ -# example of Dockerfile that builds release of electrumx-1.13.0 -# ENV variables can be overrided on the `docker run` command - -FROM ubuntu:18.04 - -WORKDIR / -ADD https://github.com/kyuupichan/electrumx/archive/1.13.0.tar.gz / -RUN tar zxvf *.tar.gz - -RUN apt-get update && \ - apt-get -y install python3.7 python3-pip librocksdb-dev libsnappy-dev libbz2-dev libz-dev liblz4-dev && \ - pip3 install aiohttp pylru python-rocksdb - -RUN cd /electrumx* && python3 setup.py install +# ElectrumX for Bitcoin, with RocksDB +# This is a simple ~minimal example Dockerfile. +# +# Build: docker build --pull -t electrumx . +# podman build --pull -t electrumx . +# Run: docker run -d --net=host -v ~/electrumx-db:/var/lib/electrumx -e DAEMON_URL="http://youruser:yourpass@localhost:8332" -e REPORT_SERVICES=tcp://example.com:50001 electrumx +# podman run -d --net=host -v ~/electrumx-db:/var/lib/electrumx -e DAEMON_URL="http://youruser:yourpass@localhost:8332" -e REPORT_SERVICES=tcp://example.com:50001 electrumx +# Stop: docker stop CONTAINER_ID +# podman stop CONTAINER_ID + +FROM python:3.14-trixie AS builder + +WORKDIR /usr/src/app + +# Install the libs needed by rocksdb (including development headers) +RUN apt-get update \ + && apt-get -y --no-install-recommends install \ + build-essential \ + git \ + librocksdb-dev \ + pkg-config \ + && rm -rf /var/lib/apt/lists/* + +# Install ElectrumX +ARG ELECTRUMX_VERSION=2.0.0 +#ARG ELECTRUMX_VERSION=b8f6e227f251fe5cba96b940ffaa094439636417 +RUN python -m venv venv \ + && venv/bin/pip install --no-cache-dir \ + "e_x[rocksdb] @ git+https://github.com/spesmilo/electrumx.git@${ELECTRUMX_VERSION}" + + +FROM python:3.14-slim-trixie + +# Install the libs needed by rocksdb (no development headers or statics) +RUN apt-get update && apt-get -y --no-install-recommends install librocksdb9.10 && rm -rf /var/lib/apt/lists/* ENV SERVICES="tcp://:50001" -ENV COIN=BitcoinSV -ENV DB_DIRECTORY=/db +ENV COIN=Bitcoin +ENV DB_DIRECTORY=/var/lib/electrumx ENV DAEMON_URL="http://username:password@hostname:port/" ENV ALLOW_ROOT=true ENV DB_ENGINE=rocksdb -ENV MAX_SEND=10000000 -ENV BANDWIDTH_UNIT_COST=50000 -ENV CACHE_MB=2000 +#ENV CACHE_MB=2000 -VOLUME /db +WORKDIR /usr/src/app +COPY --from=builder /usr/src/app . -RUN mkdir -p "$DB_DIRECTORY" && ulimit -n 1048576 +VOLUME /var/lib/electrumx -CMD ["/usr/bin/python3", "/usr/local/bin/electrumx_server"] +RUN mkdir -p "$DB_DIRECTORY" -# build it with eg.: `docker build -t electrumx .` -# run it with eg.: -# `docker run -d --net=host -v /home/electrumx/db/:/db -e DAEMON_URL="http://youruser:yourpass@localhost:8332" -e REPORT_SERVICES=tcp://example.com:50001 electrumx` -# for a proper clean shutdown, send TERM signal to the running container eg.: `docker kill --signal="TERM" CONTAINER_ID` +CMD ["/usr/src/app/venv/bin/python", "/usr/src/app/venv/bin/electrumx_server"] diff --git a/contrib/Dockerfile_beefyserver b/contrib/Dockerfile_beefyserver new file mode 100644 index 0000000..e56fcd9 --- /dev/null +++ b/contrib/Dockerfile_beefyserver @@ -0,0 +1,82 @@ +# ElectrumX for Bitcoin, with RocksDB +# This Dockerfile is intended for modern multi-core servers with at least 16 GB of RAM. +# We compile our own free-threaded Python, with --disable-gil (which is experimental!), which makes +# ElectrumX able to truly utilise multiple CPU cores and makes processing blocks significantly faster. +# +# Build: docker build -f Dockerfile_beefyserver --pull -t electrumx . +# Run: docker run -d --net=host -v ~/electrumx-db:/var/lib/electrumx -e DAEMON_URL="http://youruser:yourpass@localhost:8332" -e REPORT_SERVICES=tcp://example.com:50001 electrumx +# Stop: docker stop CONTAINER_ID + +FROM debian:trixie + +ENV LC_ALL=C.UTF-8 LANG=C.UTF-8 +ENV DEBIAN_FRONTEND=noninteractive + +WORKDIR /usr/src/app + +# Install libs needed to build CPython +RUN apt-get update \ + && apt-get -y install \ + git \ + wget \ + make \ + autotools-dev \ + autoconf \ + libtool \ + xz-utils \ + libssl-dev \ + libssl3 \ + openssl \ + zlib1g-dev \ + libffi-dev \ + libncurses5-dev \ + libsqlite3-dev \ + && rm -rf /var/lib/apt/lists/* + +# Install libs needed to build RocksDB +RUN apt-get update \ + && apt-get -y --no-install-recommends install \ + build-essential \ + git \ + librocksdb-dev \ + pkg-config \ + && rm -rf /var/lib/apt/lists/* + +# build CPython (free-threaded, with --disable-gil) +ARG PYTHON_VERSION="v3.14.6" +RUN cd /opt && \ + git clone https://github.com/python/cpython && \ + cd cpython/ && \ + git checkout "${PYTHON_VERSION}" && \ + ./configure \ + --prefix="/usr" \ + --enable-ipv6 \ + --enable-shared \ + --disable-gil \ + -q && \ + make -s "-j$(nproc)" && \ + make install + +RUN python3 -m pip install --upgrade pip setuptools wheel + +# Install ElectrumX +ARG ELECTRUMX_VERSION=2.0.0 +#ARG ELECTRUMX_VERSION=b8f6e227f251fe5cba96b940ffaa094439636417 +RUN python3 -m pip install \ + "e_x[rocksdb] @ git+https://github.com/spesmilo/electrumx.git@${ELECTRUMX_VERSION}" + + +ENV SERVICES="tcp://:50001" +ENV COIN=Bitcoin +ENV DB_DIRECTORY=/var/lib/electrumx +ENV DAEMON_URL="http://username:password@hostname:port/" +ENV ALLOW_ROOT=true +ENV DB_ENGINE=rocksdb +ENV CACHE_MB=2000 + + +VOLUME /var/lib/electrumx + +RUN mkdir -p "$DB_DIRECTORY" + +CMD ["electrumx_server"] diff --git a/contrib/query.py b/contrib/query.py index 56dbda8..077fd6c 100755 --- a/contrib/query.py +++ b/contrib/query.py @@ -14,7 +14,6 @@ import argparse import asyncio -import sys from electrumx import Env from electrumx.server.db import DB @@ -82,7 +81,7 @@ async def query(args): n = None utxos = await db.all_utxos(hashX) for n, utxo in enumerate(utxos, start=1): - print(f'UTXO #{n:,d}: tx_hash {hash_to_hex_str(utxo.tx_hash)} ' + print(f'UTXO #{n:,d}: tx_hash {hash_to_hex_str(utxo.txid_rev)} ' f'tx_pos {utxo.tx_pos:,d} height {utxo.height:,d} ' f'value {utxo.value:,d}') if n == limit: @@ -106,8 +105,7 @@ def main(): parser.add_argument('scripts', nargs='*', default=[], type=str, help='hex scripts to query') args = parser.parse_args() - loop = asyncio.get_event_loop() - loop.run_until_complete(query(args)) + asyncio.run(query(args)) if __name__ == '__main__': main() diff --git a/contrib/raspberrypi3/install_electrumx.sh b/contrib/raspberrypi3/install_electrumx.sh index fb80043..8c7ad77 100644 --- a/contrib/raspberrypi3/install_electrumx.sh +++ b/contrib/raspberrypi3/install_electrumx.sh @@ -22,6 +22,6 @@ sudo apt-get install git sudo pip3 install plyvel # install electrumx -git clone https://github.com/kyuupichan/electrumx.git +git clone https://github.com/spesmilo/electrumx.git cd electrumx -sudo python3 setup.py install +sudo python3 -m pip install . diff --git a/contrib/raspberrypi3/run_electrumx.sh b/contrib/raspberrypi3/run_electrumx.sh index db23f0d..a022262 100644 --- a/contrib/raspberrypi3/run_electrumx.sh +++ b/contrib/raspberrypi3/run_electrumx.sh @@ -4,7 +4,7 @@ ############### # configure electrumx -export COIN=BitcoinSegwit +export COIN=Bitcoin export DAEMON_URL=http://rpcuser:rpcpassword@127.0.0.1 export NET=mainnet export CACHE_MB=400 diff --git a/contrib/sdist/Dockerfile b/contrib/sdist/Dockerfile new file mode 100644 index 0000000..31597d0 --- /dev/null +++ b/contrib/sdist/Dockerfile @@ -0,0 +1,30 @@ +FROM debian:bookworm@sha256:b877a1a3fdf02469440f1768cf69c9771338a875b7add5e80c45b756c92ac20a + +ENV LC_ALL=C.UTF-8 LANG=C.UTF-8 +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update -q && \ + apt-get install -qy \ + git \ + python3 \ + python3-pip \ + python3-setuptools \ + python3-venv \ + python3-build \ + && \ + rm -rf /var/lib/apt/lists/* && \ + apt-get autoremove -y && \ + apt-get clean + +# create new user to avoid using root; but with sudo access and no password for convenience. +ARG UID=1000 +ENV USER="user" +ENV HOME_DIR="/home/${USER}" +ENV WORK_DIR="${HOME_DIR}/wspace" \ + PATH="${HOME_DIR}/.local/bin:${PATH}" +RUN useradd --uid $UID --create-home --shell /bin/bash ${USER} +RUN usermod -append --groups sudo ${USER} +RUN echo "%sudo ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers +WORKDIR ${WORK_DIR} +RUN chown --recursive ${USER} ${WORK_DIR} +USER ${USER} diff --git a/contrib/sdist/README.md b/contrib/sdist/README.md new file mode 100644 index 0000000..63e4566 --- /dev/null +++ b/contrib/sdist/README.md @@ -0,0 +1,29 @@ +# Source tarballs + +✓ _These tarballs should be reproducible, meaning you should be able to generate + distributables that match the official releases._ + +This assumes an Ubuntu (x86_64) host, but it should not be too hard to adapt to another +similar system. + + +## Build steps + +1. Install Docker + + See [`docker_notes.md`](https://github.com/spesmilo/electrum/blob/master/contrib/docker_notes.md). + + (worth reading even if you already have docker) + +2. Build tarball + + ``` + $ ./contrib/sdist/build.sh + ``` + If you want reproducibility, try instead e.g.: + ``` + $ ELECBUILD_COMMIT=HEAD ELECBUILD_NOCACHE=1 ./contrib/sdist/build.sh + ``` + +3. The generated distributables are in `./dist`. + diff --git a/contrib/sdist/build.sh b/contrib/sdist/build.sh new file mode 100755 index 0000000..29a2d5b --- /dev/null +++ b/contrib/sdist/build.sh @@ -0,0 +1,70 @@ +#!/bin/bash +# +# env vars: +# - ELECBUILD_NOCACHE: if set, forces rebuild of docker image +# - ELECBUILD_COMMIT: if set, do a fresh clone and git checkout + +set -e + +PROJECT_ROOT="$(dirname "$(readlink -e "$0")")/../.." +PROJECT_ROOT_OR_FRESHCLONE_ROOT="$PROJECT_ROOT" +CONTRIB="$PROJECT_ROOT/contrib" +CONTRIB_SDIST="$CONTRIB/sdist" +DISTDIR="$PROJECT_ROOT/dist" +BUILD_UID=$(/usr/bin/stat -c %u "$PROJECT_ROOT") + +export SOURCE_DATE_EPOCH=1530212462 +export PYTHONHASHSEED=22 + + +DOCKER_BUILD_FLAGS="" +if [ ! -z "$ELECBUILD_NOCACHE" ] ; then + echo "ELECBUILD_NOCACHE is set. forcing rebuild of docker image." + DOCKER_BUILD_FLAGS="--pull --no-cache" +fi + +if [ -z "$ELECBUILD_COMMIT" ] ; then # local dev build + DOCKER_BUILD_FLAGS="$DOCKER_BUILD_FLAGS --build-arg UID=$BUILD_UID" +fi + +echo "building docker image." +docker build \ + $DOCKER_BUILD_FLAGS \ + -t electrumx-builder-img \ + "$CONTRIB_SDIST" + +# maybe do fresh clone +if [ ! -z "$ELECBUILD_COMMIT" ] ; then + echo "ELECBUILD_COMMIT=$ELECBUILD_COMMIT. doing fresh clone and git checkout." + FRESH_CLONE="/tmp/electrumx_build/fresh_clone/electrumx" + rm -rf "$FRESH_CLONE" 2>/dev/null || ( echo "we need sudo to rm prev FRESH_CLONE." && sudo rm -rf "$FRESH_CLONE" ) + umask 0022 + git clone "$PROJECT_ROOT" "$FRESH_CLONE" + cd "$FRESH_CLONE" + git checkout "$ELECBUILD_COMMIT" + PROJECT_ROOT_OR_FRESHCLONE_ROOT="$FRESH_CLONE" +else + echo "not doing fresh clone." +fi + +echo "building binary..." +# check uid and maybe chown. see #8261 +if [ ! -z "$ELECBUILD_COMMIT" ] ; then # fresh clone (reproducible build) + if [ $(id -u) != "1000" ] || [ $(id -g) != "1000" ] ; then + echo "need to chown -R FRESH_CLONE dir. prompting for sudo." + sudo chown -R 1000:1000 "$FRESH_CLONE" + fi +fi +docker run -it \ + --name electrumx-builder-cont \ + -v "$PROJECT_ROOT_OR_FRESHCLONE_ROOT":/opt/electrumx \ + --rm \ + --workdir /opt/electrumx/contrib/sdist \ + electrumx-builder-img \ + ./make_sdist.sh + +# make sure resulting binary location is independent of fresh_clone +if [ ! -z "$ELECBUILD_COMMIT" ] ; then + mkdir --parents "$DISTDIR/" + cp -f "$FRESH_CLONE/dist"/* "$DISTDIR/" +fi diff --git a/contrib/sdist/make_sdist.sh b/contrib/sdist/make_sdist.sh new file mode 100755 index 0000000..7348b55 --- /dev/null +++ b/contrib/sdist/make_sdist.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +set -e + +PROJECT_ROOT="$(dirname "$(readlink -e "$0")")/../.." +CONTRIB="$PROJECT_ROOT/contrib" +CONTRIB_SDIST="$CONTRIB/sdist" +BUILDDIR="$CONTRIB_SDIST/build" +DISTDIR="$PROJECT_ROOT/dist" + +cd "$PROJECT_ROOT" + +rm -rf "$BUILDDIR" +mkdir -p "$BUILDDIR" "$DISTDIR" + + +git_status=$(git status --porcelain) +if [ ! -z "$git_status" ]; then + echo "$git_status" + echo "git repo not clean, aborting" + exit 1 +fi + + +cd "$PROJECT_ROOT" +python3 -m build --sdist . --outdir "$BUILDDIR/dist1" + + +# the initial tar.gz is not reproducible, see https://github.com/pypa/setuptools/issues/2133 +# so we untar, fix timestamps, and then re-tar +DISTNAME=$(find "$BUILDDIR/dist1/" -type f -name 'e_x-*.tar.gz' -printf "%f\\n") +DIST_BASENAME=$(basename --suffix ".tar.gz" "$DISTNAME") +mkdir -p "$BUILDDIR/dist2" +cd "$BUILDDIR/dist2" +tar -xzf "$BUILDDIR/dist1/$DISTNAME" +find -exec touch -h -d '2000-11-11T11:11:11+00:00' {} + +GZIP=-n tar --sort=name -czf "$DISTNAME" "$DIST_BASENAME/" +mv "$DISTNAME" "$DISTDIR/$DISTNAME" + diff --git a/contrib/systemd/electrumx.conf b/contrib/systemd/electrumx.conf index 9787037..bf51db7 100644 --- a/contrib/systemd/electrumx.conf +++ b/contrib/systemd/electrumx.conf @@ -5,7 +5,7 @@ DB_DIRECTORY = /db # Bitcoin Node RPC Credentials DAEMON_URL = http://username:password@hostname:port/ -# COIN = BitcoinSegwit +# COIN = Bitcoin -# See http://electrumx.readthedocs.io/en/latest/environment.html for +# See https://electrumx-spesmilo.readthedocs.io/en/latest/environment.html for # information about other configuration settings you probably want to consider. diff --git a/contrib/systemd/electrumx.service b/contrib/systemd/electrumx.service index 22e3e9d..24a6368 100644 --- a/contrib/systemd/electrumx.service +++ b/contrib/systemd/electrumx.service @@ -5,6 +5,7 @@ After=network.target [Service] EnvironmentFile=/etc/electrumx.conf ExecStart=/usr/local/bin/electrumx_server +ExecStop=/usr/local/bin/electrumx_rpc -p 8000 stop User=electrumx LimitNOFILE=8192 TimeoutStopSec=30min diff --git a/docs/HOWTO.rst b/docs/HOWTO.rst index 99b279b..e563d67 100644 --- a/docs/HOWTO.rst +++ b/docs/HOWTO.rst @@ -15,19 +15,20 @@ small - pull requests are welcome. ================ ======================== Package Notes ================ ======================== -Python3 ElectrumX uses asyncio. Python version >= 3.7 is +Python3 ElectrumX uses asyncio. Python version >= 3.10 is **required**. `aiohttp`_ Python library for asynchronous HTTP. Version >= 2.0 required. -`pylru`_ Python LRU cache package. DB Engine A database engine package is required; two are supported (see `Database Engine`_ below). ================ ======================== Some coins need an additional package, typically for their block hash -functions. For example, `x11_hash`_ is required for DASH. +functions. For example, `dash_hash`_ is required for DASH. Scrypt coins +require a Python interpreter compiled and/or linked with OpenSSL 1.1.0 +or higher. -You **must** to be running a non-pruning bitcoin daemon with:: +You **must** be running a non-pruning bitcoin daemon with:: txindex=1 @@ -61,15 +62,28 @@ Database Engine =============== You can choose from LevelDB and RocksDB to store transaction -information on disk. The time taken and DB size is not significantly -different. We tried to support LMDB but its history write performance +information on disk. On an SSD, RocksDB performs better than LevelDB. +We tried to support LMDB, but its history write performance was much worse. You will need to install one of: -+ `plyvel `_ for LevelDB -+ `python-rocksdb `_ for RocksDB (`pip3 install python-rocksdb`) -+ `pyrocksdb `_ for an unmaintained version that doesn't work with recent releases of RocksDB ++ `plyvel `_ for LevelDB. + + ``apt install libleveldb-dev build-essential`` + + ``pip3 install plyvel`` , or use the ``[leveldb]`` extra install option to ElectrumX. + ++ `rocksdb-ng `_ for RocksDB + + (Should work with RocksDB versions 8-10) + + ``apt install librocksdb-dev build-essential pkg-config`` + + ``pip3 install rocksdb-ng`` , or use the ``[rocksdb]`` extra install option to ElectrumX. + ++ `python-rocksdb `_ for an unmaintained version + that doesn't work with recent releases of RocksDB (only <7.0) Running ======= @@ -78,11 +92,23 @@ Install the prerequisites above. Check out the code from Github:: - git clone https://github.com/kyuupichan/electrumx.git + git clone https://github.com/spesmilo/electrumx.git cd electrumx -You can install with :file:`setup.py` or run the code from the source -tree or a copy of it. +You can install with:: + + pip3 install . + +There are many extra Python dependencies available to fit the needs of your +system or coins. For example, to install the RocksDB dependencies and a faster +JSON (de)serialization library:: + + pip3 install ".[rocksdb,orjson]" + +see pyproject.toml's ``project.optional-dependencies`` for a complete list. + +You can also run the code from the source tree or a copy of it. + You should create a standard user account to run the server under; your own is probably adequate unless paranoid. The paranoid might @@ -101,7 +127,12 @@ live on an SSD:: Process limits -------------- -You must ensure the ElectrumX process has a large open file limit. +The ElectrumX process needs a large open file limit. On Linux systems, +the default (:command:`ulimit -n`) (soft) open file limit is usually 1,024. +ElectrumX tries to raise this automatically during startup to the hard limit, +which is usually 100,000+. If that fails (which would get logged), +you might have to manually increase the limit. + During sync it should not need more than about 1,024 open files. When serving it will use approximately 256 for LevelDB plus the number of incoming connections. It is not unusual to have 1,000 to 2,000 @@ -421,11 +452,10 @@ You can then set the port as follows and advertise the service externally on the REPORT_SSL_PORT=110 -.. _`contrib/systemd/electrumx.service`: https://github.com/kyuupichan/electrumx/blob/master/contrib/systemd/electrumx.service +.. _`contrib/systemd/electrumx.service`: https://github.com/spesmilo/electrumx/blob/master/contrib/systemd/electrumx.service .. _`daemontools`: http://cr.yp.to/daemontools.html .. _`runit`: http://smarden.org/runit/index.html .. _`aiohttp`: https://pypi.python.org/pypi/aiohttp -.. _`pylru`: https://pypi.python.org/pypi/pylru -.. _`x11_hash`: https://pypi.python.org/pypi/x11_hash -.. _`contrib/raspberrypi3/install_electrumx.sh`: https://github.com/kyuupichan/electrumx/blob/master/contrib/raspberrypi3/install_electrumx.sh -.. _`contrib/raspberrypi3/run_electrumx.sh`: https://github.com/kyuupichan/electrumx/blob/master/contrib/raspberrypi3/run_electrumx.sh +.. _`dash_hash`: https://pypi.python.org/pypi/dash_hash +.. _`contrib/raspberrypi3/install_electrumx.sh`: https://github.com/spesmilo/electrumx/blob/master/contrib/raspberrypi3/install_electrumx.sh +.. _`contrib/raspberrypi3/run_electrumx.sh`: https://github.com/spesmilo/electrumx/blob/master/contrib/raspberrypi3/run_electrumx.sh diff --git a/docs/PERFORMANCE-NOTES b/docs/PERFORMANCE-NOTES index 9335a7b..ab044ee 100644 --- a/docs/PERFORMANCE-NOTES +++ b/docs/PERFORMANCE-NOTES @@ -33,4 +33,13 @@ account in the code. - multiple assignment is faster using tuples only for 3 or more items - retrieving a previously stored length of a bytes object can be over 200% - faster than a new call to len(b) \ No newline at end of file + faster than a new call to len(b) + +- The cpython and pypy stdlibs use a Python-based JSON serializer/deserializer, + which is stable and easy to use and install, but somewhat slower than + system-compiled variants. ElectrumX now looks for known popular + C/Rust-based JSON libraries for Python. If it finds one, + it's used instead of the stdlib module for improved performance. + + To install a known-compatible system-compiled JSON lib, install ElectrumX + with the "orjson" extras parameter. diff --git a/docs/architecture.rst b/docs/architecture.rst index 6ac192d..edaea58 100644 --- a/docs/architecture.rst +++ b/docs/architecture.rst @@ -7,7 +7,7 @@ Architecture Env --- -Holds configuration taken from the environment, with apprioriate +Holds configuration taken from the environment, with appropriate defaulting appropriately. Generally passed to the constructor of other components which take their settings from it. diff --git a/docs/authors.rst b/docs/authors.rst index ee20e39..13d2964 100644 --- a/docs/authors.rst +++ b/docs/authors.rst @@ -5,7 +5,11 @@ Authors * Neil Booth - Creator and maintainer. + Original creator, past maintainer. + +* Electrum developers + + Current maintainers. * Johann Bauer diff --git a/docs/changelog.rst b/docs/changelog.rst index 4e395e1..1a3ecfe 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,16 +2,212 @@ ChangeLog =========== -.. note:: Version 1.15.0 will be the final ElectrumX release with altcoin support, future - releases will be Bitcoin-only. ElectrumX needs to scale quickly and support for - various other coins and their idiosyncracies is distracting and unhelpful. Anyone - wishing to maintain an ElectrumX repository with altcoin support is free to do so as - explained in the licence. +Version 2.0.0 (03 July 2026) +============================== -.. note:: It is strongly recommended you upgrade to Python 3.7, which - fixes bugs in asyncio that caused an ever-growing open file count - and memory consumption whilst serving clients. Those problems - should not occur with Python 3.7. +TL;DR: + +- new DB schema, so you need to rebuild from scratch. No more "DB::flush_count overflow". +- Bitcoin Core >=31 is now required, with `txospenderindex=1` +- added support for Electrum Protocol 1.7 +- running with free-threaded CPython is now (experimentally) supported, and is much faster + +----- + +* Breaking changes to on-disk database. Server operators need to manually delete old DB, + no migration path. New DB will be rebuilt while rescanning again from genesis. + + - in return, no more manual "history compaction" and unexpected downtimes every year: + no more "DB::flush_count overflow" (`spesmilo/electrumx#88`_) + - the size of the new DB is comparable to the old one, however: + +* We now require Bitcoin Core to have :code:`txospenderindex=1` (added in Bitcoin Core 31) + in addition to :code:`txindex=1`. This is needed to serve the :code:`blockchain.outpoint.subscribe` RPC + added in Electrum Protocol 1.7. + + For reference, on Bitcoin mainnet around height=950k, + + - the ElectrumX db uses around 122 GiB (using LevelDB, roughly same for both of e-x 1.x and 2.0), + - note: using RocksDB 9, it is a bit smaller, around 106 GiB + - :code:`.bitcoin/blocks/` uses 788 GiB, + - :code:`.bitcoin/chainstate/` uses 12 GiB, + - :code:`.bitcoin/indexes/txindex/` uses 66 GiB, + - :code:`.bitcoin/indexes/txospenderindex/` uses 88 GiB (new!) + + - This also means that we now require Bitcoin Core 31.0 or newer (for :code:`COIN=Bitcoin`). + +* env: the previously optional envvar :code:`DB_ENGINE`, is now mandatory. + + - choose either :code:`leveldb` or :code:`rocksdb` + - in ElectrumX 1.x versions, the default was leveldb. + + You need to install the appropriate dependencies for your engine, see the :code:`[leveldb]` and + :code:`[rocksdb]` pip extras and the "Database Engine" section of the HOWTO. + + Previously RocksDB was difficult to use as the python bindings for it have been unmaintained for years. + By the super-slop-powers of LLMs, we revived the python bindings as :code:`rocksdb-ng` and made it compatible with modern + cpython, cython, and rocksdb. The changes are reviewable and not really slop. (see `spesmilo/electrumx#347`_) + rocksdb-ng (starting with version 2.3) is also compatible with `free-threaded python`_, see "performance" section. + + LevelDB was written with HDDs in mind. RocksDB is more modern + and on an SSD takes around ~25% less time than LevelDB to sync from genesis. + Given that this release contains breaking DB changes and a resync is needed anyway, maybe try out RocksDB :) + +* performance: + + - the BlockProcessor is now massively multi-threaded, both inside a single block and across a span of blocks. + This can mainly only be taken advantage of if the GIL (Global Interpreter Lock) is disabled, that is, + when using a `free-threaded python`_ interpreter: e.g. cpython :code:`3.14t`, compiled with :code:`--disable-gil`. + On a decent modern machine with a multi-core CPU, this has been measured to cut initial sync time + from genesis to a bit less than half. OTOH, much more RAM is required for the parallel processing + (example: approx 2 GB when syncing with GIL, 8 GB when syncing without GIL). + + - processing a single block at the tip, when already caught up, is expected to take around half the time, + regardless of the GIL. Still, without the GIL, it is even faster. + + - for more details, see `spesmilo/electrumx#369`_ + + - the (block)Prefetcher and the Mempool code are now partially event-based (but still also polling): + e.g. a mempool update is triggered as soon as processing a new block begins (and part of the work + is done concurrently) + + - json (de)serialization can now optionally use the Rust-based + :code:`orjson` library (via the :code:`orjson` pip extra), replacing the + previous :code:`ujson` and :code:`rapidjson` extras. + +* Dockerfiles: the existing Dockerfile was updated and cleaned up a bit, and it serves + as a simple minimal example to show how to run ElectrumX. There is now also a second Dockerfile + that sets up a free-threaded python for a beefy server to utilise the above-mentioned performance gains. + See them in :code:`contrib/`. (also see the new runtime logline "Python GIL enabled" to verify effect) + +* protocol: + - new: implement electrum protocol version 1.7 (`spesmilo/electrum-protocol#2`_). + The min supported protocol version remains 1.4, the max is now 1.7. + +* LocalRPC: new command: "inspect_session" (`spesmilo/electrumx#361`_). + + +Version 1.20.0 (03 June 2026) +============================= + +Small fixes and general maintenance. + +After this release, the master branch will move towards a 2.0 release, which will require +a resync from genesis. So for a while, master might be less stable than usual. + +* fix: lib/tx.py: vsize calculation was off by 6-8 vbytes (`spesmilo/electrumx#328`_). + Added tests. The discrepancy was externally observable via the `mempool.get_fee_histogram` RPC. +* fix: don't get stuck during shutdown waiting for SessionManager to stop (`spesmilo/electrumx#339`_) +* fix: protocol: name of arg for broadcast_package RPC: tx_package->raw_txs (`c9b72e7e`_) +* changed: peer discovery: allow short chain splits (`spesmilo/electrumx#351`_) +* changed: rm 'attrs' as a direct dependency, use stdlib `dataclasses` instead (`spesmilo/electrumx#345`_) +* new: daemon: check early "txindex" is enabled for bitcoind, error if missing (`spesmilo/electrumx#349`_) +* new: coins: add Bitcoin mutinynet (`spesmilo/electrumx#352`_) +* maintenance: coins: some clean-up (`spesmilo/electrumx#344`_, `spesmilo/electrumx#348`_, ...) +* maintenance: moved CI from Cirrus CI to Github Actions (`spesmilo/electrumx#340`_) + +(We also made some changes to make RocksDB practical and nicer to use, but if you currently use LevelDB, +maybe consider waiting for 2.0 before you resync the DB) + + +Version 1.19.0 (11 Nov 2025) +============================= + +* protocol: + - new: implement electrum protocol version 1.6 (`spesmilo/electrumx#317`_, ...) + (`spesmilo/electrum-protocol#6`_). + The min supported protocol version remains 1.4, the max is now 1.6. +* coins: + - changed: for `COIN=Bitcoin`, there is now a min required bitcoind version: + Bitcoin Core 28.0 (or Knots 28) (`spesmilo/electrumx#316`_). + This is due to protocol 1.6 requiring a bitcoind with working `submitpackage` RPC. +* security: + - fix DOS vector: if MAX_SESSIONS was reached, when using python 3.12+, the server + stopped accepting new incoming sessions until full-restart (`spesmilo/electrumx#312`_) +* session: + - changed: add warmup budget to PaddedRSTransport (`spesmilo/electrumx#323`_): + this is a small relaxation of the traffic analysis countermeasures added in 1.18, + where the first 1024 bytes we send are now exempt from buffering (and hence delays) + - fix ReplyAndDisconnect for PaddedRSTransport: flush message buffer (`spesmilo/electrumx#322`_) +* env: + - rm DROP_CLIENT_UNKNOWN opt-in env var: this behaviour is now mandated by protocol 1.6 + and we just always enforce it (~small breaking change, affected clients would have + already been considered misbehaving) +* misc: + - new: try to raise `ulimit -n` open file limit automatically at startup + (`spesmilo/electrumx#326`_) + + +Version 1.18.0 (14 June 2025) +============================= + +* protocol: + - add basic countermeasures against traffic analysis by padding the jsonrpc payload + with whitespaces to have ~uniform-size TCP packets, and by artificially delaying + sending messages a bit. (`spesmilo/electrumx#301`_) +* dependencies: + - bump required aiorpcx to >=0.25.0,<0.26 + + +Version 1.17.0 (16 Apr 2025) +============================ + +Long time no see! + +* coins: + - rm support for legacy coin name "BitcoinSegwit" (`ca59151d`_) + (users should just change their config files to :code:`COIN=Bitcoin`) + - add support for Bitcoin Signet (`spesmilo/electrumx#122`_) + - add support for Bitcoin Testnet4 (`spesmilo/electrumx#273`_) +* dependencies: + - bump required python to >=3.10 + - bump required aiorpcx to >=0.23.0,<0.25 + - rm "pylru" dep, instead bundle stripped-down "cachetools" (`spesmilo/electrumx#248`_) +* session: log IP address of excessive resusage sessions (`spesmilo/electrumx#75`_) +* robustness: + - session: more robust session handling (`25339328`_, `8198ab99`_) + - daemon: catch RPC_PARSE_ERROR from bitcoind and simply retry (`31489365`_) +* env: + - add MAX_RECV environment variable (`a0d3053f`_) + - increase default for MAX_SEND (`2969ab11`_) +* protocol: increase resolution of :code:`mempool.get_fee_histogram` (`265a5a87`_) +* LocalRPC: + - add --timeout arg (`5f4dd2cd`_) + - add commands to help debug memory leaks (`f5582b29`_) +* build/maintenance: + - add scripts for reproducible build (`0ba87447`_) + - migrate from setup.py to pyproject.toml + - refactor file layout flat->src (`spesmilo/electrumx#298`_) +* some internal refactors in preparation for future changes + + +Version 1.16.0 (10 Dec 2020) +============================ + +Note: this is the first release since forking from kyuupichan/electrumx. +kyuupichan has the :code:`electrumx` name on PyPI, so we needed a new name there. +We are using the `e-x `_ name on PyPI, so you can +install this package via e.g. :code:`pip install e-x`. + +* security: a vulnerability has been fixed that allowed a remote attacker to + crash electrumx if peer discovery was enabled (`#22`_) +* fixed some peer-discovery-related bugs (e.g. `#35`_) +* ENV: when using Bitcoin, the COIN ENV var can now be set to :code:`Bitcoin`. + For compatibility, using :code:`BitcoinSegwit` will also keep working. + (`#5`_) +* session resource limits: made more useful in general. connection-time-based + grouping has been removed (`#70`_). Disconnects of over-limit sessions happen + sooner (`4b3f6510`_). Subnet sizes for IP-based grouping can now be + configured (`a61136c5`_). +* protocol: :code:`mempool.get_fee_histogram` now returns fee rates with + 0.1 sat/byte resolution instead of 1 sat/byte, and the compact histogram + is calculated differently. (`#67`_) +* performance: bitcoind is now queried less frequently as estimatefee, + relayfee, and server.banner requests are now cached (`#24`_) +* performance: json ser/deser is now abstracted away and the :code:`ujson` and + :code:`rapidjson` exras can be used for somewhat faster block processing. + (`#11`_) +* multiple other small performance improvements Version 1.15.0 (27 May 2020) @@ -157,8 +353,15 @@ Version 1.9.0 (10 Jan 2019) * other minor tweaks (Michael Schmoock, Michael Taborsky) +Original author of ElectrumX: + **Neil Booth** kyuupichan@gmail.com https://github.com/kyuupichan +This fork maintained by: + +**Electrum developers** electrumdev@gmail.com https://github.com/spesmilo + + .. _#554: https://github.com/kyuupichan/electrumx/issues/554 .. _#566: https://github.com/kyuupichan/electrumx/issues/566 .. _#653: https://github.com/kyuupichan/electrumx/issues/653 @@ -170,3 +373,59 @@ Version 1.9.0 (10 Jan 2019) .. _#731: https://github.com/kyuupichan/electrumx/issues/731 .. _#795: https://github.com/kyuupichan/electrumx/issues/795 .. _#909: https://github.com/kyuupichan/electrumx/issues/909 + + +.. _#5: https://github.com/spesmilo/electrumx/pull/5 +.. _#11: https://github.com/spesmilo/electrumx/pull/11 +.. _#22: https://github.com/spesmilo/electrumx/issues/22 +.. _#24: https://github.com/spesmilo/electrumx/pull/24 +.. _#35: https://github.com/spesmilo/electrumx/pull/35 +.. _#67: https://github.com/spesmilo/electrumx/pull/67 +.. _#70: https://github.com/spesmilo/electrumx/pull/70 +.. _spesmilo/electrumx#75: https://github.com/spesmilo/electrumx/pull/75 +.. _spesmilo/electrumx#88: https://github.com/spesmilo/electrumx/issues/88 +.. _spesmilo/electrumx#122: https://github.com/spesmilo/electrumx/pull/122 +.. _spesmilo/electrumx#248: https://github.com/spesmilo/electrumx/pull/248 +.. _spesmilo/electrumx#273: https://github.com/spesmilo/electrumx/pull/273 +.. _spesmilo/electrumx#298: https://github.com/spesmilo/electrumx/pull/298 +.. _spesmilo/electrumx#301: https://github.com/spesmilo/electrumx/pull/301 +.. _spesmilo/electrumx#312: https://github.com/spesmilo/electrumx/pull/312 +.. _spesmilo/electrumx#316: https://github.com/spesmilo/electrumx/pull/316 +.. _spesmilo/electrumx#317: https://github.com/spesmilo/electrumx/pull/317 +.. _spesmilo/electrumx#322: https://github.com/spesmilo/electrumx/pull/322 +.. _spesmilo/electrumx#323: https://github.com/spesmilo/electrumx/pull/323 +.. _spesmilo/electrumx#326: https://github.com/spesmilo/electrumx/pull/326 +.. _spesmilo/electrumx#328: https://github.com/spesmilo/electrumx/pull/328 +.. _spesmilo/electrumx#339: https://github.com/spesmilo/electrumx/pull/339 +.. _spesmilo/electrumx#340: https://github.com/spesmilo/electrumx/pull/340 +.. _spesmilo/electrumx#344: https://github.com/spesmilo/electrumx/pull/344 +.. _spesmilo/electrumx#345: https://github.com/spesmilo/electrumx/pull/345 +.. _spesmilo/electrumx#347: https://github.com/spesmilo/electrumx/pull/347 +.. _spesmilo/electrumx#348: https://github.com/spesmilo/electrumx/pull/348 +.. _spesmilo/electrumx#349: https://github.com/spesmilo/electrumx/pull/349 +.. _spesmilo/electrumx#351: https://github.com/spesmilo/electrumx/pull/351 +.. _spesmilo/electrumx#352: https://github.com/spesmilo/electrumx/pull/352 +.. _spesmilo/electrumx#361: https://github.com/spesmilo/electrumx/pull/361 +.. _spesmilo/electrumx#369: https://github.com/spesmilo/electrumx/pull/369 + + +.. _4b3f6510: https://github.com/spesmilo/electrumx/commit/4b3f6510e94670a013c1abe6247cdd2b0e7e6f8c +.. _a61136c5: https://github.com/spesmilo/electrumx/commit/a61136c596d6a0290a6be9d21fb7c095c3cea21e +.. _ca59151d: https://github.com/spesmilo/electrumx/commit/ca59151d7365aabd2394fb39aac1faa25949b49d +.. _25339328: https://github.com/spesmilo/electrumx/commit/25339328a7468235071d152f728b214df10d4c56 +.. _8198ab99: https://github.com/spesmilo/electrumx/commit/8198ab99603a7e74083b3569c15dad13850c721e +.. _31489365: https://github.com/spesmilo/electrumx/commit/314893655a7cc0bfcc216b07900fa77b5f66e148 +.. _a0d3053f: https://github.com/spesmilo/electrumx/commit/a0d3053f60074264b54b3ef7583f2600f4b73ead +.. _2969ab11: https://github.com/spesmilo/electrumx/commit/2969ab110412bebddc3f3e815467fb59538b613d +.. _265a5a87: https://github.com/spesmilo/electrumx/commit/265a5a87b8ad01f739049c0b1e80923aab318f58 +.. _5f4dd2cd: https://github.com/spesmilo/electrumx/commit/5f4dd2cdb414464484407affbbaae6b7407696cb +.. _f5582b29: https://github.com/spesmilo/electrumx/commit/f5582b29792625e8cca7cf137a6718c2520bb9cb +.. _0ba87447: https://github.com/spesmilo/electrumx/commit/0ba87447cb293cfc4a8a26c1c27842b95666875a +.. _c9b72e7e: https://github.com/spesmilo/electrumx/commit/c9b72e7ea76bfb706b2df3e90c739116bf49678d + + +.. _spesmilo/electrum-protocol#2: https://github.com/spesmilo/electrum-protocol/pull/2 +.. _spesmilo/electrum-protocol#6: https://github.com/spesmilo/electrum-protocol/pull/6 + +.. _free-threaded python: https://docs.python.org/3/howto/free-threading-python.html + diff --git a/docs/conf.py b/docs/conf.py index 94d2387..2b4e070 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -15,13 +15,13 @@ import os import sys sys.path.insert(0, os.path.abspath('..')) -VERSION="ElectrumX 1.15.0" +VERSION="ElectrumX 2.0.0" # -- Project information ----------------------------------------------------- project = 'ElectrumX' -copyright = '2016-2020, Neil Booth' -author = 'Neil Booth' +copyright = '2016-2026, various' +author = 'Electrum developers' # The full version including branding release = VERSION @@ -58,7 +58,7 @@ # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = None +language = 'en' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. @@ -82,7 +82,7 @@ # html_theme_options = { 'description': 'Lightweight Electrum Server in Python', - 'github_user': 'kyuupichan', + 'github_user': 'spesmilo', 'github_repo': 'electrumx', 'github_button': True, 'github_type': 'star', diff --git a/docs/environment.rst b/docs/environment.rst index 03d842c..8217fc6 100644 --- a/docs/environment.rst +++ b/docs/environment.rst @@ -19,17 +19,23 @@ Required These environment variables are always required: -.. envvar:: COIN - - Must be a :attr:`NAME` from one of the :class:`Coin` classes in - `lib/coins.py`_. - .. envvar:: DB_DIRECTORY The path to the database directory. Relative paths should be relative to the parent process working directory. This is the directory of the `run` script if you use it. +.. envvar:: DB_ENGINE + + Database engine for the UTXO and history database. + Choose one of ``leveldb`` or ``rocksdb``. + You will need to install the appropriate python package for your engine. + In ElectrumX 1.x versions, the default was leveldb. + Warning: It is not possible to switch back and forth, + the on-disk DB formats are not compatible with each other: you have to resync from genesis. + LevelDB was written with HDDs in mind. RocksDB is more modern + and on an SSD takes around ~25% less time than LevelDB to sync from genesis. + .. envvar:: DAEMON_URL A comma-separated list of daemon URLs. If more than one is provided @@ -157,9 +163,9 @@ Here are some examples of valid services:: Here is an example value of the :envvar:`REPORT_SERVICES` environment variable:: - tcp://sv.usebsv.com:50001,ssl://sv.usebsv.com:50002,wss://sv.usebsv.com:50004 + tcp://example.com:50001,ssl://example.com:50002,wss://example.com:50004 - This advertizes **tcp**, **ssl**, **wss** services at :const:`sv.usebsv.com` on ports + This advertizes **tcp**, **ssl**, **wss** services at :const:`example.com` on ports 50001, 50002 and 50004 respectively. .. note:: Certificate Authority-signed certificates don't work over Tor, so you should @@ -183,6 +189,16 @@ Miscellaneous These environment variables are optional: +.. envvar:: COIN + + Must be a :attr:`NAME` from one of the :class:`Coin` classes in + `lib/coins.py`_. Defaults to ``Bitcoin``. + +.. envvar:: NET + + Must be a :attr:`NET` from one of the :class:`Coin` classes in + `lib/coins.py`_. Defaults to ``mainnet``. + .. envvar:: LOG_FORMAT The Python logging `format string @@ -199,18 +215,6 @@ These environment variables are optional: Set this environment variable to anything non-empty to allow running ElectrumX as root. -.. envvar:: NET - - Must be a :envvar:`NET` from one of the **Coin** classes in - `lib/coins.py`_. Defaults to ``mainnet``. - -.. envvar:: DB_ENGINE - - Database engine for the UTXO and history database. The default is - ``leveldb``. The other alternative is ``rocksdb``. You will need - to install the appropriate python package for your engine. The - value is not case sensitive. - .. envvar:: DONATION_ADDRESS The server donation address reported to Electrum clients. Defaults @@ -299,10 +303,18 @@ raise them. SSL listening sockets are closed until the session count drops naturally to 95% of the limit. Defaults to 1,000. +.. envvar:: MAX_RECV + + The maximum size of an incoming message in bytes, the default is 1,000,000 bytes. + Note that the smallest sane/safe value for Bitcoin is ~800,100 bytes, + as the largest standard tx can have a weight of 400K but the protocol hex-encodes that, + plus there is a few bytes of protocol overhead. Setting this to lower than that + would preclude clients from broadcasting txs that could propagate over the network. + .. envvar:: MAX_SEND The maximum size of a response message to send over the wire, in - bytes. Defaults to 1,000,000 (except for AuxPoW coins, which default + bytes. Defaults to 8,100,000 (except for AuxPoW coins, which default to 10,000,000). Values smaller than 350,000 are taken as 350,000 because standard Electrum protocol header "chunk" requests are almost that large. @@ -311,19 +323,27 @@ raise them. served all at once or not at all, an obvious avenue for abuse. :envvar:`MAX_SEND` is a stop-gap until the protocol is improved to admit incremental history requests. Each history entry is - approximately 100 bytes so the default is equivalent to a history - limit of around 10,000 entries, which should be ample for most - legitimate users. If you use a higher default bear in mind one - client can request history for multiple addresses. Also note that - the largest raw transaction you will be able to serve to a client is - just under half of :envvar:`MAX_SEND`, as each raw byte becomes 2 - hexadecimal ASCII characters on the wire. Very few transactions on - Bitcoin mainnet are over 500KB in size. + approximately 100 bytes, so the default is equivalent to a history + limit of around 80,000 entries, which should be ample for most + legitimate users. If you use a higher default, bear in mind one + client can request history for multiple addresses. + + Also note that the largest raw transaction you will be able to serve to a client is + just under half of :envvar:`MAX_SEND`, as each raw byte becomes 2 hexadecimal + ASCII characters on the wire. For Bitcoin, the consensus limit for tx size is + 4 M weight units, while the standardness (policy) limit is 400 K weight units. + When serialized as bytes, 4 M weight units is 4 MB, and when encoded as hex that is 8 MB. + The default was chosen so that such a tx could still be served. + (Note: in multi-party protocol, such as Lightning, a counterparty might collude with a miner + to get a tx mined. Being able to obtain such a raw tx might be part of the security + requirements of a protocol.) .. envvar:: COST_SOFT_LIMIT .. envvar:: COST_HARD_LIMIT .. envvar:: REQUEST_SLEEP .. envvar:: INITIAL_CONCURRENT +.. envvar:: SESSION_GROUP_BY_SUBNET_IPV4 +.. envvar:: SESSION_GROUP_BY_SUBNET_IPV6 All values are integers. :envvar:`COST_SOFT_LIMIT` defaults to :const:`1,000`, :envvar:`COST_HARD_LIMIT` to :const:`10,000`, :envvar:`REQUEST_SLEEP` to :const:`2,500` @@ -354,6 +374,10 @@ raise them. If a session disconnects, ElectrumX continues to associate its cost with its IP address, so if it immediately reconnects it will re-acquire its previous cost allocation. + Moreover, sessions are also grouped together based on their IP address subnets, and cost + is accrued for the whole group. What subnet sizes to use can be configured via + :envvar:`SESSION_GROUP_BY_SUBNET_IPV4` (by default /24) and + :envvar:`SESSION_GROUP_BY_SUBNET_IPV6` (by default /48). A server operator should experiment with different values according to server loads. It is not necessarily true that e.g. having a low soft limit, decreasing concurrency and @@ -396,7 +420,7 @@ If you are not running a Tor proxy ElectrumX will be unable to connect to onion server peers, in which case rather than returning no onion peers it will fall back to a hard-coded list. -To give incoming clients a full range of onion servers you will need +To give incoming clients a full range of onion servers, you will need to be running a Tor proxy for ElectrumX to use. ElectrumX will perform peer-discovery by default and announce itself @@ -482,5 +506,5 @@ your available physical RAM: I do not recommend raising this above 2000. -.. _lib/coins.py: https://github.com/kyuupichan/electrumx/blob/master/electrumx/lib/coins.py +.. _lib/coins.py: https://github.com/spesmilo/electrumx/blob/master/src/electrumx/lib/coins.py .. _uvloop: https://pypi.python.org/pypi/uvloop diff --git a/docs/features.rst b/docs/features.rst index 2018148..8e5f869 100644 --- a/docs/features.rst +++ b/docs/features.rst @@ -39,9 +39,7 @@ blockchain indexer: and history. Electrum Server stores full transaction hash and height for each UTXO, and does the same in its pruned history. In contrast ElectrumX just stores the transaction number in the linear - history of transactions. For at least another 5 years this - transaction number will fit in a 4-byte integer, and when necessary - expanding to 5 or 6 bytes is trivial. ElectrumX can determine block + history of transactions. ElectrumX can determine block height from a simple binary search of tx counts stored on disk. ElectrumX stores historical transaction hashes in a linear array on disk. @@ -59,30 +57,3 @@ blockchain indexer: Python's :mod:`asyncio` means ElectrumX has no (direct) use for threads and associated complications. - -Roadmap -======= - -* break ElectrumX up into simple services that initially can be run in - separate processes on a single host. Then support running them on - different hosts, and finally support sharding. With this we can - take advantage of multiple cores and hosts, and scale to much larger - block sizes. This should solve several issues with the current - history storage mechanism. -* fully asynchronous operation. At present too much is synchronous, such - as file system access. -* protocol improvements targeting better client and server scalability - to large wallets (100k addresses) and address histories. Some - aspects of the current protocol are very inefficient. -* investigate speaking the Bitcoin protocol and connecting to the - Bitcoin network directly for some queries. This could lead to - ElectrumX being runnable with a node without a tx index, or a - pruning node, or not needing to run a node at all. ElectrumX would - store all blocks itself and index the transactions therein. -* lifting internal limits such as maximum 4 billion transactions -* supporting better user privacy. I believe significantly improved - user address privacy might be possible with a simple addition to the - protocol, and assuming a server network of which a reasonable - fraction (40%?) are cooperative and non-colluding -* new features such as possibly adding label or wallet server - functionality diff --git a/docs/index.rst b/docs/index.rst index b2fdc78..5b3643f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -2,12 +2,17 @@ ElectrumX ========= -.. image:: https://travis-ci.org/kyuupichan/electrumx.svg?branch=master - :target: https://travis-ci.org/kyuupichan/electrumx -.. image:: https://coveralls.io/repos/github/kyuupichan/electrumx/badge.svg - :target: https://coveralls.io/github/kyuupichan/electrumx +.. image:: https://github.com/spesmilo/electrumx/actions/workflows/ci.yml/badge.svg?branch=master + :target: https://github.com/spesmilo/electrumx/actions/workflows/ci.yml -A reimplementation of Electrum-Server for a future with bigger blocks. +This project is a fork of `kyuupichan/electrumx `_. +The original author dropped support for Bitcoin, which we intend to keep. + +ElectrumX allows users to run their own Electrum server. It connects to your +full node and indexes the blockchain, allowing efficient querying of history of +arbitrary addresses. The server can be exposed publicly, and joined to the public network +of servers via peer discovery. As of May 2020, a significant chunk of the public +Electrum server network runs ElectrumX. The current version is |release|. @@ -15,22 +20,22 @@ Source Code =========== The project is hosted on `GitHub -`_. and uses `Travis -`_ for Continuous +`_. and uses `GitHub Actions +`_ for Continuous Integration. Please submit an issue on the `bug tracker -`_ if you have found a +`_ if you have found a bug or have a suggestion to improve the server. Authors and License =================== Neil Booth wrote the vast majority of the code; see :ref:`Authors`. -Python version at least 3.7 is required. +Python version at least 3.10 is required. The code is released under the `MIT Licence -`_. +`_. Getting Started =============== @@ -39,7 +44,7 @@ See :ref:`HOWTO`. There is also an `installer`_ available that simplifies the installation on various Linux-based distributions, and a `Dockerfile`_ -available . +available. .. _installer: https://github.com/bauerj/electrumx-installer .. _Dockerfile: https://github.com/lukechilds/docker-electrumx diff --git a/docs/peer_discovery.rst b/docs/peer_discovery.rst index 1a4c1f0..423156c 100644 --- a/docs/peer_discovery.rst +++ b/docs/peer_discovery.rst @@ -25,7 +25,7 @@ process. Ideally it should have at least 4 servers that have shown commitment to reliable service. In ElectrumX this is a per-coin property in `lib/coins.py -`_. +`_. server.peers.subscribe diff --git a/docs/protocol-basics.rst b/docs/protocol-basics.rst deleted file mode 100644 index 9dbbd54..0000000 --- a/docs/protocol-basics.rst +++ /dev/null @@ -1,205 +0,0 @@ -Protocol Basics -=============== - -Message Stream --------------- - -Clients and servers communicate using **JSON RPC** over an unspecified underlying stream -transport. Examples include TCP, SSL, WS and WSS. - -Two standards `JSON RPC 1.0 -`_ and `JSON RPC 2.0 -`_ are specified; use of version -2.0 is encouraged but not required. Server support of batch requests -is encouraged for version 1.0 but not required. - -.. note:: A client or server should only indicate JSON RPC 2.0 by - setting the `jsonrpc - `_ member of - its messages to ``"2.0"`` if it supports the version 2.0 protocol in - its entirety. ElectrumX does and will expect clients advertizing so - to function correctly. Those that do not will be disconnected and - possibly blacklisted. - -Clients making batch requests should limit their size depending on the -nature of their query, because servers will limit response size as an -anti-DoS mechanism. - -Over TCP and SSL raw sockets each RPC call, and each response, MUST be terminated by a -single newline to delimit messages. Websocket messages are already framed so they MUST -NOT be newline terminated. The JSON specification does not permit control characters -within strings, so no confusion is possible there. However it does permit newlines as -extraneous whitespace between elements; client and server MUST NOT use newlines in such a -way. - -If using JSON RPC 2.0's feature of parameter passing by name, the -names shown in the description of the method or notification in -question MUST be used. - -A server advertising support for a particular protocol version MUST -support each method documented for that protocol version, unless the -method is explicitly marked optional. It may support other methods or -additional parameters with unspecified behaviour. Use of additional -parameters is discouraged as it may conflict with future versions of -the protocol. - - -Notifications -------------- - -Some RPC calls are subscriptions which, after the initial response, -will send a JSON RPC :dfn:`notification` each time the thing -subscribed to changes. The `method` of the notification is the same -as the method of the subscription, and the `params` of the -notification (and their names) are given in the documentation of the -method. - - -Version Negotiation -------------------- - -It is desirable to have a way to enhance and improve the protocol -without forcing servers and clients to upgrade at the same time. - -Protocol versions are denoted by dotted number strings with at least -one dot. Examples: "1.5", "1.4.1", "2.0". In "a.b.c" *a* is the -major version number, *b* the minor version number, and *c* the -revision number. - -A party to a connection will speak all protocol versions in a range, -say from `protocol_min` to `protocol_max`, which may be the same. -When a connection is made, both client and server must initially -assume the protocol to use is their own `protocol_min`. - -The client should send a :func:`server.version` RPC call as early as -possible in order to negotiate the precise protocol version; see its -description for more detail. All responses received in the stream -from and including the server's response to this call will use its -negotiated protocol version. - - -.. _script hashes: - -Script Hashes -------------- - -A :dfn:`script hash` is the hash of the binary bytes of the locking -script (ScriptPubKey), expressed as a hexadecimal string. The hash -function to use is given by the "hash_function" member of -:func:`server.features` (currently :func:`sha256` only). Like for -block and transaction hashes, when converting the big-endian binary -hash to a hexadecimal string the least-significant byte appears first, -and the most-significant byte last. - -For example, the legacy Bitcoin address from the genesis block:: - - 1A1zP1eP5QGefi2DMPTfTL5SLmv7DivfNa - -has P2PKH script:: - - 76a91462e907b15cbf27d5425399ebf6f0fb50ebb88f1888ac - -with SHA256 hash:: - - 6191c3b590bfcfa0475e877c302da1e323497acf3b42c08d8fa28e364edf018b - -which is sent to the server reversed as:: - - 8b01df4e368ea28f8dc0423bcf7a4923e3a12d307c875e47a0cfbf90b5c39161 - -By subscribing to this hash you can find P2PKH payments to that address. - -One public key, the genesis block public key, among the trillions for -that address is:: - - 04678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb - 649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5f - -which has P2PK script:: - - 4104678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb - 649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5fac - -with SHA256 hash:: - - 3318537dfb3135df9f3d950dbdf8a7ae68dd7c7dfef61ed17963ff80f3850474 - -which is sent to the server reversed as:: - - 740485f380ff6379d11ef6fe7d7cdd68aea7f8bd0d953d9fdf3531fb7d531833 - -By subscribing to this hash you can find P2PK payments to the genesis -block public key. - -.. note:: The Genesis block coinbase is uniquely unspendable and - therefore not indexed. It will not show with the above P2PK script - hash subscription. - - -.. _status: - -Status ------- - -To calculate the `status` of a :ref:`script hash