diff --git a/.ci/crawl.sh b/.ci/crawl.sh new file mode 100755 index 0000000000..23247d4336 --- /dev/null +++ b/.ci/crawl.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +devp2p discv4 crawl -timeout "$ETH_DNS_DISCV4_CRAWLTIME" all.json \ No newline at end of file diff --git a/.ci/deploy.sh b/.ci/deploy.sh new file mode 100755 index 0000000000..e1b9917b93 --- /dev/null +++ b/.ci/deploy.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +set -e + +proto_groups=(all les) + +for network in "$@"; do + + echo "Deploy: $network" + + for p in "${proto_groups[@]}"; do + echo -n "Deploy: ${p}.${network}.${ETH_DNS_DISCV4_PARENT_DOMAIN}" + + # Ensure that we actually have a nodeset to deploy to DNS. + [[ ! -d ${p}.${network}.${ETH_DNS_DISCV4_PARENT_DOMAIN} ]] || [[ ! -f ${p}.${network}.${ETH_DNS_DISCV4_PARENT_DOMAIN}/nodes.json ]] && { echo " | DNE, skipping"; continue; } + + echo + devp2p dns to-cloudflare --zoneid "$ETH_DNS_CLOUDFLARE_ZONEID" "${p}.${network}.${ETH_DNS_DISCV4_PARENT_DOMAIN}" + done +done diff --git a/.ci/deps.sh b/.ci/deps.sh new file mode 100755 index 0000000000..89af252d7f --- /dev/null +++ b/.ci/deps.sh @@ -0,0 +1,38 @@ +#!/bin/sh + +set -e + +# Check programs we depend on. +command -v devp2p >/dev/null 2>&1 && echo "OK: devp2p command in PATH" || { echo "Please install devp2p"; exit 1; } +command -v ethkey >/dev/null 2>&1 && echo "OK: ethkey command in PATH" || { echo "Please install ethkey"; exit 1; } + +# Check that we have key and keypass file. +if [ ! -f $ETH_DNS_DISCV4_KEY_PATH ] || [ ! -f $ETH_DNS_DISCV4_KEYPASS_PATH ]; then + echo " +No key found at key file path or no password file found at ${ETH_DNS_DISCV4_KEYPASS_PATH}. +Use 'ethkey generate ${ETH_DNS_DISCV4_KEY_PATH}' +Save the password in plaintext in ${ETH_DNS_DISCV4_KEYPASS_PATH} +" + exit 1 +else + echo "OK: Key and password file exist." +fi + +# Check that we have deploy variables set. +if [ -z $CLOUDFLARE_API_TOKEN ]; then + echo "Missing CLOUDFLARE_API_TOKEN env var" + exit 1 +else + echo "OK: environment variable CLOUDFLARE_API_TOKEN is not empty." +fi + +# I'm not sure why I couldn't get devp2p to work without using the --zoneid flag; kept getting 403 bad perms. +# Using the flag seems to fix it, and this var gets set as the value for that flag. +# It's associated with the specific domain name that Cloudflare is managing and that we want to deploy to. +if [ -z $ETH_DNS_CLOUDFLARE_ZONEID ]; then + echo "Missing ETH_DNS_CLOUDFLARE_ZONEID env var" + exit 1 +else + echo "OK: environment variable ETH_DNS_CLOUDFLARE_ZONEID is not empty." +fi + diff --git a/.ci/filter_and_sign.sh b/.ci/filter_and_sign.sh new file mode 100755 index 0000000000..9d31353484 --- /dev/null +++ b/.ci/filter_and_sign.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +set -e + +proto_groups=(all les) + +for network in "$@"; do + + echo "Filter: $network" + + mkdir -p "all.${network}.${ETH_DNS_DISCV4_PARENT_DOMAIN}" + devp2p nodeset filter all.json -eth-network "$network" >"all.${network}.${ETH_DNS_DISCV4_PARENT_DOMAIN}/nodes.json" + + mkdir -p "les.${network}.${ETH_DNS_DISCV4_PARENT_DOMAIN}" + devp2p nodeset filter all.json -les-server -eth-network "$network" >"les.${network}.${ETH_DNS_DISCV4_PARENT_DOMAIN}/nodes.json" + + + echo "Sign: $network" + + for p in "${proto_groups[@]}"; do + echo -n "Sign: ${p}.${network}.${ETH_DNS_DISCV4_PARENT_DOMAIN}" + + # Ensure that we actually have a nodeset to sign. + [ ! -d ${p}.${network}.${ETH_DNS_DISCV4_PARENT_DOMAIN} ] || [ ! -f ${p}.${network}.${ETH_DNS_DISCV4_PARENT_DOMAIN}/nodes.json ] && { echo " | DNE, skipping"; continue; } + + echo + cat "${ETH_DNS_DISCV4_KEYPASS_PATH}" | devp2p dns sign "${p}.${network}.${ETH_DNS_DISCV4_PARENT_DOMAIN}" "${ETH_DNS_DISCV4_KEY_PATH}" && echo "OK" + + done + +done \ No newline at end of file diff --git a/.github/workflows/crawl.yml b/.github/workflows/crawl.yml new file mode 100644 index 0000000000..7961a9c02b --- /dev/null +++ b/.github/workflows/crawl.yml @@ -0,0 +1,71 @@ +name: Discv4 Crawl and DNS Update +on: + schedule: + - cron: '0 */2 * * *' +jobs: + build: + if: github.repository == 'ethereum/discv4-dns-lists' + name: Discv4-DNS-Crawler + runs-on: ubuntu-latest + env: + ETH_DNS_DISCV4_CRAWLTIME: 30m + ETH_DNS_DISCV4_PARENT_DOMAIN: ethdisco.net + ETH_DNS_DISCV4_KEY_PATH: ./secrets/dnskey.json + ETH_DNS_DISCV4_KEYPASS_PATH: ./secrets/dnskey_password.txt + CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} + ETH_DNS_CLOUDFLARE_ZONEID: ${{ secrets.ETH_DNS_CLOUDFLARE_ZONEID }} + ETH_DNS_DISCV4_KEY: ${{ secrets.ETH_DNS_DISCV4_KEY }} + ETH_DNS_DISCV4_KEYPASS: ${{ secrets.ETH_DNS_DISCV4_KEYPASS }} + + + steps: + - name: Set up Go + uses: actions/setup-go@v2-beta + with: + go-version: 1.13.8 + id: go + + - run: go version + + - name: Check out code + uses: actions/checkout@v2 + + - name: Install dependencies + run: | + go get -u github.com/ethereum/go-ethereum/cmd/devp2p + go get -u github.com/ethereum/go-ethereum/cmd/ethkey + + - name: Setup secrets + run: | + mkdir secrets + echo "$ETH_DNS_DISCV4_KEY" > "$ETH_DNS_DISCV4_KEY_PATH" + echo "$ETH_DNS_DISCV4_KEYPASS" > "$ETH_DNS_DISCV4_KEYPASS_PATH" + + - name: Check env and secrets + run: | + ./.ci/deps.sh + + - name: Crawl + run: | + ./.ci/crawl.sh + + - name: Filter and sign + run: | + ./.ci/filter_and_sign.sh mainnet ropsten rinkeby goerli + + - name: Commit and Push + env: + GITHUB_USER: FIXME + GITHUB_PAT: FIXME + run: | + git config --local user.name 'crawler' + git config --local user.email 'noreply@users.noreply.github.com' + git add all* les* + git commit --author 'crawler <>' -m "automatic update: crawl time ${ETH_DNS_DISCV4_CRAWLTIME} ci ${GITHUB_RUN_ID}:${GITHUB_RUN_NUMBER}" + git remote set-url origin https://${GITHUB_USER}:${GITHUB_PAT}@github.com/${GITHUB_REPOSITORY}.git + git push origin master + + - name: Deploy to DNS + run: | + ./.ci/deploy.sh mainnet ropsten rinkeby goerli + diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000..4bd922a161 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +secrets/