Build V2Ray rules dat files #1157

Workflow file for this run

	name: Build V2Ray rules dat files
	on:
	workflow_dispatch:
	schedule:
	- cron: "30 0 * * *"
	push:
	branches:
	- master
	paths-ignore:
	- "**/README.md"
	- "**/emergency_release.yml"
	jobs:
	build:
	runs-on: ubuntu-latest
	steps:
	- name: Set variables
	run: \|
	echo "RELEASE_NAME=Released on $(date +%Y%m%d%H%M)" >> $GITHUB_ENV
	echo "TAG_NAME=$(date +%Y%m%d%H%M)" >> $GITHUB_ENV
	echo "CHINA_DOMAINS_URL=https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/accelerated-domains.china.conf" >> $GITHUB_ENV
	echo "GOOGLE_DOMAINS_URL=https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/google.china.conf" >> $GITHUB_ENV
	echo "APPLE_DOMAINS_URL=https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/apple.china.conf" >> $GITHUB_ENV
	echo "EASYLISTCHINA_EASYLIST_REJECT_URL=https://easylist-downloads.adblockplus.org/easylistchina+easylist.txt" >> $GITHUB_ENV
	echo "PETERLOWE_REJECT_URL=https://pgl.yoyo.org/adservers/serverlist.php?hostformat=hosts&showintro=1&mimetype=plaintext" >> $GITHUB_ENV
	echo "ADGUARD_DNS_REJECT_URL=https://adguardteam.github.io/AdGuardSDNSFilter/Filters/filter.txt" >> $GITHUB_ENV
	echo "DANPOLLOCK_REJECT_URL=https://someonewhocares.org/hosts/hosts" >> $GITHUB_ENV
	echo "CUSTOM_DIRECT=https://raw.githubusercontent.com/Loyalsoldier/domain-list-custom/release/cn.txt" >> $GITHUB_ENV
	echo "CUSTOM_PROXY=https://raw.githubusercontent.com/Loyalsoldier/domain-list-custom/release/geolocation-!cn.txt" >> $GITHUB_ENV
	echo "WIN_SPY=https://raw.githubusercontent.com/crazy-max/WindowsSpyBlocker/master/data/hosts/spy.txt" >> $GITHUB_ENV
	echo "WIN_UPDATE=https://raw.githubusercontent.com/crazy-max/WindowsSpyBlocker/master/data/hosts/update.txt" >> $GITHUB_ENV
	echo "WIN_EXTRA=https://raw.githubusercontent.com/crazy-max/WindowsSpyBlocker/master/data/hosts/extra.txt" >> $GITHUB_ENV
	echo "HALF_PRICE_IN_IRAN=https://raw.githubusercontent.com/IRConf/Iranian-Half-Price-Traffic-Websites-List/main/domains" >> $GITHUB_ENV
	echo "BLOCKED_BY_TAIWAN=https://raw.githubusercontent.com/rootmelo92118/blocked-by-taiwan/release/blocked-by-taiwan.txt" >> $GITHUB_ENV
	echo "BLOCKED_BY_RUSSIA=https://raw.githubusercontent.com/georgekart/RuBannedList/master/category-banned-ru" >> $GITHUB_ENV
	shell: bash

	- name: Checkout the "hidden" branch of this repo
	uses: actions/checkout@v6
	with:
	ref: hidden

	- name: Checkout Loyalsoldier/domain-list-custom
	uses: actions/checkout@v6
	with:
	repository: Loyalsoldier/domain-list-custom
	path: custom

	- name: Checkout v2fly/domain-list-community
	uses: actions/checkout@v6
	with:
	repository: v2fly/domain-list-community
	path: community

	- name: Checkout cokebar/gfwlist2dnsmasq
	uses: actions/checkout@v6
	with:
	repository: cokebar/gfwlist2dnsmasq
	path: gfwlist2dnsmasq

	- name: Checkout rootmelo92118/sing-geosite
	uses: actions/checkout@v6
	with:
	repository: rootmelo92118/sing-geosite
	path: sing-geosite

	- name: Setup Go
	uses: actions/setup-go@v6
	with:
	go-version-file: ./custom/go.mod

	- name: Setup Python
	uses: actions/setup-python@v4
	with:
	python-version: '3.10'

	- name: Get geoip.dat relative files
	run: \|
	wget https://github.com/rootmelo92118/geoip-custom/raw/release/geoip.dat
	wget https://github.com/rootmelo92118/geoip-custom/raw/release/geoip.dat.sha256sum

	- name: Generate GFWList domains
	run: \|
	cd gfwlist2dnsmasq \|\| exit 1
	chmod +x ./gfwlist2dnsmasq.sh
	./gfwlist2dnsmasq.sh -l -o ./temp-gfwlist.txt

	- name: Get and add direct domains into temp-direct.txt file
	run: \|
	curl -sSL $CHINA_DOMAINS_URL \| perl -ne '/^server=\/([^\/]+)\// && print "$1\n"' > temp-direct.txt
	curl -sSL ${CUSTOM_DIRECT} \| perl -ne '/^(domain):([^:]+)(\n$\|:@.+)/ && print "$2\n"' >> temp-direct.txt

	- name: Get and add proxy domains into temp-proxy.txt file
	run: \|
	cat ./gfwlist2dnsmasq/temp-gfwlist.txt \| perl -ne '/^((?=^.{3,255})[a-zA-Z0-9][-_a-zA-Z0-9]{0,62}(\.[a-zA-Z0-9][-_a-zA-Z0-9]{0,62})+)/ && print "$1\n"' > temp-proxy.txt
	curl -sSL $GOOGLE_DOMAINS_URL \| perl -ne '/^server=\/([^\/]+)\// && print "$1\n"' >> temp-proxy.txt
	curl -sSL $APPLE_DOMAINS_URL \| perl -ne '/^server=\/([^\/]+)\// && print "$1\n"' >> temp-proxy.txt
	curl -sSL ${CUSTOM_PROXY} \| grep -Ev ":@cn" \| perl -ne '/^(domain):([^:]+)(\n$\|:@.+)/ && print "$2\n"' >> temp-proxy.txt

	- name: Get and add reject domains into temp-reject.txt file
	run: \|
	curl -sSL $EASYLISTCHINA_EASYLIST_REJECT_URL \| perl -ne '/^\\|\\|([-_0-9a-zA-Z]+(\.[-_0-9a-zA-Z]+){1,64})\^$/ && print "$1\n"' \| perl -ne 'print if not /^[0-9]{1,3}(\.[0-9]{1,3}){3}$/' > temp-reject.txt
	curl -sSL $ADGUARD_DNS_REJECT_URL \| perl -ne '/^\\|\\|([-_0-9a-zA-Z]+(\.[-_0-9a-zA-Z]+){1,64})\^$/ && print "$1\n"' \| perl -ne 'print if not /^[0-9]{1,3}(\.[0-9]{1,3}){3}$/' >> temp-reject.txt
	curl -sSL $PETERLOWE_REJECT_URL \| perl -ne '/^127\.0\.0\.1\s([-_0-9a-zA-Z]+(\.[-_0-9a-zA-Z]+){1,64})$/ && print "$1\n"' >> temp-reject.txt
	curl -sSL $DANPOLLOCK_REJECT_URL \| perl -ne '/^127\.0\.0\.1\s([-_0-9a-zA-Z]+(\.[-_0-9a-zA-Z]+){1,64})/ && print "$1\n"' \| sed '1d' >> temp-reject.txt

	- name: Reserve `full`, `regexp` and `keyword` type of rules from custom lists to "reserve" files
	run: \|
	curl -sSL ${CUSTOM_DIRECT} \| perl -ne '/^((full\|regexp\|keyword):[^:]+)(\n$\|:@.+)/ && print "$1\n"' \| sort --ignore-case -u > direct-reserve.txt
	curl -sSL ${CUSTOM_PROXY} \| grep -Ev ":@cn" \| perl -ne '/^((full\|regexp\|keyword):[^:]+)(\n$\|:@.+)/ && print "$1\n"' \| sort --ignore-case -u > proxy-reserve.txt

	- name: Add proxy, direct and reject domains from "hidden" branch to appropriate temp files
	run: \|
	cat proxy.txt >> temp-proxy.txt
	cat direct.txt >> temp-direct.txt
	cat reject.txt >> temp-reject.txt

	- name: Sort and generate redundant lists
	run: \|
	cat temp-proxy.txt \| sort --ignore-case -u > proxy-list-with-redundant
	cat temp-direct.txt \| sort --ignore-case -u > direct-list-with-redundant
	cat temp-reject.txt \| sort --ignore-case -u > reject-list-with-redundant

	- name: Remove redundant domains
	run: \|
	chmod +x findRedundantDomain.py
	./findRedundantDomain.py ./direct-list-with-redundant ./direct-list-deleted-unsort
	./findRedundantDomain.py ./proxy-list-with-redundant ./proxy-list-deleted-unsort
	./findRedundantDomain.py ./reject-list-with-redundant ./reject-list-deleted-unsort
	[ ! -f "direct-list-deleted-unsort" ] && touch direct-list-deleted-unsort
	[ ! -f "proxy-list-deleted-unsort" ] && touch proxy-list-deleted-unsort
	[ ! -f "reject-list-deleted-unsort" ] && touch reject-list-deleted-unsort
	sort ./direct-list-deleted-unsort > ./direct-list-deleted-sort
	sort ./proxy-list-deleted-unsort > ./proxy-list-deleted-sort
	sort ./reject-list-deleted-unsort > ./reject-list-deleted-sort
	diff ./direct-list-deleted-sort ./direct-list-with-redundant \| awk '/^>/{print $2}' > ./direct-list-without-redundant
	diff ./proxy-list-deleted-sort ./proxy-list-with-redundant \| awk '/^>/{print $2}' > ./proxy-list-without-redundant
	diff ./reject-list-deleted-sort ./reject-list-with-redundant \| awk '/^>/{print $2}' > ./reject-list-without-redundant

	- name: Remove domains from "need-to-remove" lists in "hidden" branch
	run: \|
	diff ./direct-need-to-remove.txt ./direct-list-without-redundant \| awk '/^>/{print $2}' > temp-cn.txt
	diff ./proxy-need-to-remove.txt ./proxy-list-without-redundant \| awk '/^>/{print $2}' > temp-geolocation-\!cn.txt
	diff ./reject-need-to-remove.txt ./reject-list-without-redundant \| awk '/^>/{print $2}' > temp-category-ads-all.txt

	- name: Write lists to data directory
	run: \|
	cat temp-cn.txt \| sort --ignore-case -u \| perl -ne '/^((?=^.{1,255})[a-zA-Z0-9][-_a-zA-Z0-9]{0,62}(\.[a-zA-Z0-9][-_a-zA-Z0-9]{0,62})*)/ && print "$1\n"' > ./community/data/cn
	cat temp-cn.txt \| sort --ignore-case -u \| perl -ne 'print if not /^((?=^.{3,255})[a-zA-Z0-9][-_a-zA-Z0-9]{0,62}(\.[a-zA-Z0-9][-_a-zA-Z0-9]{0,62})+)/' > direct-tld-list.txt
	cat temp-geolocation-\!cn.txt \| sort --ignore-case -u \| perl -ne '/^((?=^.{1,255})[a-zA-Z0-9][-_a-zA-Z0-9]{0,62}(\.[a-zA-Z0-9][-_a-zA-Z0-9]{0,62})*)/ && print "$1\n"' > ./community/data/geolocation-\!cn
	cat temp-geolocation-\!cn.txt \| sort --ignore-case -u \| perl -ne 'print if not /^((?=^.{3,255})[a-zA-Z0-9][-_a-zA-Z0-9]{0,62}(\.[a-zA-Z0-9][-_a-zA-Z0-9]{0,62})+)/' > proxy-tld-list.txt
	cat temp-category-ads-all.txt \| sort --ignore-case -u \| perl -ne '/^((?=^.{1,255})[a-zA-Z0-9][-_a-zA-Z0-9]{0,62}(\.[a-zA-Z0-9][-_a-zA-Z0-9]{0,62})*)/ && print "$1\n"' > ./community/data/category-ads-all
	cat temp-category-ads-all.txt \| sort --ignore-case -u \| perl -ne 'print if not /^((?=^.{3,255})[a-zA-Z0-9][-_a-zA-Z0-9]{0,62}(\.[a-zA-Z0-9][-_a-zA-Z0-9]{0,62})+)/' > reject-tld-list.txt

	- name: Add `full`, `regexp` and `keyword` type of rules back into "cn", "geolocation-!cn" and "category-ads-all" list
	run: \|
	[ -f "direct-reserve.txt" ] && cat direct-reserve.txt >> ./community/data/cn
	[ -f "proxy-reserve.txt" ] && cat proxy-reserve.txt >> ./community/data/geolocation-\!cn
	[ -f "reject-reserve.txt" ] && cat reject-reserve.txt >> ./community/data/category-ads-all
	cp ./community/data/cn direct-list.txt
	cp ./community/data/geolocation-\!cn proxy-list.txt
	cp ./community/data/category-ads-all reject-list.txt

	- name: Create `china-list`、`google-cn`、`apple-cn`、`gfw`、`category-ir-half-price`、`blocked-by-taiwan` lists
	env:
	CENSORED_LIST: ${{ secrets.FATI_CENSOR_DOMAIN }}
	run: \|
	curl -sSL $CHINA_DOMAINS_URL \| perl -ne '/^server=\/([^\/]+)\// && print "$1\n"' > ./community/data/china-list
	curl -sSL $CHINA_DOMAINS_URL \| perl -ne '/^server=\/([^\/]+)\// && print "$1\n"' > china-list.txt
	curl -sSL $GOOGLE_DOMAINS_URL \| perl -ne '/^server=\/([^\/]+)\// && print "full:$1\n"' > ./community/data/google-cn
	curl -sSL $GOOGLE_DOMAINS_URL \| perl -ne '/^server=\/([^\/]+)\// && print "full:$1\n"' > google-cn.txt
	curl -sSL $APPLE_DOMAINS_URL \| perl -ne '/^server=\/([^\/]+)\// && print "full:$1\n"' > ./community/data/apple-cn
	curl -sSL $APPLE_DOMAINS_URL \| perl -ne '/^server=\/([^\/]+)\// && print "full:$1\n"' > apple-cn.txt
	cat ./gfwlist2dnsmasq/temp-gfwlist.txt \| perl -ne '/^((?=^.{3,255})[a-zA-Z0-9][-_a-zA-Z0-9]{0,62}(\.[a-zA-Z0-9][-_a-zA-Z0-9]{0,62})+)/ && print "$1\n"' >> ./community/data/gfw
	cat ./community/data/gfw \| sort --ignore-case -u > gfw.txt
	curl -sSL $WIN_SPY \| grep "0.0.0.0" \| awk '{print $2}' > ./community/data/win-spy
	curl -sSL $WIN_SPY \| grep "0.0.0.0" \| awk '{print $2}' > win-spy.txt
	curl -sSL $WIN_UPDATE \| grep "0.0.0.0" \| awk '{print $2}' > ./community/data/win-update
	curl -sSL $WIN_UPDATE \| grep "0.0.0.0" \| awk '{print $2}' > win-update.txt
	curl -sSL $WIN_EXTRA \| grep "0.0.0.0" \| awk '{print $2}' > ./community/data/win-extra
	curl -sSL $WIN_EXTRA \| grep "0.0.0.0" \| awk '{print $2}' > win-extra.txt
	curl -sSL $HALF_PRICE_IN_IRAN >> ./community/data/category-ir-half-price
	curl -sSL $BLOCKED_BY_TAIWAN >> ./community/data/blocked-by-taiwan
	cat ./community/data/blocked-by-taiwan >> blocked-by-taiwan.txt
	curl -sSL $BLOCKED_BY_RUSSIA >> ./community/data/blocked-by-russia
	curl -sSL ${CENSORED_LIST} >> ./community/data/faticensor

	- name: Build geosite.dat file and export FATIcensor
	run: \|
	cd custom \|\| exit 1
	go run ./ --datapath=../community/data --exportlists=faticensor

	- name: Move and zip files and generate sha256 hash
	run: \|
	install -Dp ./geoip.dat ./publish/geoip.dat
	install -Dp ./geoip.dat.sha256sum ./publish/geoip.dat.sha256sum
	install -Dp ./custom/publish/geosite.dat ./publish/geosite.dat
	install -p {proxy,direct,reject}-tld-list.txt ./publish/
	install -p {proxy,direct,reject}-list.txt ./publish/
	install -p {china-list,apple-cn,google-cn,gfw,win-spy,win-update,win-extra,blocked-by-taiwan}.txt ./publish/
	cd ./publish \|\| exit 1
	zip rules.zip {proxy,direct,reject}-list.txt geoip.dat geosite.dat
	sha256sum geosite.dat > geosite.dat.sha256sum
	sha256sum rules.zip > rules.zip.sha256sum

	- name: Generate sing-box RULE-SET format
	env:
	LOCAL_GEOSITE_PATH: ../publish/geosite.dat
	run: \|
	cd sing-geosite \|\| exit 1
	go run -v .

	- name: Collect sing-box outputs and make sha256sum
	run: \|
	mkdir -p sing-output
	cp -r sing-geosite/rule-set sing-output/
	cp -r sing-geosite/rule-set-unstable sing-output/
	sha256sum sing-geosite/geosite.db > sing-geosite/geosite.db.sha256sum
	sha256sum sing-geosite/geosite-cn.db > sing-geosite/geosite-cn.db.sha256sum
	cp sing-geosite/geosite* ./publish

	- name: Release and upload assets
	uses: svenstaro/upload-release-action@v2
	with:
	repo_token: ${{ secrets.GITHUB_TOKEN }}
	release_name: ${{ env.RELEASE_NAME }}
	tag: ${{ env.TAG_NAME }}
	file_glob: true
	file: ./publish/*

	- name: Move FATIcensor to publish folder
	run: \|
	install -p ./custom/publish/faticensor.txt ./publish/

	- name: Git push assets to "release" branch
	run: \|
	cd publish \|\| exit 1
	git init
	git config --local user.name "github-actions[bot]"
	git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com"
	git checkout -b release
	git add .
	git commit -m "${{ env.RELEASE_NAME }}"
	git remote add origin "https://${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }}"
	git push -f -u origin release

	- name: Purge jsdelivr CDN
	run: \|
	cd publish \|\| exit 1
	for file in $(ls); do
	curl -i "https://purge.jsdelivr.net/gh/${{ github.repository }}@release/${file}"
	done

	- name: Generate plaintext and Convert to the rules of AdGuard format
	run: \|
	cd custom \|\| exit 1
	curl -OL https://raw.githubusercontent.com/rootmelo92118/v2ray-rules-adguard/master/GenerateAndConvert.py
	mkdir AdGuard_Rule
	sed -i 's/--outputdir/--outputpath/g' GenerateAndConvert.py
	python3 GenerateAndConvert.py ../community/data ./AdGuard_Rule

	- name: Modify the head of FATIcensor
	run: \|
	cd custom/AdGuard_Rule \|\| exit 1
	sed -i 's/Source: https:\/\/github.com\/v2fly\/domain-list-community\/blob\/master\/data\/faticensor/It is the censorship list of Full Access To Internet./g' faticensor.txt

	- name: Git push assets to "AdGuard" branch
	run: \|
	cd custom/AdGuard_Rule \|\| exit 1
	git init
	git config --local user.name "github-actions[bot]"
	git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com"
	git checkout -b AdGuard
	git add *.txt
	git commit -m "${{ env.RELEASE_NAME }}"
	git remote add origin "https://${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }}"
	git push -f -u origin AdGuard
	rm -rf .git

	- name: Generate plaintext and Convert to the rules of clash RULE-SET format
	run: \|
	cd custom \|\| exit 1
	curl -OL https://raw.githubusercontent.com/rootmelo92118/v2ray-rules-adguard/master/GenerateAndConvertForClash.py
	mkdir clash_Rule_Set
	sed -i 's/--outputdir/--outputpath/g' GenerateAndConvertForClash.py
	python3 GenerateAndConvertForClash.py ../community/data ./clash_Rule_Set

	- name: Modify the head of FATIcensor
	run: \|
	cd custom/clash_Rule_Set \|\| exit 1
	sed -i 's/Source: https:\/\/github.com\/v2fly\/domain-list-community\/blob\/master\/data\/faticensor/It is the censorship list of Full Access To Internet./g' faticensor.yaml

	- name: Git push assets to "clash" branch
	run: \|
	cd custom/clash_Rule_Set \|\| exit 1
	git init
	git config --local user.name "github-actions[bot]"
	git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com"
	git checkout -b clash
	git add *.yaml
	git commit -m "${{ env.RELEASE_NAME }}"
	git remote add origin "https://${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }}"
	git push -f -u origin clash
	rm -rf .git

	- name: Git push assets to "sing-box" branch
	run: \|
	cd sing-output \|\| exit 1
	git init
	git config --local user.name "github-actions[bot]"
	git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com"
	git checkout -b sing-box
	git add .
	git commit -m "${{ env.RELEASE_NAME }}"
	git remote add origin "https://${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }}"
	git push -f -u origin sing-box
	rm -rf .git

Navigation Menu

Build V2Ray rules dat files #1157

Workflow file

Build V2Ray rules dat files #1157

Uh oh!

Workflow file for this run