Skip to content

Commit 9fa6a9b

Browse files
dvershinindvershinin
committed
feat: add region unblock, improve FileCache fallback, enhance tests
- Add unblock_region() with proper Americas/subregion normalization - Make FileCache import/init graceful with fallback to in-memory cache - Add FDS_NOCACHE env var for test/container environments - Handle HTTP errors when fetching country network lists - Ensure download destination directories exist - Silence cloudflare PendingDeprecationWarning - Pin cloudflare<2.20, add filelock dependency - Rewrite firewalld-tests.sh to work without systemd - Add continent block/unblock functional tests - Add Makefile for Docker-based test workflow - Add pytest smoke test for region unblock
1 parent 11dd7b2 commit 9fa6a9b

File tree

8 files changed

+333
-17
lines changed

8 files changed

+333
-17
lines changed

‎Makefile‎

Lines changed: 147 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,147 @@
1+
# Makefile to build and run firewalld tests in Docker
2+
3+
IMAGE ?= my-firewalld-image
4+
CONTAINER ?= firewalld-container
5+
DOCKERFILE ?= firewalld.Dockerfile
6+
UNAME_S := $(shell uname -s)
7+
DOCKER ?= docker
8+
DOCKER_CONTEXT := $(shell docker context show 2>/dev/null)
9+
10+
# On some hosts (e.g. macOS), you may need to override CGROUP_FLAGS to empty:
11+
# make test CGROUP_FLAGS=
12+
CGROUP_FLAGS ?= --volume=/sys/fs/cgroup:/sys/fs/cgroup:rw --cgroupns=host
13+
TMPFS_FLAGS ?= --tmpfs /run --tmpfs /run/lock
14+
RUN_FLAGS ?= --detach --privileged $(CGROUP_FLAGS) --name $(CONTAINER)
15+
TEST_SCRIPT := $(abspath firewalld-tests.sh)
16+
PKG_DIR := $(abspath fds)
17+
18+
.PHONY: build rebuild ensure-image start start-mac stop rm logs shell exec test test-mac tests tests-colima ensure-env stop-colima-if-needed clean up down
19+
20+
build:
21+
@DOCKER_BUILDKIT=0 $(DOCKER) build -t $(IMAGE) -f $(DOCKERFILE) .
22+
23+
rebuild:
24+
@DOCKER_BUILDKIT=0 $(DOCKER) build --no-cache -t $(IMAGE) -f $(DOCKERFILE) .
25+
26+
ensure-image:
27+
@$(DOCKER) image inspect $(IMAGE) >/dev/null 2>&1 || (echo "Building $(IMAGE)..." && DOCKER_BUILDKIT=0 $(DOCKER) build -t $(IMAGE) -f $(DOCKERFILE) .)
28+
29+
start:
30+
@-$(DOCKER) rm -f $(CONTAINER) >/dev/null 2>&1 || true
31+
@$(DOCKER) run $(RUN_FLAGS) $(IMAGE)
32+
@sleep 10
33+
@running=$$($(DOCKER) inspect -f '{{.State.Running}}' $(CONTAINER) 2>/dev/null || true); \
34+
if [ "$$running" != "true" ]; then \
35+
echo "Container failed to start. Logs:"; \
36+
$(DOCKER) logs $(CONTAINER) || true; \
37+
$(DOCKER) inspect $(CONTAINER) || true; \
38+
exit 1; \
39+
fi
40+
41+
stop:
42+
@-$(DOCKER) stop $(CONTAINER) >/dev/null 2>&1 || true
43+
44+
rm: stop
45+
@-$(DOCKER) rm $(CONTAINER) >/dev/null 2>&1 || true
46+
47+
logs:
48+
@$(DOCKER) logs -f $(CONTAINER)
49+
50+
shell:
51+
@$(DOCKER) exec -it $(CONTAINER) /bin/bash
52+
53+
exec:
54+
@$(DOCKER) exec $(CONTAINER) /bin/bash -c "$(CMD)"
55+
56+
test: ensure-image start
57+
@$(DOCKER) cp $(TEST_SCRIPT) $(CONTAINER):/app/firewalld-tests.sh
58+
@$(DOCKER) cp -a $(PKG_DIR) $(CONTAINER):/usr/local/lib/python3.9/site-packages/
59+
@$(DOCKER) exec -e FDS_NOCACHE=1 $(CONTAINER) /bin/bash -c "./firewalld-tests.sh"
60+
61+
start-mac: CGROUP_FLAGS=
62+
start-mac:
63+
@-$(DOCKER) rm -f $(CONTAINER) >/dev/null 2>&1 || true
64+
@$(DOCKER) run --detach --privileged $(TMPFS_FLAGS) --name $(CONTAINER) $(IMAGE) sleep infinity
65+
@sleep 10
66+
@running=$$($(DOCKER) inspect -f '{{.State.Running}}' $(CONTAINER) 2>/dev/null || true); \
67+
if [ "$$running" != "true" ]; then \
68+
echo "Container failed to start. Logs:"; \
69+
$(DOCKER) logs $(CONTAINER) || true; \
70+
$(DOCKER) inspect $(CONTAINER) || true; \
71+
exit 1; \
72+
fi
73+
74+
test-mac: CGROUP_FLAGS=
75+
test-mac: ensure-image start-mac
76+
@$(DOCKER) cp $(TEST_SCRIPT) $(CONTAINER):/app/firewalld-tests.sh
77+
@$(DOCKER) cp -a $(PKG_DIR) $(CONTAINER):/usr/local/lib/python3.9/site-packages/
78+
@$(DOCKER) exec -e FDS_NOCACHE=1 $(CONTAINER) /bin/bash -c "./firewalld-tests.sh"
79+
80+
ensure-env:
81+
@# Ensure environment for tests (start Colima on macOS if available)
82+
@if [ "$(UNAME_S)" = "Darwin" ]; then \
83+
if command -v colima >/dev/null 2>&1; then \
84+
if ! colima status 2>/dev/null | grep -q Running; then \
85+
echo "Starting Colima for systemd-compatible Docker..."; \
86+
colima start --cpu 2 --memory 4 --disk 20 || true; \
87+
touch .colima-started-by-make; \
88+
fi; \
89+
else \
90+
if command -v brew >/dev/null 2>&1; then \
91+
echo "Colima not found. Installing via Homebrew..."; \
92+
brew install colima || true; \
93+
if command -v colima >/dev/null 2>&1; then \
94+
echo "Starting Colima..."; \
95+
colima start --cpu 2 --memory 4 --disk 20 || true; \
96+
touch .colima-started-by-make; \
97+
fi; \
98+
fi; \
99+
fi; \
100+
fi
101+
102+
tests: ensure-env
103+
@status=0; \
104+
if [ "$(UNAME_S)" = "Darwin" ] && command -v colima >/dev/null 2>&1 && colima status 2>/dev/null | grep -q Running; then \
105+
echo "Using Colima Docker context"; \
106+
$(MAKE) DOCKER='docker --context colima' DOCKER_BUILDKIT=0 test; \
107+
status=$$?; \
108+
else \
109+
if [ "$(UNAME_S)" = "Darwin" ]; then \
110+
echo "Colima not available; using Docker Desktop fallback"; \
111+
$(MAKE) DOCKER='docker' test-mac; \
112+
status=$$?; \
113+
else \
114+
$(MAKE) DOCKER='docker' test; \
115+
status=$$?; \
116+
fi; \
117+
fi; \
118+
$(MAKE) stop-colima-if-needed; \
119+
exit $$status
120+
121+
tests-colima:
122+
@status=0; \
123+
if ! command -v colima >/dev/null 2>&1 || ! colima status 2>/dev/null | grep -q Running; then \
124+
echo "Starting Colima for systemd-compatible Docker..."; \
125+
colima start --cpu 2 --memory 4 --disk 20 || true; \
126+
touch .colima-started-by-make; \
127+
fi; \
128+
$(MAKE) DOCKER='docker --context colima' DOCKER_BUILDKIT=0 test; \
129+
status=$$?; \
130+
$(MAKE) stop-colima-if-needed; \
131+
exit $$status
132+
133+
stop-colima-if-needed:
134+
@if [ -f .colima-started-by-make ]; then \
135+
echo "Stopping Colima started by tests..."; \
136+
colima stop; \
137+
rm -f .colima-started-by-make; \
138+
fi
139+
140+
clean: rm
141+
@-$(DOCKER) rmi $(IMAGE) >/dev/null 2>&1 || true
142+
143+
up: start
144+
145+
down: rm
146+
147+

‎cds/__init__.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1 +1,7 @@
1+
import warnings
2+
3+
# Silence PendingDeprecationWarning emitted by the 'cloudflare' library
4+
# during client initialization; we pin <2.20 but some releases still warn.
5+
warnings.filterwarnings('ignore', category=PendingDeprecationWarning)
6+
17
from .cds import cf

‎fds/WebClient.py‎

Lines changed: 31 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,12 @@
33
import logging as log
44

55
import requests
6+
import os
67
from cachecontrol import CacheControl
7-
from cachecontrol.caches import FileCache
8+
try:
9+
from cachecontrol.caches import FileCache
10+
except Exception:
11+
FileCache = None
812
from tqdm import tqdm
913

1014
from .__about__ import __version__
@@ -41,7 +45,18 @@ class WebClient:
4145
def __init__(self):
4246
s = requests.session()
4347
s.headers.update({'User-Agent': 'fds/{}'.format(__version__)})
44-
self.cs = CacheControl(s, cache=FileCache('/var/cache/fds'))
48+
# Allow disabling on-disk cache via env var (useful in tests/containers)
49+
if os.getenv('FDS_NOCACHE') == '1':
50+
self.cs = CacheControl(s)
51+
else:
52+
cache = None
53+
if FileCache is not None:
54+
try:
55+
cache = FileCache('/var/cache/fds')
56+
except Exception:
57+
cache = None
58+
# Fallback to in-memory cache if file cache cannot be used
59+
self.cs = CacheControl(s, cache=cache) if cache is not None else CacheControl(s)
4560

4661
def download_file(self, url, local_filename=None, display_name=None,
4762
return_type='filename'):
@@ -70,6 +85,10 @@ def download_file(self, url, local_filename=None, display_name=None,
7085
desc='Downloading {}'.format(local_filename if not display_name else display_name),
7186
leave=True # progressbar stays
7287
)
88+
# Ensure destination directory exists
89+
dest_dir = os.path.dirname(local_filename)
90+
if dest_dir and not os.path.isdir(dest_dir):
91+
os.makedirs(dest_dir, exist_ok=True)
7392
with open(local_filename, 'wb') as f:
7493
for chunk in r.iter_content(chunk_size=chunk_size):
7594
if chunk: # filter out keep-alive new chunks
@@ -91,12 +110,16 @@ def get_country_networks(self, country):
91110
# )
92111
url = get_country_ipblocks_url(country)
93112
log.debug('Downloading {}'.format(url))
94-
content = self.download_file(
95-
url,
96-
display_name='{} networks list'.format(country.getNation()),
97-
local_filename=get_country_zone_filename(country),
98-
return_type='contents'
99-
)
113+
try:
114+
content = self.download_file(
115+
url,
116+
display_name='{} networks list'.format(country.getNation()),
117+
local_filename=get_country_zone_filename(country),
118+
return_type='contents'
119+
)
120+
except requests.exceptions.HTTPError as e:
121+
log.warning('Failed to fetch networks for %s (%s): %s', country.name, country.code, e)
122+
return []
100123
return content.splitlines()
101124

102125
def get_tor_exits(self, family=4):

‎fds/fds.py‎

Lines changed: 30 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,10 +39,33 @@ def block_region(region):
3939
countries = Countries()
4040
fw = FirewallWrapper() # Create a single instance outside the loop
4141
for country in countries:
42-
if country.data['region'] == region:
42+
# normalize region same way as Countries.get_continents()
43+
country_region = country.data['region']
44+
if country_region == 'Americas':
45+
country_region = country.data['subregion']
46+
if country_region in ['Caribbean', 'Central America']:
47+
country_region = 'North America'
48+
if country_region == region:
4349
action_block(country.name, reload=False, fw=fw)
4450

4551

52+
def unblock_region(region):
53+
"""
54+
Unblock all countries within a given region/continent.
55+
Mirrors block_region's normalization logic.
56+
"""
57+
countries = Countries()
58+
fw = FirewallWrapper()
59+
for country in countries:
60+
country_region = country.data['region']
61+
if country_region == 'Americas':
62+
country_region = country.data['subregion']
63+
if country_region in ['Caribbean', 'Central America']:
64+
country_region = 'North America'
65+
if country_region == region:
66+
fw.unblock_country(country.name)
67+
68+
4669
def action_block(ip_or_country_name, ipset_name=None, reload=True, fw=None):
4770
# FD
4871
if fw is None:
@@ -92,7 +115,12 @@ def action_unblock(ip_or_country_name):
92115
fw.unblock_ip(ip_or_country_name)
93116
cw.unblock_ip(ip_or_country_name)
94117
except ValueError:
95-
fw.unblock_country(ip_or_country_name)
118+
countries = Countries()
119+
regions = countries.get_continents()
120+
if ip_or_country_name in regions:
121+
unblock_region(ip_or_country_name)
122+
else:
123+
fw.unblock_country(ip_or_country_name)
96124

97125

98126
def action_reset():

‎firewalld-tests.sh‎

Lines changed: 70 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,74 @@
11
#!/bin/bash
2+
set -euo pipefail
23
set -x
34

4-
systemctl start dbus
5-
sleep 5
6-
systemctl start firewalld
7-
sleep 5
5+
have_systemd() {
6+
# True if PID 1 is systemd
7+
[ "$(cat /proc/1/comm 2>/dev/null || echo)" = "systemd" ]
8+
}
89

9-
fds block 1.2.3.4
10+
start_dbus_nosystemd() {
11+
mkdir -p /var/run/dbus /run/lock
12+
if command -v dbus-uuidgen >/dev/null 2>&1; then
13+
dbus-uuidgen --ensure
14+
elif command -v systemd-machine-id-setup >/dev/null 2>&1; then
15+
systemd-machine-id-setup
16+
elif [ ! -s /etc/machine-id ]; then
17+
cat /proc/sys/kernel/random/uuid | tr -d '-' | head -c 32 > /etc/machine-id
18+
fi
19+
# Start system bus in background
20+
dbus-daemon --system --fork
21+
}
22+
23+
start_firewalld_nosystemd() {
24+
# Start firewalld in background and wait for it to become ready
25+
firewalld --nofork &
26+
for i in {1..20}; do
27+
if firewall-cmd --state >/dev/null 2>&1; then
28+
break
29+
fi
30+
sleep 1
31+
done
32+
# Do not fail if it's not running; just print status for diagnostics
33+
firewall-cmd --state || echo "firewalld not running"
34+
}
35+
36+
if have_systemd; then
37+
systemctl start dbus
38+
sleep 3
39+
systemctl start firewalld
40+
# Give some time for firewalld to come up
41+
for i in {1..20}; do
42+
if firewall-cmd --state >/dev/null 2>&1; then
43+
break
44+
fi
45+
sleep 1
46+
done
47+
else
48+
start_dbus_nosystemd
49+
sleep 1
50+
start_firewalld_nosystemd
51+
fi
52+
53+
# Simple smoke test (only if firewalld is up)
54+
if firewall-cmd --state 2>/dev/null | grep -q running; then
55+
# Smoke test: block single IP
56+
fds block 1.2.3.4
57+
58+
# Functional test: block and unblock a continent (Europe)
59+
# Expect an EU country's ipset (e.g., Germany) to appear then disappear
60+
echo "Blocking continent: Europe"
61+
fds block Europe
62+
firewall-cmd --get-ipsets | grep -q 'fds-de-4'
63+
64+
echo "Unblocking continent: Europe"
65+
fds unblock Europe
66+
if firewall-cmd --get-ipsets | grep -q 'fds-de-4'; then
67+
echo 'Expected fds-de-4 to be removed after unblocking Europe';
68+
exit 1;
69+
fi
70+
else
71+
echo "Firewalld is not running in this environment. Skipping functional firewall test."
72+
fi
73+
74+
exit 0

‎firewalld.Dockerfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
FROM dvershinin/systemd-base:latest
22

33
## Install systemd and firewalld
4-
RUN yum install -y systemd firewalld dbus python3-pip git \
4+
RUN yum install -y systemd firewalld dbus dbus-daemon dbus-tools which python3-pip git \
55
&& yum clean all \
66
&& systemctl enable firewalld
77

‎setup.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,11 +14,12 @@
1414
'six',
1515
'netaddr',
1616
'cachecontrol',
17+
'filelock>=3.0.0',
1718
'tqdm',
1819
# required for cf.user.tokens.verify.get() and cf.accounts.get()
1920
# neither endpoints are available in CentOS 7, thus we rebuild pkg from
2021
# https://github.com/dvershinin/python-cloudflare/releases/tag/2.7.1
21-
'cloudflare>=2.7.1,<3',
22+
'cloudflare>=2.7.1,<2.20',
2223
'ipaddress; python_version < "3.0.0"'
2324
]
2425
tests_requires = ["pytest", "flake8", "faker"]

0 commit comments

Comments
 (0)