Skip to content

Commit e8cf52f

Browse files
mboot-githubmboot-github
committed
prep global tld and updates others after analyze run
1 parent 046aa70 commit e8cf52f

26 files changed

+613
-587
lines changed

‎Makefile‎

Lines changed: 7 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -4,27 +4,23 @@
44

55
SHELL := /bin/bash -l
66

7-
RLSECURE := ~/tmp/rl-secure/rl-secure
8-
RLSTORE := ~/
9-
107
WHAT := whoisdomain
118
DOCKER_WHO := mbootgithub
129

1310
SIMPLEDOMAINS = $(shell ls testdata)
1411

15-
TEST_OPTIONS_ALL = --withPublicSuffix --extractServers --stripHttpStatus
12+
TEST_OPTIONS_ALL = \
13+
--withPublicSuffix \
14+
--extractServers \
15+
--stripHttpStatus
1616

1717
# PHONY targets: make will run its recipe regardless of whether a file with that name exists or what its last modification time is.
1818
.PHONY: TestSimple TestSimple2 TestAll clean
1919

20-
first: reformat mypy pylint testP39
21-
22-
testP39:
23-
./test1.py # now tests with python 3.9
20+
first: reformat mypy pylint testP310
2421

25-
#testP36:
26-
# docker build -t df36 -f Df-36 .
27-
# docker run -v .:/context df36 -d google.com
22+
testP310:
23+
./test1.py # now tests with python 3.10
2824

2925
second: first test2 test3 test
3026

@@ -51,48 +47,6 @@ build: first
5147
./bin/testLocalWhl.sh 2>2 | tee 1
5248
./bin/test.sh 2>2 | tee 1
5349

54-
# ==========================================================
55-
# scan the most recent build and fail if the status fails
56-
rlsecure: rlsecure-scan rlsecure-list rlsecure-status rlsecure-report rlsecure-version
57-
58-
rlsecure-scan:
59-
@export VERSION=$(shell cat work/version) && \
60-
$(RLSECURE) scan \
61-
--rl-store $(RLSTORE) \
62-
--purl mboot-github/$(WHAT)@$${VERSION} \
63-
--file-path dist/$(WHAT)-$${VERSION}*.whl \
64-
--replace \
65-
--no-tracking
66-
67-
rlsecure-list:
68-
@export VERSION=$(shell cat work/version) && \
69-
$(RLSECURE) list \
70-
--rl-store $(RLSTORE) \
71-
--show-all \
72-
--purl mboot-github/$(WHAT)@$${VERSION} \
73-
--no-color | tee rlsecure/list-$${VERSION}.txt
74-
75-
rlsecure-status:
76-
@export VERSION=$(shell cat work/version) && \
77-
$(RLSECURE) status \
78-
--rl-store $(RLSTORE) \
79-
--purl mboot-github/$(WHAT)@$${VERSION} \
80-
--show-all \
81-
--return-status \
82-
--no-color | tee rlsecure/status-$${VERSION}.txt
83-
84-
rlsecure-report:
85-
@export VERSION=$(shell cat work/version) && \
86-
$(RLSECURE) report \
87-
--rl-store $(RLSTORE) \
88-
--purl mboot-github/$(WHAT)@$${VERSION} \
89-
--format=all \
90-
--bundle=report-$(WHAT)-$${VERSION}.zip \
91-
--output-path ./rlsecure
92-
93-
rlsecure-version:
94-
@$(RLSECURE) --version
95-
9650
# ==========================================================
9751
# build a docker images with the latest python and run a test -a
9852
dockerTests: docker dockerRunLocal dockerTestdata

‎analizer/Makefile‎

Lines changed: 91 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,95 @@
1+
# Makefile tab=tab,ts=4
2+
# ==========================================
13

2-
all: clean format
3-
./analizeIanaTld.py
4-
./investigateTld.py 2>2 | tee 1
4+
SHELL := /bin/bash
5+
export SHELL
6+
7+
VENV := ./vtmp/
8+
export VENV
9+
10+
# tested on 3.10-3.14
11+
MIN_PYTHON_VERSION := $(shell basename $$( ls /usr/bin/python3.[0-9][0-9] | awk '{print $0; exit}' ) )
12+
export MIN_PYTHON_VERSION
13+
14+
PIP_INSTALL := pip3 -q \
15+
--require-virtualenv \
16+
--disable-pip-version-check \
17+
--no-color install --no-cache-dir
18+
19+
# ==========================================
20+
# Code formatting and checks
21+
PY_FILES := \
22+
*.py
23+
24+
LINE_LENGTH := 120
25+
PL_LINTERS := eradicate,mccabe,pycodestyle,pyflakes,pylint
26+
27+
# C0114 Missing module docstring [pylint]
28+
# C0115 Missing class docstring [pylint]
29+
# C0116 Missing function or method docstring [pylint]
30+
# E203 whitespace before ':' [pycodestyle]
31+
#
32+
# W0105 String statement has no effect
33+
# C901 : is to complex
34+
35+
PL_IGNORE := C0114,C0115,C0116,E203
36+
37+
MYPY_INSTALL := \
38+
types-requests \
39+
types-python-dateutil \
40+
spectra-assure-sdk
41+
42+
COMMON_VENV := rm -rf $(VENV); \
43+
$(MIN_PYTHON_VERSION) -m venv $(VENV); \
44+
source ./$(VENV)/bin/activate;
545

6-
format:
7-
black --line-length 160 *.py
8-
-pylama -m 160 *.py
46+
.PHONEY: clean prep
947

10-
clean:
11-
rm IanaDb.sqlite
48+
# ==========================================
49+
50+
all: clean prep test
51+
52+
clean: cleanupVenv
53+
rm -f *.1 *.2 *.log *.tmp 1 2
54+
rm -rf .mypy_cache
55+
rm -f IanaDb.sqlite
1256
rm -rf .iana_cache/ .psl_cache/
57+
58+
# cleanup the virtual env
59+
cleanupVenv:
60+
rm -rf $(VENV)
61+
62+
# ======================================
63+
# prep the code with format, lint typing
64+
65+
prep: black pylama
66+
67+
black:
68+
$(COMMON_VENV) \
69+
$(PIP_INSTALL) black; \
70+
black \
71+
--line-length $(LINE_LENGTH) \
72+
$(PY_FILES)
73+
74+
pylama:
75+
$(COMMON_VENV) \
76+
$(PIP_INSTALL) setuptools pylama; \
77+
pylama \
78+
--max-line-length $(LINE_LENGTH) \
79+
--linters "${PL_LINTERS}" \
80+
--ignore "${PL_IGNORE}" \
81+
$(PY_FILES)
82+
83+
mypy:
84+
$(COMMON_VENV) \
85+
$(PIP_INSTALL) mypy $(MYPY_INSTALL); \
86+
mypy \
87+
--strict \
88+
--no-incremental \
89+
$(PY_FILES)
90+
91+
test:
92+
$(COMMON_VENV) \
93+
$(PIP_INSTALL) -r requirements.txt ;\
94+
./analizeIanaTld.py; \
95+
./investigateTld.py 2>2 | tee 1

‎analizer/analizeIanaTld.py‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,17 +4,13 @@
44
Analyze all tld's currently in the iana root db
55
"""
66

7-
# from typing import ( Any)
8-
97
import io
108
import re
119
from dns.resolver import (
1210
Resolver,
1311
LRUCache,
1412
)
1513

16-
# import json
17-
1814
from ianaCrawler import IanaCrawler
1915
from pslGrabber import PslGrabber
2016
from ianaDatabase import IanaDatabase

‎analizer/investigateTld.py‎

Lines changed: 14 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818

1919
# the next 2 belong together
2020
sys.path.append("..")
21-
from whoisdomain.tldDb import tld_regexpr
21+
from whoisdomain.tldDb import tld_regexpr # noqa: E402
2222

2323
MM = {
2424
"com": [
@@ -232,6 +232,16 @@ def processRow(
232232
allTld: Dict[str, Any],
233233
ss: List[str],
234234
):
235+
sequence = [
236+
self._skipSpecialResolve,
237+
self._doUtf8Preparations,
238+
self._skipKnowTld,
239+
self._doNoManagerTld,
240+
self._doFoundTld,
241+
self._doNoWhois,
242+
self._doCleanuphois,
243+
]
244+
235245
self.row = row
236246
self.allTld = allTld
237247
self.ss = ss
@@ -248,26 +258,9 @@ def processRow(
248258

249259
self.thisTld = self.allKnownTldDict[self.tld]
250260

251-
if self._skipSpecialResolve():
252-
return
253-
254-
if self._doUtf8Preparations():
255-
return
256-
257-
if self._skipKnowTld():
258-
return
259-
260-
if self._doNoManagerTld():
261-
return
262-
263-
if self._doFoundTld():
264-
return
265-
266-
if self._doNoWhois():
267-
return
268-
269-
if self._doCleanuphois():
270-
return
261+
for n in sequence:
262+
if n():
263+
return
271264

272265
print("# MISSING", self.tld, self.tld2, self.tld3, self.manager.replace("\n", ";"), self.w, self.resolve, self.reg)
273266

0 commit comments

Comments
 (0)