1+ import fnmatch
2+ import os
13import re
24import subprocess
35from collections import Counter , defaultdict
6+ from contextlib import contextmanager
47from pathlib import Path
8+ from typing import Dict , List , Literal , NamedTuple , Optional , Set
59
610from packaging .version import Version
711
12+
13+ @contextmanager
14+ def printing_table (columns : List [str ]):
15+ print ("|" + "|" .join (columns ) + "|" )
16+ print ("|" + "|" .join (["-" * len (c ) for c in columns ]) + "|" )
17+
18+ yield
19+
20+ # print("|" + "|".join(["-" * len(c) for c in columns]) + "|")
21+
22+
823BEFORE_PATTERN = re .compile (r"^-([\w-]+)==([0-9\.post]+)" )
924AFTER_PATTERN = re .compile (r"^\+([\w-]+)==([0-9\.post]+)" )
1025
@@ -38,7 +53,7 @@ def parse_changes(filename: Path):
3853 return before , after , Counter (changes )
3954
4055
41- def main ():
56+ def main_changes_stats ():
4257
4358 filepath = Path ("changes.ignore.keep.log" )
4459 if not filepath .exists ():
@@ -47,41 +62,131 @@ def main():
4762 before , after , counts = parse_changes (filepath )
4863
4964 # format
50- print ("Stats " )
65+ print ("Overview of changes in dependencies " )
5166 print ("- #packages before:" , len (before ))
5267 print ("- #packages after :" , len (after ))
5368 print ()
5469
5570 COLUMNS = ["#" , "name" , "before" , "after" , "upgrade" , " count" ]
5671
57- print ("|" + "|" .join (COLUMNS ) + "|" )
58- print ("|" + "|" .join (["-" * len (c ) for c in COLUMNS ]) + "|" )
59- for i , name in enumerate (sorted (before .keys ()), start = 1 ):
60- # TODO: where are these libraries?
61- # TODO: are they first dependencies?
62- # TODO: if major, get link to release notes
63- from_versions = set (str (v ) for v in before [name ])
64- to_versions = set (str (v ) for v in after [name ])
65-
66- print (
67- "|" ,
68- f"{ i :2d} ,
69- "|" ,
70- f"{ name :25s} ,
71- "|" ,
72- f'{ ", " .join (from_versions ):15s} ,
73- "|" ,
74- f'{ "," .join (to_versions ) if to_versions else "removed" :10s} ,
75- "|" ,
76- # how big the version change is
77- f"{ tag_upgrade (sorted (set (before [name ]))[- 1 ], sorted (set (after [name ]))[- 1 ]):10s}
78- if to_versions
79- else "" ,
80- "|" ,
81- counts [name ],
82- "|" ,
72+ with printing_table (COLUMNS ):
73+
74+ for i , name in enumerate (sorted (before .keys ()), start = 1 ):
75+ # TODO: where are these libraries?
76+ # TODO: are they first dependencies?
77+ # TODO: if major, get link to release notes
78+ from_versions = set (str (v ) for v in before [name ])
79+ to_versions = set (str (v ) for v in after [name ])
80+
81+ print (
82+ "|" ,
83+ f"{ i :3d} ,
84+ "|" ,
85+ f"{ name :25s} ,
86+ "|" ,
87+ f'{ ", " .join (from_versions ):15s} ,
88+ "|" ,
89+ f'{ "," .join (to_versions ) if to_versions else "removed" :10s} ,
90+ "|" ,
91+ # how big the version change is
92+ f"{ tag_upgrade (sorted (set (before [name ]))[- 1 ], sorted (set (after [name ]))[- 1 ]):10s}
93+ if to_versions
94+ else "" ,
95+ "|" ,
96+ counts [name ],
97+ "|" ,
98+ )
99+
100+
101+ ## Stats on installed packages (i.e. defined in txt files)
102+ DEPENDENCY = re .compile (r"([\w_-]+)==([0-9\.-]+)" )
103+
104+
105+ def parse_dependencies_in_reqfile (reqfile : Path ) -> Dict [str , Version ]:
106+ name2version = {}
107+ for name , version in DEPENDENCY .findall (reqfile .read_text ()):
108+ # TODO: typing-extensions==4.0.1 ; python_version < "3.9" might intro multiple versions
109+ assert name not in name2version , f"{ name } { reqfile }
110+ name2version [name ] = Version (version )
111+ return name2version
112+
113+
114+ class ReqFile (NamedTuple ):
115+ path : Path
116+ target : Literal ["base" , "test" , "tool" , "other" ]
117+ dependencies : Dict [str , Version ]
118+
119+
120+ def parse_dependencies (
121+ repodir : Path , * , exclude : Optional [Set ] = None
122+ ) -> List [ReqFile ]:
123+ reqs = []
124+ exclude = exclude or set ()
125+ for reqfile in repodir .rglob ("**/requirements/_*.txt" ):
126+ if any (fnmatch .fnmatch (reqfile , x ) for x in exclude ):
127+ continue
128+ try :
129+ t = {"_base.txt" : "base" , "_test.txt" : "test" , "_tools.txt" : "tool" }[
130+ reqfile .name
131+ ]
132+ except KeyError :
133+ if "test" in f"{ reqfile .parent } :
134+ t = "test"
135+ else :
136+ t = "other"
137+
138+ reqs .append (
139+ ReqFile (
140+ path = reqfile ,
141+ target = t ,
142+ dependencies = parse_dependencies_in_reqfile (reqfile ),
143+ )
83144 )
145+ return reqs
146+
147+
148+ def main_dep_stats (exclude : Optional [Set ] = None ):
149+ repodir = Path (os .environ .get ("REPODIR" , "." ))
150+ reqs = parse_dependencies (repodir , exclude = exclude )
151+
152+ # format
153+ print ("Overview of libraries used repo-wide" )
154+ print ("- #reqs files parsed:" , len (reqs ))
155+ print ()
156+
157+ deps = defaultdict (lambda : defaultdict (list ))
158+ for r in reqs :
159+ for name , version in r .dependencies .items ():
160+ deps [name ]["name" ] = name
161+ deps [name ][r .target ].append (version )
162+
163+ with printing_table (
164+ columns = ["#" , "name" , "versions-base" , "versions-test" , "versons-tool" ]
165+ ):
166+ for i , name in enumerate (sorted (deps .keys ()), start = 1 ):
167+
168+ def _norm (thing ):
169+ return [f"{ v } for v in sorted (list (set (thing )))]
170+
171+ bases = _norm (deps [name ]["base" ])
172+ tests = _norm (deps [name ]["test" ])
173+ tools = _norm (deps [name ]["tool" ])
174+
175+ print (
176+ "|" ,
177+ f"{ i :3d} ,
178+ "|" ,
179+ f"{ name :25s} ,
180+ "|" ,
181+ f'{ ", " .join (bases ):25s} ,
182+ "|" ,
183+ f'{ ", " .join (tests ):25s} ,
184+ "|" ,
185+ f'{ ", " .join (tools ):25s} ,
186+ "|" ,
187+ )
84188
85189
86190if __name__ == "__main__" :
87- main ()
191+ # main_changes_stats()
192+ main_dep_stats (exclude = {"*/director/*" })
0 commit comments