|
| 1 | +#!/usr/bin/env python3 |
| 2 | +""" |
| 3 | +Analyzes out.requests.json in test root dir and pretty prints HTTP requests. |
| 4 | +
|
| 5 | +By default ignores requests with method==GET (unless --get option is passed). |
| 6 | +Free standing arguments are substrings matching path. |
| 7 | +If argument starts with ! then it's a negation filter. |
| 8 | +
|
| 9 | +Examples: |
| 10 | + print_requests.py //jobs # Show non-GET requests with /jobs in path |
| 11 | + print_requests.py --get //jobs # Show all requests with /jobs in path |
| 12 | + print_requests.py --sort '^//import-file/' # Show non-GET requests, exclude /import-file/, sort output |
| 13 | + print_requests.py --keep //jobs # Show requests and do not delete out.requests.json afterwards |
| 14 | +
|
| 15 | +This replaces custom jq wrappers like: |
| 16 | + jq --sort-keys 'select(.method != "GET" and (.path | contains("/jobs")))' < out.requests.txt |
| 17 | +
|
| 18 | +>>> test_requests = [ |
| 19 | +... {"method": "GET", "path": "/api/2.0/clusters/list"}, |
| 20 | +... {"method": "POST", "path": "/api/2.1/jobs/create", "body": {"name": "test"}}, |
| 21 | +... {"method": "GET", "path": "/api/2.0/jobs/123"}, |
| 22 | +... {"method": "PUT", "path": "/api/2.0/jobs/123", "body": {"name": "updated"}}, |
| 23 | +... {"method": "POST", "path": "/api/2.0/workspace/import-file/test.py"}, |
| 24 | +... {"method": "DELETE", "path": "/api/2.0/jobs/123"} |
| 25 | +... ] |
| 26 | +
|
| 27 | +>>> def short_name(x): |
| 28 | +... ind = test_requests.index(x) |
| 29 | +... return f'R{ind} {x["method"]}' |
| 30 | +>>> def test(*args): |
| 31 | +... r = filter_requests(*args) |
| 32 | +... for x in r: |
| 33 | +... print(short_name(x)) |
| 34 | +
|
| 35 | +>>> test(test_requests, ["//jobs"], False, False) |
| 36 | +R1 POST |
| 37 | +R3 PUT |
| 38 | +R5 DELETE |
| 39 | +
|
| 40 | +>>> test(test_requests, ["//jobs"], True, False) |
| 41 | +R1 POST |
| 42 | +R2 GET |
| 43 | +R3 PUT |
| 44 | +R5 DELETE |
| 45 | +
|
| 46 | +>>> test(test_requests, ["^//import-file/"], False, False) |
| 47 | +R1 POST |
| 48 | +R3 PUT |
| 49 | +R5 DELETE |
| 50 | +
|
| 51 | +>>> # Test multiple positive filters (OR logic) |
| 52 | +>>> test(test_requests, ["//clusters", "//import-file"], True, False) |
| 53 | +R0 GET |
| 54 | +R4 POST |
| 55 | +
|
| 56 | +>>> # Test positive + negative filters (AND logic) |
| 57 | +>>> test(test_requests, ["//api", "^/jobs"], False, False) |
| 58 | +R4 POST |
| 59 | +""" |
| 60 | + |
| 61 | +import os |
| 62 | +import sys |
| 63 | +import json |
| 64 | +import argparse |
| 65 | +from pathlib import Path |
| 66 | + |
| 67 | + |
| 68 | +# I've originally tried ADD_PREFIX to be empty, so you can just do "print_requests.py /jobs" |
| 69 | +# However, that causes test to fail on Windows CI because "/jobs" becomes "C:/Program Files/Git/jobs" |
| 70 | +# This behaviour can be disabled with MSYS_NO_PATHCONV=1 but that causes other failures, so we require extra slash here. |
| 71 | +ADD_PREFIX = "/" |
| 72 | +NEGATE_PREFIX = "^/" |
| 73 | + |
| 74 | + |
| 75 | +def read_json_many(s): |
| 76 | + result = [] |
| 77 | + |
| 78 | + try: |
| 79 | + dec = json.JSONDecoder() |
| 80 | + pos = 0 |
| 81 | + n = len(s) |
| 82 | + while True: |
| 83 | + # skip whitespace between objects |
| 84 | + while pos < n and s[pos].isspace(): |
| 85 | + pos += 1 |
| 86 | + if pos >= n: |
| 87 | + break |
| 88 | + obj, idx = dec.raw_decode(s, pos) |
| 89 | + result.append(obj) |
| 90 | + pos = idx |
| 91 | + |
| 92 | + except Exception as ex: |
| 93 | + sys.exit(str(ex)) |
| 94 | + |
| 95 | + if not result and s: |
| 96 | + sys.stderr.write(f"WARNING: could not parse {len(s)} chars: {s!r}\n") |
| 97 | + |
| 98 | + return result |
| 99 | + |
| 100 | + |
| 101 | +# quick self-test |
| 102 | +test = '{"method": "GET"}\n{"method":\n"POST"\n}\n' |
| 103 | +result = read_json_many(test) |
| 104 | +assert result == [{"method": "GET"}, {"method": "POST"}], result |
| 105 | + |
| 106 | + |
| 107 | +def filter_requests(requests, path_filters, include_get, should_sort): |
| 108 | + """Filter requests based on method and path filters.""" |
| 109 | + positive_filters = [] |
| 110 | + negative_filters = [] |
| 111 | + |
| 112 | + for f in path_filters: |
| 113 | + if f.startswith(ADD_PREFIX): |
| 114 | + positive_filters.append(f.removeprefix(ADD_PREFIX)) |
| 115 | + elif f.startswith(NEGATE_PREFIX): |
| 116 | + negative_filters.append(f.removeprefix(NEGATE_PREFIX)) |
| 117 | + else: |
| 118 | + sys.exit(f"Unrecognized filter: {f!r}") |
| 119 | + |
| 120 | + filtered_requests = [] |
| 121 | + for req in requests: |
| 122 | + # Skip GET requests unless include_get is True |
| 123 | + if req.get("method") == "GET" and not include_get: |
| 124 | + continue |
| 125 | + |
| 126 | + # Apply path filters |
| 127 | + path = req.get("path", "") |
| 128 | + should_include = True |
| 129 | + |
| 130 | + # Check positive filters - if any exist, at least one must match (OR logic) |
| 131 | + if positive_filters: |
| 132 | + has_match = any(f in path for f in positive_filters) |
| 133 | + if not has_match: |
| 134 | + should_include = False |
| 135 | + |
| 136 | + # Check negative filters - if any match, exclude the request (AND logic with positive) |
| 137 | + if should_include and negative_filters: |
| 138 | + has_negative_match = any(f in path for f in negative_filters) |
| 139 | + if has_negative_match: |
| 140 | + should_include = False |
| 141 | + |
| 142 | + if should_include: |
| 143 | + filtered_requests.append(req) |
| 144 | + |
| 145 | + if should_sort: |
| 146 | + filtered_requests.sort(key=str) |
| 147 | + |
| 148 | + return filtered_requests |
| 149 | + |
| 150 | + |
| 151 | +def main(): |
| 152 | + parser = argparse.ArgumentParser() |
| 153 | + parser.add_argument("path_filters", nargs="*", help=f"Path substring filters") |
| 154 | + parser.add_argument("-v", "--verbose", action="store_true", help="Enable diagnostic messages") |
| 155 | + parser.add_argument("--get", action="store_true", help="Include GET requests (excluded by default)") |
| 156 | + parser.add_argument("--keep", action="store_true", help="Keep out.requests.json file after processing") |
| 157 | + parser.add_argument("--sort", action="store_true", help="Sort requests before output") |
| 158 | + parser.add_argument("--fname", default="out.requests.txt") |
| 159 | + args = parser.parse_args() |
| 160 | + |
| 161 | + test_tmp_dir = os.environ.get("TEST_TMP_DIR") |
| 162 | + if test_tmp_dir: |
| 163 | + requests_file = Path(test_tmp_dir) / args.fname |
| 164 | + else: |
| 165 | + requests_file = Path(args.fname) |
| 166 | + |
| 167 | + if not requests_file.exists(): |
| 168 | + sys.exit(f"File {requests_file} not found") |
| 169 | + |
| 170 | + with open(requests_file) as fobj: |
| 171 | + data = fobj.read() |
| 172 | + |
| 173 | + if not data: |
| 174 | + return |
| 175 | + |
| 176 | + requests = read_json_many(data) |
| 177 | + filtered_requests = filter_requests(requests, args.path_filters, args.get, args.sort) |
| 178 | + if args.verbose: |
| 179 | + print( |
| 180 | + f"Read {len(data)} chars, {len(requests)} requests, {len(filtered_requests)} after filtering", |
| 181 | + file=sys.stderr, |
| 182 | + flush=True, |
| 183 | + ) |
| 184 | + |
| 185 | + for req in filtered_requests: |
| 186 | + print(json.dumps(req, indent=2), flush=True) |
| 187 | + |
| 188 | + if not args.keep: |
| 189 | + requests_file.unlink() |
| 190 | + |
| 191 | + |
| 192 | +if __name__ == "__main__": |
| 193 | + main() |
0 commit comments