Add examples for episode 579 & 580

Author: bozhodimitrov

sample_code/ep579/README.md

@@ -0,0 +1,45 @@
+# [oops I'm the pyuwsgi maintainer now (intermediate)](https://youtu.be/WILYaDNez4g)
+
+today I take you down a rabbit hole of deadlocks and worker reloads and python 3.12 sadness.  unfortunately despite the "conclusion" in this video this doesn't actually fix the problem -- more on that in the future!
+
+## Interactive examples
+
+### Python
+
+```python
+import pyuwsgi
+```
+
+### Bash
+
+Session 1:
+
+```bash
+virtualenv venv -ppython3.11
+. venv/bin/activate
+pip install pyuwsgi
+
+ls venv/lib/python3.11/site-packages/
+python
+
+chmod +x t.sh
+bash t.sh
+kill -9 %1
+
+pip freeze | grep pyuwsgi
+deactivate
+
+virtualenv venv -ppython3.12
+. venv/bin/activate
+pip install pyuwsgi
+
+bash t.sh
+```
+
+Session 2:
+
+```bash
+yes | head -100 | xargs -P4 --replace curl localhost:9001/health-check/; echo
+yes | head -1000 | xargs -P4 --replace curl localhost:9001/health-check/; echo
+yes | head -1000 | xargs -P40 --replace curl localhost:9001/health-check/; echo
+```

sample_code/ep579/rev01/t.sh

@@ -0,0 +1,35 @@
+UWSGI_AUTO_PROCNAME=true \
+UWSGI_BINARY_PATH=$PWD/venv/bin/python3 \
+UWSGI_BUFFER_SIZE=32768 \
+UWSGI_DIE_ON_TERM=true \
+UWSGI_DISABLE_LOGGING=false \
+UWSGI_DISABLE_WRITE_EXCEPTION=true \
+UWSGI_ENABLE_THREADS=true \
+UWSGI_HARAKIRI=600 \
+UWSGI_HONOUR_STDIN=true \
+UWSGI_HTTP_CHUNKED_INPUT=true \
+UWSGI_HTTP_KEEPALIVE=true \
+UWSGI_HTTP_SOCKET=127.0.0.1:9001 \
+UWSGI_HTTP_TIMEOUT=600 \
+UWSGI_IGNORE_SIGPIPE=true \
+UWSGI_IGNORE_WRITE_ERRORS=true \
+UWSGI_LAZY_APPS=true \
+UWSGI_LIMIT_POST=1073741824 \
+UWSGI_LOG_X_FORWARDED_FOR=false \
+UWSGI_MASTER=true \
+UWSGI_MAX_REQUESTS=20 \
+UWSGI_MEMORY_REPORT=true \
+UWSGI_MODULE=wsgi:application \
+UWSGI_NEED_APP=true \
+UWSGI_POST_BUFFERING=65536 \
+UWSGI_PROCNAME_PREFIX_SPACED=[Sentry] \
+UWSGI_PROTOCOL=http \
+UWSGI_RELOAD_ON_RSS=600 \
+UWSGI_SINGLE_INTERPRETER=true \
+UWSGI_THUNDER_LOCK=false \
+UWSGI_VACUUM=true \
+UWSGI_VIRTUALENV=$PWD/venv \
+UWSGI_WORKERS=2 \
+UWSGI_THREADS=2 \
+UWSGI_WSGI_FILE=wsgi.py \
+    uwsgi

sample_code/ep579/rev01/wsgi.py

@@ -0,0 +1,11 @@
+def application(environ, start_response):
+    status = '200 OK'
+    output = b'!'
+
+    response_headers = [
+        ('Content-Type', 'text/plain'),
+        ('Content-Length', str(len(output))),
+    ]
+    start_response(status, response_headers)
+
+    return [output]

sample_code/ep579/rev02/t.sh

@@ -0,0 +1,34 @@
+UWSGI_AUTO_PROCNAME=true \
+UWSGI_BINARY_PATH=$PWD/venv/bin/python3 \
+UWSGI_BUFFER_SIZE=32768 \
+UWSGI_DIE_ON_TERM=true \
+UWSGI_DISABLE_LOGGING=false \
+UWSGI_DISABLE_WRITE_EXCEPTION=true \
+UWSGI_ENABLE_THREADS=true \
+UWSGI_HARAKIRI=600 \
+UWSGI_HONOUR_STDIN=true \
+UWSGI_HTTP_CHUNKED_INPUT=true \
+UWSGI_HTTP_KEEPALIVE=true \
+UWSGI_HTTP_SOCKET=127.0.0.1:9001 \
+UWSGI_HTTP_TIMEOUT=600 \
+UWSGI_IGNORE_SIGPIPE=true \
+UWSGI_IGNORE_WRITE_ERRORS=true \
+UWSGI_LAZY_APPS=true \
+UWSGI_LIMIT_POST=1073741824 \
+UWSGI_LOG_X_FORWARDED_FOR=false \
+UWSGI_MASTER=true \
+UWSGI_MAX_REQUESTS=20 \
+UWSGI_MEMORY_REPORT=true \
+UWSGI_MODULE=wsgi:application \
+UWSGI_NEED_APP=true \
+UWSGI_POST_BUFFERING=65536 \
+UWSGI_PROCNAME_PREFIX_SPACED=[Sentry] \
+UWSGI_PROTOCOL=http \
+UWSGI_RELOAD_ON_RSS=600 \
+UWSGI_SINGLE_INTERPRETER=true \
+UWSGI_THUNDER_LOCK=false \
+UWSGI_VACUUM=true \
+UWSGI_VIRTUALENV=$PWD/venv \
+UWSGI_WORKERS=2 \
+UWSGI_WSGI_FILE=wsgi.py \
+    uwsgi

sample_code/ep580/README.md

@@ -0,0 +1,99 @@
+# [debugging and fixing pyuwsgi in python 3.12 (advanced)](https://youtu.be/Y4n2xCIF2Jg)
+
+welcome to a whirlwind tour of bisecting, GIL, strace, gdb, forking, threadstates, undefined behaviour, and deadlocking.  I go through the process debugging the most difficult bug I've encountered yet and ultimately how I fixed the problem and understood what changed and why it resulted in this issue!
+
+## Interactive examples
+
+Session 1:
+
+### Bash
+
+```bash
+virtualenv venv -ppython3.12
+. venv/bin/activate
+pip install pyuwsgi==2.0.26
+
+chmod +x t.sh
+bash t.sh
+pkill -9 -f uwsgi
+fg
+
+cd cpython
+git bisect start
+git bisect good v3.12.0a2
+git bisect bad v3.12.0rc1
+git bisect run ../venv-bisect/bin/python3 ../bisect-uwsgi.py
+
+git clone git@github.com:lincolnloop/pyuwsgi-wheels
+cd pyuwsgi-wheels/uwsgi/
+
+git grep IsInit
+nano plugins/python/python_plugin.c
+nano plugins/pyuwsgi/pyuwsgi.c
+
+git grep PyEval_RestoreThread
+git grep -n PyEval_RestoreThread
+
+git grep uwsgi_fork
+nano core/master_utils.c
+
+./venv.uwsgi/bin/pip uninstall uwsgi
+./venv.uwsgi/bin/pip install ./uwsgi
+
+chmod +x t.sh
+bash t.sh
+
+git -C ../../uwsgi diff -- core/
+git -C ../../uwsgi diff -- core/ | git apply
+
+./venv/bin/pip uninstall pyuwsgi
+./venv/bin/pip install ./pyuwsgi-wheels/uwsgi
+
+nano pyuwsgi-wheels/uwsgi/plugins/pyuwsgi/pyuwsgi.c
+```
+
+Session 2:
+
+```bash
+yes | head -60 | xargs --replace curl localhost:9001/health-check/; echo
+```
+
+Session 3:
+
+```bash
+which strace
+strace --help
+
+ps -ef | grep uwsgi
+sudo strace -p <pid>
+sudo gdb -p <pid>
+
+# where
+# bt
+# q
+
+ps -ef | grep uwsgi
+sudo gdb -p <pid>
+
+# b plugins/python/gil.c:12
+# c
+# p _Py_tss_tstate
+# c
+# q
+
+sudo gdb -p <pid>
+
+# b plugins/python/gil.c:12
+# c
+# p_Py_tss_tstate->interp->ceval->gil->locked
+# p _Py_tss_tstate
+# q
+
+sudo gdb -p <pid>
+
+# b plugins/python/gil.c:12
+# c
+# p _PyRuntime.ceval.gil->locked
+# p _Py_tss_tstate
+# q
+```

sample_code/ep580/rev01/bisect-uwsgi.py

@@ -0,0 +1,136 @@
+from __future__ import annotations
+import contextlib
+import os.path
+# import shlex
+import subprocess
+import urllib.request
+import sys
+import time
+from typing import Generator
+# from typing import IO
+import psutil
+
+
+rev = subprocess.check_output(('git', 'rev-parse', 'HEAD'), text=True).strip()
+
+prefix = os.path.abspath('../prefix')
+venv = os.path.abspath('../venv')
+logs = os.path.abspath('../logs')
+uwsgi_src = os.path.abspath('../pyuwsgi-wheels/uwsgi')
+
+
+def tee() -> None:
+    ...
+
+
+def _cmd_q() -> None:
+    ...
+
+
+def _build_cpython() -> None:
+    print('!!! building cpython')
+    _cmd_q('git', 'clean', '-fxfd')
+    _cmd_q('./configure', '--prefix', prefix)
+    _cmd_q('make', '-j5')
+    _cmd_q('make', 'install')
+
+
+def _make_venv() -> None:
+    print('!!! making venv')
+    _cmd_q('rm', '-rf', venv)
+    _cmd_q(os.path.join(prefix, 'bin', 'python3'), '-mvenv', venv)
+
+
+def _install_uwsgi() -> None:
+    print('!!! installing uwsgi')
+    _cmd_q(os.path.join(venv, 'bin', 'pip'), 'install', uwsgi_src)
+
+
+def _req() -> None:
+    urllib.request.urlopen('http://127.0.0.1:9001/', timeout=1).read()
+
+
+def _wait_for_ready() -> None:
+    for _ in range(10):
+        try:
+            _req()
+        except OSError:
+            print('...waiting for start', file=sys.stderr)
+            time.sleep(.25)
+        else:
+            return
+
+
+def _child_pids(pid: int) -> tuple[int, ...]:
+    return tuple(child.pid for child in psutil.Process(pid).children())
+
+
+@contextlib.contextmanager
+def uwsgi_proc() -> Generator[int]:
+    uwsgi = os.path.join(venv, 'bin', 'uwsgi')
+    _cmd_q(uwsgi, '--help')
+    env = {
+        **os.environ,
+        'UWSGI_MASTER': 'true',
+        'UWSGI_BINARY_PATH': os.path.join(venv, 'bin', 'python3'),
+        'UWSGI_VIRTUALENV': venv,
+        'UWSGI_WORKERS': '2',
+        'UWSGI_ENABLE_THREADS': 'true',
+        'UWSGI_MAX_REQUESTS': '2',
+        'UWSGI_SINGLE_INTERPRETER': 'true',
+        'UWSGI_HTTP_SOCKET': 'localhost:9001',
+        'UWSGI_WSGI_FILE': 'wsgi.py',
+        'UWSGI_DISABLE_LOGGING': '1',
+    }
+    with open(os.path.join(logs, rev), 'a+') as logfile:
+        tee('+ uwsgi', logfile, sys.stderr)
+        proc = subprocess.Popen(
+            (uwsgi, ),
+            env=env,
+            cwd='..',
+            stdout=logfile,
+            stderr=logfile,
+        )
+        _wait_for_ready()
+        try:
+            yield proc.pid
+        finally:
+            to_kill = [str(proc.pid)]
+            for child_pid in _child_pids(proc.pid):
+                to_kill.append(str(child_pid))
+            subprocess.call(('kill', '-9', *to_kill))
+
+
+def _run_test() -> int:
+    with uwsgi_proc() as pid:
+        children = set(_child_pids(pid))
+        print(f'started with pid: {pid} {children}!', file=sys.stderr)
+        while True:
+            time.sleep(.25)
+            try:
+                _req()
+            except OSError:
+                cand_children = set(_child_pids(pid))
+                assert not children & cand_children, (children, cand_children)
+                print('\ngot deadlock!!')
+                return 1
+            else:
+                cand_children = set(_child_pids(pid))
+                if cand_children & children:
+                    print('z', flush=True, end='')
+                    continue
+                else:
+                    print('\nsuccessful request after all children recycled!')
+                    return 0
+
+
+def main() -> int:
+    assert os.path.exists('Python'), 'not in cpython'
+    _build_cpython()
+    _make_venv()
+    _install_uwsgi()
+    return _run_test()
+
+
+if __name__ == '__main__':
+    raise SystemExit(main())

sample_code/ep580/rev01/t.sh

@@ -0,0 +1,10 @@
+UWSGI_MASTER=true \
+UWSGI_BINARY_PATH=$PWD/venv/bin/python3 \
+UWSGI_VIRTUALENV=$PWD/venv \
+UWSGI_WORKERS=2 \
+UWSGI_ENABLE_THREADS=true \
+UWSGI_MAX_REQUESTS=5 \
+UWSGI_HTTP_SOCKET=localhost:9001 \
+UWSGI_WSGI_FILE=wsgi.py \
+UWSGI_DISABLE_LOGGING=1 \
+    venv/bin/uwsgi

sample_code/ep580/rev01/wsgi.py

@@ -0,0 +1,11 @@
+def application(environ, start_response):
+    print('.', end='', flush=True)
+
+    status = '200 OK'
+    output = b'!'
+
+    response_headers = [('Content-type', 'text/plain'),
+                        ('Content-Length', str(len(output)))]
+    start_response(status, response_headers)
+
+    return [output]