Merge pull request #274 from Distributive-Network/wes/DCP-4054/make-dcp-client-grok-pythonmonkey

dcp-client compatibility

Author: philippedistributive

CMakeLists.txt

@@ -44,7 +44,7 @@ if(CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME)
   include(FetchContent)
 
   SET(COMPILE_FLAGS "-ggdb -Ofast -fno-rtti") # optimize but also emit debug symbols
-  SET( CMAKE_CXX_FLAGS  "${CMAKE_CXX_FLAGS} ${COMPILE_FLAGS}" )
+  SET(CMAKE_CXX_FLAGS  "${CMAKE_CXX_FLAGS} ${COMPILE_FLAGS} $ENV{EXTRA_CMAKE_CXX_FLAGS}")
 
   set(CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/cmake/modules)
   if(APPLE)

Makefile

@@ -0,0 +1,46 @@
+# @file		Makefile
+#               Not part of the PythonMonkey build - just workflow helper for Wes.
+# @author       Wes Garland, [email protected]
+# @date         March 2024
+#
+
+BUILD  = debug
+PYTHON = python3
+RUN    = poetry run
+
+PYTHON_BUILD_ENV = VERBOSE=1 EXTRA_CMAKE_CXX_FLAGS="$(EXTRA_CMAKE_CXX_FLAGS)"
+OS_NAME := $(shell uname -s)
+
+ifeq ($(OS_NAME),Linux)
+CPU_COUNT=$(shell cat /proc/cpuinfo  | grep -c processor)
+MAX_JOBS=10
+CPUS := $(shell test $(CPU_COUNT) -lt $(MAX_JOBS) && echo $(CPU_COUNT) || echo $(MAX_JOBS))
+PYTHON_BUILD_ENV += CPUS=$(CPUS)
+endif
+
+EXTRA_CMAKE_CXX_FLAGS = -Wno-invalid-offsetof $(JOBS)
+
+ifeq ($(BUILD),debug)
+EXTRA_CMAKE_CXX_FLAGS += -O0
+endif
+
+.PHONY: build test all clean debug
+build:
+	$(PYTHON_BUILD_ENV) $(PYTHON) ./build.py
+
+test:
+	$(RUN) ./peter-jr tests
+	$(RUN) pytest tests/python
+
+all:	build test
+
+clean:
+	rm -rf build/src/CMakeFiles/pythonmonkey.dir
+	rm -f build/src/pythonmonkey.so
+	rm -f python/pythonmonkey.so
+
+debug:
+	@echo EXTRA_CMAKE_CXX_FLAGS=$(EXTRA_CMAKE_CXX_FLAGS)
+	@echo JOBS=$(JOBS)
+	@echo CPU_COUNT=$(CPU_COUNT)
+	@echo OS_NAME=$(OS_NAME)

build.py

@@ -14,7 +14,7 @@
 BUILD_DIR = os.path.join(TOP_DIR, "build")
 
 # Get number of CPU cores
-CPUS = os.cpu_count() or 1
+CPUS = os.getenv('CPUS') or os.cpu_count() or 1
 
 def execute(cmd: str, cwd: Optional[str] = None):
     popen = subprocess.Popen(cmd, stdout = subprocess.PIPE, stderr = subprocess.STDOUT,

python/pythonmonkey/builtin_modules/XMLHttpRequest-internal.py

@@ -42,36 +42,44 @@ async def request(
     onNetworkError: Callable[[aiohttp.ClientError], None],
     /
 ):
+    debug = pm.bootstrap.require("debug");
+
     class BytesPayloadWithProgress(aiohttp.BytesPayload):
         _chunkMaxLength = 2**16 # aiohttp default
 
         async def write(self, writer) -> None:
+            debug('xhr:io')('begin chunked write')
             buf = io.BytesIO(self._value)
             chunk = buf.read(self._chunkMaxLength)
             while chunk:
+                debug('xhr:io')('  writing', len(chunk), 'bytes')
                 await writer.write(chunk)
                 processRequestBodyChunkLength(len(chunk))
                 chunk = buf.read(self._chunkMaxLength)
             processRequestEndOfBody()
+            debug('xhr:io')('finish chunked write')
 
     if isinstance(body, str):
         body = bytes(body, "utf-8")
 
     # set default headers
     headers.setdefault("user-agent", f"Python/{platform.python_version()} PythonMonkey/{pm.__version__}")
+    debug('xhr:headers')('after set default\n', headers)
 
     if timeoutMs > 0:
         timeoutOptions = aiohttp.ClientTimeout(total=timeoutMs/1000) # convert to seconds
     else:
         timeoutOptions = aiohttp.ClientTimeout() # default timeout
 
     try:
+        debug('xhr:aiohttp')('creating request for', url)
         async with aiohttp.request(method=method,
                                 url=yarl.URL(url, encoded=True),
                                 headers=headers,
                                 data=BytesPayloadWithProgress(body) if body else None,
                                 timeout=timeoutOptions,
         ) as res:
+            debug('xhr:aiohttp')('got', res.content_type, 'result')
             def getResponseHeader(name: str):
                 return res.headers.get(name)
             def getAllResponseHeaders():
@@ -81,6 +89,7 @@ def getAllResponseHeaders():
                 headers.sort()
                 return "\r\n".join(headers)
             def abort():
+                debug('xhr:io')('abort')
                 res.close()
 
             # readyState HEADERS_RECEIVED
@@ -92,15 +101,12 @@ def abort():
                 'getResponseHeader': getResponseHeader,
                 'getAllResponseHeaders': getAllResponseHeaders,
                 'abort': abort,
-                
                 'contentLength': res.content_length or 0,
             }
             processResponse(responseData)
 
-            # readyState LOADING
             async for data in res.content.iter_any():
                 processBodyChunk(bytearray(data)) # PythonMonkey only accepts the mutable bytearray type
-            
             # readyState DONE
             processEndOfBody()
     except asyncio.TimeoutError as e:

python/pythonmonkey/builtin_modules/XMLHttpRequest.js

@@ -7,11 +7,39 @@
  * 
  * @copyright Copyright (c) 2023 Distributive Corp.
  */
+'use strict';
 
 const { EventTarget, Event } = require('event-target');
 const { DOMException } = require('dom-exception');
 const { URL, URLSearchParams } = require('url');
 const { request, decodeStr } = require('XMLHttpRequest-internal');
+const debug = globalThis.python.eval('__import__("pythonmonkey").bootstrap.require')('debug');
+
+/**
+ * Truncate a string-like thing for display purposes, returning a string.
+ * @param {any}     what     The thing to truncate; must have a slice method and index property.
+ *                           Works with string, array, typedarray, etc.
+ * @param {number}  maxlen   The maximum length for truncation
+ * @param {boolean} coerce   Not false = coerce to printable character codes  
+ * @returns {string}
+ */
+function trunc(what, maxlen, coerce)
+{
+  if (coerce !== false && typeof what !== 'string')
+  {
+    what = Array.from(what).map(x => {
+      if (x > 31 && x < 127)
+        return String.fromCharCode(x);
+      else if (x < 32)
+        return String.fromCharCode(0x2400 + Number(x));
+      else if (x === 127)
+        return '\u2421';
+      else
+        return '\u2423';
+    }).join('');
+  }
+  return `${what.slice(0, maxlen)}${what.length > maxlen ? '\u2026' : ''}`;
+}
 
 // exposed
 /**
@@ -29,6 +57,7 @@ class ProgressEvent extends Event
     this.lengthComputable = eventInitDict.lengthComputable ?? false;
     this.loaded = eventInitDict.loaded ?? 0;
     this.total = eventInitDict.total ?? 0;
+    this.debugTag = 'xhr:';
   }
 }
 
@@ -112,6 +141,7 @@ class XMLHttpRequest extends XMLHttpRequestEventTarget
    */
   open(method, url, async = true, username = null, password = null)
   {
+    debug('xhr:open')('open start, method=' + method);
     // Normalize the method.
     // @ts-expect-error
     method = method.toString().toUpperCase();
@@ -125,7 +155,8 @@ class XMLHttpRequest extends XMLHttpRequestEventTarget
       parsedURL.username = username;
     if (password)
       parsedURL.password = password;
-    
+    debug('xhr:open')('url is ' + parsedURL.href);
+
     // step 11
     this.#sendFlag = false;
     this.#uploadListenerFlag = false;
@@ -144,6 +175,7 @@ class XMLHttpRequest extends XMLHttpRequestEventTarget
       this.#state = XMLHttpRequest.OPENED;
       this.dispatchEvent(new Event('readystatechange'));
     }
+    debug('xhr:open')('finished open, state is ' + this.#state);
   }
 
   /**
@@ -153,6 +185,7 @@ class XMLHttpRequest extends XMLHttpRequestEventTarget
    */
   setRequestHeader(name, value)
   {
+    debug('xhr:headers')(`set header ${name}=${value}`);
     if (this.#state !== XMLHttpRequest.OPENED)
       throw new DOMException('setRequestHeader can only be called when state is OPEN', 'InvalidStateError');
     if (this.#sendFlag)
@@ -218,6 +251,7 @@ class XMLHttpRequest extends XMLHttpRequestEventTarget
    */
   send(body = null)
   {
+    debug('xhr:send')(`sending; body length=${body?.length}`);
     if (this.#state !== XMLHttpRequest.OPENED) // step 1
       throw new DOMException('connection must be opened before send() is called', 'InvalidStateError');
     if (this.#sendFlag) // step 2
@@ -248,10 +282,9 @@ class XMLHttpRequest extends XMLHttpRequestEventTarget
 
       const originalAuthorContentType = this.#requestHeaders['content-type'];
       if (!originalAuthorContentType && extractedContentType)
-      {
         this.#requestHeaders['content-type'] = extractedContentType;
-      }
     }
+    debug('xhr:send')(`content-type=${this.#requestHeaders['content-type']}`);
 
     // step 5
     if (this.#uploadObject._hasAnyListeners())
@@ -276,6 +309,7 @@ class XMLHttpRequest extends XMLHttpRequestEventTarget
    */
   #sendAsync()
   {
+    debug('xhr:send')('sending in async mode');
     this.dispatchEvent(new ProgressEvent('loadstart', { loaded:0, total:0 })); // step 11.1
 
     let requestBodyTransmitted = 0; // step 11.2
@@ -308,6 +342,7 @@ class XMLHttpRequest extends XMLHttpRequestEventTarget
     let responseLength = 0;
     const processResponse = (response) =>
     {
+      debug('xhr:response')(`response headers ----\n${response.getAllResponseHeaders()}`);
       this.#response = response; // step 11.9.1
       this.#state = XMLHttpRequest.HEADERS_RECEIVED; // step 11.9.4
       this.dispatchEvent(new Event('readystatechange')); // step 11.9.5
@@ -318,6 +353,7 @@ class XMLHttpRequest extends XMLHttpRequestEventTarget
 
     const processBodyChunk = (/** @type {Uint8Array} */ bytes) =>
     {
+      debug('xhr:response')(`recv chunk, ${bytes.length} bytes (${trunc(bytes, 100)})`);
       this.#receivedBytes.push(bytes);
       if (this.#state === XMLHttpRequest.HEADERS_RECEIVED)
         this.#state = XMLHttpRequest.LOADING;
@@ -330,16 +366,22 @@ class XMLHttpRequest extends XMLHttpRequestEventTarget
      */
     const processEndOfBody = () =>
     {
+      debug('xhr:response')(`end of body, received ${this.#receivedLength} bytes`);
       const transmitted = this.#receivedLength; // step 3
       const length = responseLength || 0; // step 4
+
       this.dispatchEvent(new ProgressEvent('progress', { loaded:transmitted, total:length })); // step 6
       this.#state = XMLHttpRequest.DONE; // step 7
       this.#sendFlag = false; // step 8
+
       this.dispatchEvent(new Event('readystatechange')); // step 9
       for (const eventType of ['load', 'loadend']) // step 10, step 11
         this.dispatchEvent(new ProgressEvent(eventType, { loaded:transmitted, total:length }));
     };
 
+    debug('xhr:send')(`${this.#requestMethod} ${this.#requestURL.href}`);
+    debug('xhr:headers')('headers=' + Object.entries(this.#requestHeaders));
+
     // send() step 6
     request(
       this.#requestMethod,
@@ -362,8 +404,8 @@ class XMLHttpRequest extends XMLHttpRequestEventTarget
    */
   #sendSync()
   {
+    /* Synchronous XHR deprecated. /wg march 2024 */
     throw new DOMException('synchronous XHR is not supported', 'NotSupportedError');
-    // TODO: handle synchronous request
   }
 
   /**
@@ -376,7 +418,6 @@ class XMLHttpRequest extends XMLHttpRequestEventTarget
       return;
     if (this.#timedOutFlag) // step 2
       return this.#reportRequestError('timeout', new DOMException(e.toString(), 'TimeoutError'));
-    console.error(e); // similar to browsers, print out network errors even then the error will be handled by `xhr.onerror`
     if (this.#response === null /* network error */) // step 4
       return this.#reportRequestError('error', new DOMException(e.toString(), 'NetworkError'));
     else // unknown errors
@@ -652,6 +693,10 @@ class XMLHttpRequest extends XMLHttpRequestEventTarget
   }
 }
 
+/* A side-effect of loading this module is to add the XMLHttpRequest and related symbols to the global
+ * object. This makes them accessible in the "normal" way (like in a browser) even in PythonMonkey JS
+ * host environments which don't include a require() symbol.
+ */
 if (!globalThis.XMLHttpRequestEventTarget)
   globalThis.XMLHttpRequestEventTarget = XMLHttpRequestEventTarget;
 if (!globalThis.XMLHttpRequestUpload)