Optional error message conforming to Python return. KXI-1467 (#100)

* Update to p.q logic for handling sys.argv

* reintroduction of old logic

* addition of initial sys argv and embedpy version check

* Addition of functionality to return Python traceback rather than q shorthand

* Memory leak fix and removal of environment variable check C side

* reformat of prr to use modifiable global rather than env variable

* Initial pass at rectifying potential buffer overflow

* update to py.c to initialize traceback import on init

* closer fit to code base style

Co-authored-by: Conor McCarthy <conormccarthy@brainpool1.mynet>

Author: cmccarthy1

.gitignore

@@ -4,3 +4,7 @@
 
 *.bak
 q
+
+k.h
+*.so
+m64/

makefile

@@ -26,6 +26,7 @@ Q     = $(QHOME)/$(QARCH)
 
 p.so: $(QARCH)/p.so
 	cp $(QARCH)/p.so .
+
 $(QARCH)/p.so: py.c py.h k.h
 	mkdir -p $(QARCH)
 	$(CC) $(CFLAGS) $(LDFLAGS) $< -o $@
@@ -42,7 +43,8 @@ k.h:
 q.lib:
 	curl -O -J -L https://github.com/KxSystems/kdb/raw/master/w64/q.lib
 
-install:
+install: p.so
+	cp p.q p.k $(QHOME)
 	install $(TGT) $(Q)
 clean:
 	rm -f p.so

py.c

@@ -2,12 +2,16 @@
 ZS zs(K x){S s=memcpy(malloc(xn+1),xG,xn);R s[xn]=0,s;}Z K1(cs){K r,y;P(xt||!xn,r1(x))DO(xn,P(10!=kK(x)[i]->t,r1(x)))r=ktn(KS,xn);DO(xn,y=kK(x)[i];kS(r)[i]=sn(y->G0,y->n))R r;}
 ZK at(K x,J i){R!xt?r1(kK(x)[i]):k(0,"@",r1(x),kj(i),0);}ZJ cn(K x){J n;x=k(0,"#:",r1(x),0);n=xt==-KJ?xj:nj;R r0(x),n;}ZK val(K x){R k(0,".:",r1(x),0);}
 ZI g1(){R PyGILState_Ensure();}ZV g0(I g){PyGILState_Release(g);}ZK ktrr(){R krr("type");}ZP ptrr(){R PyErr_BadArgument();}
-ZV*t1(){R PyEval_SaveThread();}ZV t0(V*t){PyEval_RestoreThread(t);}ZP M;ZV**N;ZP pe(K x){R PyErr_SetString(PyExc_RuntimeError,xs);}
+ZV*t1(){R PyEval_SaveThread();}ZV t0(V*t){PyEval_RestoreThread(t);}ZP M,errfmt;ZV**N;ZP pe(K x){R PyErr_SetString(PyExc_RuntimeError,xs);}
 ZP p1(P p){Py_IncRef(p);R p;}ZV p0(P p){Py_DecRef(p);}ZI pt(P p,P t){P u=PyObject_Type(p);I f;f=(u==t||PyType_IsSubtype(u,t));p0(u);R f;}
 ZP pg(K x){R(P)(kK(x)[1]);}ZV pd(K x){I g=g1();p0(pg(x));g0(g);}ZK kfp(P p){K x=knk(2,pd,p);R xt=112,x;}ZI pq(K x){R xt==112&&xn==2&&*kK(x)==(K)pd;}
 ZK kfg(P p){R PyCapsule_GetPointer(p,"k");}ZV kfd(P p){r0(kfg(p));}ZP pfk(K x){R PyCapsule_New(r1(x),"k",kfd);}
-ZK prr(S s){Z __thread C b[4096];J n=sizeof(b)-1;P t,v,d,a;*b=0;strncat(b,s,n);PyErr_Fetch(&t,&v,&d);if(t){PyErr_NormalizeException(&t,&v,&d);
- strncat(strncat(b,": ",n),PyUnicode_AsUTF8AndSize(a=PyObject_Str(v),0),n);p0(a);p0(t);p0(v);p0(d);}R krr(b);}ZK prg(S s,I g){K r=prr(s);g0(g);R r;}
+ZJ pyerr=0;ZS updbuff(S d,S s,J*n){J k=strnlen(s,*n);memcpy(d,s,k);*n=*n>k?*n-k:0;R d+k;}
+ZK prr(S s){Z __thread C b[BUFFSIZE];S p=b;J n=sizeof(b)-1;P t,v,d,a;p=updbuff(p,s,&n);PyErr_Fetch(&t,&v,&d);
+ if(t&&n){p=updbuff(p,": ",&n);PyErr_NormalizeException(&t,&v,&d);
+  if(pyerr)a=PyObject_CallFunctionObjArgs(errfmt,t,v,d,NULL),p0(v),v=a;p=updbuff(p,PyUnicode_AsUTF8AndSize(a=PyObject_Str(v),0),&n);p0(t);p0(v);p0(d);p0(a);}
+ if(!n){n=2;updbuff(p-2,"..",&n);}R *p=0,krr(b);}
+ZK prg(S s,I g){K r=prr(s);g0(g);R r;}
 
 ZK ko(P);ZK cf;//k from python
 ZK kstr(I t,P p){K r;L n;S s=t==KG?PyBytes_AsStringAndSize(p,&s,&n)<0?0:s:PyUnicode_AsUTF8AndSize(p,&n);P(!s,prr("kstr"))r=kpn(s,n);R r->t=t,r;}
@@ -43,15 +47,18 @@ Z K2(getattr){P p;I g;P(y->t!=-KS||!pq(x),ktrr())g=g1();p=PyObject_GetAttrString
 Z K3(call){P a,k,p;I g;P(!pq(x)||!pq(z)&&z->t!=XD,ktrr())g=g1();a=pq(y)?p1(pg(y)):pseq(1,y);P(!a,prg("call a",g))k=pq(z)?p1(pg(z)):pdict(z);P(!k,(p0(a),prg("call k",g)))
  p=PyObject_Call(pg(x),a,k);P(!p,(p0(a),p0(k),prg("call",g)))p0(a);p0(k);g0(g);R kfp(p);}
 Z K1(setconv){P(xt<100||xt>111,ktrr())r0(cf);cf=r1(x);R 0;}Z K1(getconv){R r1(cf);}Z K1(isp){R kb(pq(x));}
+Z K1(setpyerr){P(xt!=-KJ,ktrr())P(!errfmt,krr("traceback.format_exception"))pyerr=xj;R 0;}Z K1(getpyerr){R kj(pyerr);}
+
 
 ZV*t;EXP K2(init){
  ZI i=0;I f,g;S l,h;K n,v;P a,b;P(i,0)l=zs(x),h=zs(y);f=pyl(l);free(l);
  P(!f,krr("libpython"))Py_SetPythonHome(Py_DecodeLocale(h,0));free(h);
  Py_InitializeEx(0);if(PyEval_ThreadsInitialized()&&!PyGILState_Check())t0(PyGILState_GetThisThreadState());PyEval_InitThreads();
  M=PyModule_GetDict(PyImport_AddModule("__main__"));cf=k(0,"::",0);n=ktn(KS,0);v=ktn(0,0);
  if(a=PyImport_ImportModule("numpy.core.multiarray")){N=PyCapsule_GetPointer(b=PyObject_GetAttrString(a,"_ARRAY_API"),0);if(!N||!pyn(N))N=0;p0(b);p0(a);}PyErr_Clear();
+ if(a=PyImport_ImportModule("traceback")){errfmt=PyObject_GetAttrString(a,"format_exception");p0(a);}PyErr_Clear();
 #define F(f,i) js(&n,ss(#f));jk(&v,dl(f,i));
- F(eval,1)F(e,1)F(py2q,1)F(q2py,1)F(get,1)F(set,2)F(import,1)F(getattr,2)F(call,3)F(isp,1)F(setconv,1)F(getconv,1)js(&n,ss("numpy"));jk(&v,kb(!!N));
- t=t1();i=1;R xD(n,v);}
+ F(setpyerr,1)F(getpyerr,1)F(eval,1)F(e,1)F(py2q,1)F(q2py,1)F(get,1)F(set,2)F(import,1)F(getattr,2)F(call,3)F(isp,1)F(setconv,1)F(getconv,1)js(&n,ss("numpy"));
+ jk(&v,kb(!!N));t=t1();i=1;R xD(n,v);}
 // a kludge for python modules which try to resolve main e.g. scipy.optimize
 int main(){}

py.h

@@ -84,6 +84,8 @@ typedef struct _p _p,*P;struct _p{L r;P t;L n;union{P*p;P v[1];};};typedef struc
  X(P,PyDict_SetItem,(P,P,P))\
  X(P,PyDict_Keys,(P))\
  X(P,PyDict_Values,(P))\
+ X(P,PyObject_CallFunctionObjArgs,(P,...))\
+ X(P,PyImport_Import,(P))\
 
 //https://docs.scipy.org/doc/numpy/reference/c-api.html https://github.com/numpy/numpy/blob/master/numpy/core/code_generators/numpy_api.py
 #undef PyCFunction_New
@@ -115,6 +117,8 @@ ZI pyl(S l){
 #undef X
  R 1;}
 
+#define BUFFSIZE 4096
+
 ZI pyn(V**v){
 #define X(r,n,a,i) U(n=(T##n(*)a)v[i])
  NF