cpk: (!!) Disable compression for now. Implementation fails with our tests *somehow*.

CriCodecsEx: Properly use Py_ssize_t for buffer sizes
docs: Update docs

Author: mos9527

.gitignore

@@ -7,4 +7,5 @@ temp_*
 *.aprof
 *.pdb
 dist/
-_*
+_*
+*.so

.vscode/launch.json

@@ -5,13 +5,13 @@
     "version": "0.2.0",
     "configurations": [
         {
-            "name": "(gdb) Launch USMDecode",
+            "name": "(gdb) Launch CPK",
             "type": "cppdbg",
             "request": "launch",
             "program": "/opt/miniconda3/bin/python",
             "args": [
                 "-m",
-                "Tests.test_USMDecode"
+                "Tests.test_CPK"
             ],
             "stopAtEntry": false,
             "cwd": "${workspaceFolder}",

.vscode/settings.json

@@ -0,0 +1,13 @@
+{
+    "files.associations": {
+        "format": "cpp",
+        "__locale": "cpp",
+        "ios": "cpp",
+        "array": "cpp",
+        "string": "cpp",
+        "string_view": "cpp",
+        "ranges": "cpp",
+        "span": "cpp",
+        "vector": "cpp"
+    }
+}

CriCodecsEx/adx.cpp

@@ -542,7 +542,7 @@ static PyObject* AdxEncode(PyObject* self, PyObject* args){
         PyAdxSetError(AdxErrorCode);
         return NULL;
     }
-    unsigned int len = adx.size;
+    Py_ssize_t len = adx.size;
     return Py_BuildValue("y#", adxdata, len);
 }
 
@@ -557,6 +557,6 @@ static PyObject* AdxDecode(PyObject* self, PyObject* args){
         return NULL;
     }
     unsigned char *out = wav.WAVEBuffer;
-    unsigned int len = wav.wav.size+8;
+    Py_ssize_t len = wav.wav.size+8;
     return Py_BuildValue("y#", out, len);
 }

CriCodecsEx/crilayla.cpp

@@ -108,40 +108,41 @@ unsigned char* layla_decomp(unsigned char* data, crilayla_header header){
 	return dst;
 }
 
-unsigned int layla_comp(unsigned char* dest, unsigned int* destLen, unsigned char* src, unsigned int srcLen){
-    unsigned int n = srcLen - 1, m = *destLen - 0x1, T = 0, d = 0, p, q, i, j, k;
-    unsigned char* odest = dest;
+unsigned int layla_comp(unsigned char *dest, int *destLen, unsigned char *src, int srcLen)
+{
+    int n = srcLen - 1, m = *destLen - 0x1, T = 0, d = 0, p, q, i, j, k;
+    unsigned char *odest = dest;
     for (; n >= 0x100;)
     {
         j = n + 3 + 0x2000;
-        if (j > srcLen) j = srcLen;
-        for (i = n + 3, p = 0; i < j; i++)
+        if (j>srcLen) j = srcLen;
+        for (i = n + 3, p = 0; i<j; i++)
         {
             for (k = 0; k <= n - 0x100; k++)
             {
                 if (*(src + n - k) != *(src + i - k)) break;
             }
-            if (k > p)
+            if (k>p)
             {
                 q = i - n - 3; p = k;
             }
         }
-        if (p < 3)
+        if (p<3)
         {
             d = (d << 9) | (*(src + n--)); T += 9;
         }
         else
         {
             d = (((d << 1) | 1) << 13) | q; T += 14; n -= p;
-            if (p < 6)
+            if (p<6)
             {
                 d = (d << 2) | (p - 3); T += 2;
             }
-            else if (p < 13)
+            else if (p<13)
             {
                 d = (((d << 2) | 3) << 3) | (p - 6); T += 5;
             }
-            else if (p < 44)
+            else if (p<44)
             {
                 d = (((d << 5) | 0x1f) << 5) | (p - 13); T += 10;
             }
@@ -152,17 +153,17 @@ unsigned int layla_comp(unsigned char* dest, unsigned int* destLen, unsigned cha
                 {
                     for (; T >= 8;)
                     {
-                        *(dest + m--) = (d >> (T - 8)) & 0xff; T -= 8; d = d & ((1 << T) - 1);
+                        *(dest + m--) = (d >> (T - 8)) & 0xff; T -= 8; d = d&((1 << T) - 1);
                     }
-                    if (p < 255) break;
+                    if (p<255) break;
                     d = (d << 8) | 0xff; T += 8; p = p - 0xff;
                 }
                 d = (d << 8) | p; T += 8;
             }
         }
         for (; T >= 8;)
         {
-            *(dest + m--) = (d >> (T - 8)) & 0xff; T -= 8; d = d & ((1 << T) - 1);
+            *(dest + m--) = (d >> (T - 8)) & 0xff; T -= 8; d = d&((1 << T) - 1);
         }
     }
     if (T != 0)
@@ -176,19 +177,19 @@ unsigned int layla_comp(unsigned char* dest, unsigned int* destLen, unsigned cha
         *(dest + m--) = 0;
     }
     *destLen = *destLen - m; dest += m;
-    unsigned int l[] = { 0x4c495243,0x414c5941,srcLen - 0x100,*destLen };
-    for (j = 0; j < 4; j++)
+    int l[] = { 0x4c495243,0x414c5941,srcLen - 0x100,*destLen };
+    for (j = 0; j<4; j++)
     {
-        for (i = 0; i < 4; i++)
+        for (i = 0; i<4; i++)
         {
             *(odest + i + j * 4) = l[j] & 0xff; l[j] >>= 8;
         }
     }
-    for (j = 0, odest += 0x10; j < *destLen; j++)
+    for (j = 0, odest += 0x10; j<*destLen; j++)
     {
         *(odest++) = *(dest + j);
     }
-    for (j = 0; j < 0x100; j++)
+    for (j = 0; j<0x100; j++)
     {
         *(odest++) = *(src + j);
     }
@@ -218,18 +219,19 @@ unsigned char* CriLaylaDecompress(unsigned char* d){
 
 PyObject* CriLaylaCompress(PyObject* self, PyObject* args){
 	unsigned char *data;
-	unsigned int data_size;
+	Py_ssize_t data_size;
     if(!PyArg_ParseTuple(args, "y#", &data, &data_size)){
         return NULL;
     }
     unsigned char *buf = new unsigned char[data_size];
     memset(buf, 0, data_size);
 
+    int compressed_size = data_size;
     Py_BEGIN_ALLOW_THREADS
-    layla_comp(buf, &data_size, data, data_size);
+    layla_comp(buf, &compressed_size, data, compressed_size);
     Py_END_ALLOW_THREADS
 
-	PyObject* bufObj = Py_BuildValue("y#", buf, data_size);
+	PyObject* bufObj = Py_BuildValue("y#", buf, compressed_size);
     delete[] buf;
     return bufObj;
 }

CriCodecsEx/hca.cpp

@@ -3296,7 +3296,7 @@ static PyObject* HcaCrypt(PyObject* self, PyObject* args){
     }
 
     unsigned char *buffer = (unsigned char *)view.buf;
-    unsigned int length = view.len;
+    Py_ssize_t length = view.len;
     PyBuffer_Release(&view);
 
     clHCA* hca = (clHCA*)malloc(sizeof(clHCA));
@@ -3345,7 +3345,7 @@ static PyObject* HcaDecode(PyObject* self, PyObject* args){
     data *wavData = &wav.wav.chunks.WAVEdata;
 
     unsigned char *data;
-    unsigned int data_size;
+    Py_ssize_t data_size;
     unsigned int header_size;
     unsigned long long keycode;
     unsigned short subkey;

PyCriCodecsEx/cpk.py

@@ -253,9 +253,9 @@ def add_file(self, src : str, dst : str = None, compress=False):
             compress (bool, optional): Whether to compress the file. Defaults to False.
         
         NOTE: 
-            - In ITOC-related mode, the insertion order determines the final integer ID of the files.
-            - Compression can be VERY slow with high entropy files (e.g. encoded media). Use at discretion.
+            - In ITOC-related mode, the insertion order determines the final integer ID of the files.            
         """
+        assert not compress, "FIXME: Compression not working."
         if not dst and self.mode != 0:
             raise ValueError("Destination filename must be specified in non-ITOC mode.")
 
@@ -281,7 +281,8 @@ def _populate_files(self, threads : int = 1):
             futures = []
             for (src, _, _), (dst, compress) in zip(self.in_files,self.os_files):
                 if compress:
-                    futures.append(exec.submit(_crilayla_compress_to_file, src, dst))
+                    _crilayla_compress_to_file(src, dst)
+                    # futures.append(exec.submit(_crilayla_compress_to_file, src, dst))
             for i, fut in enumerate(as_completed(futures)):
                 fut.result()
                 self.progress_cb("Compress %s" % os.path.basename(src), i + 1, len(futures))

docs/source/examples.rst

@@ -1,6 +1,8 @@
 Load, Extract and modify ACB files
 ---------------------------
+
 .. code-block:: python
+
     from PyCriCodecsEx.acb import ACB, ACBBuilder, HCACodec, ADXCodec
     from PyCriCodecsEx.awb import AWBBuilder
     # Load the '.acb' file.
@@ -34,7 +36,9 @@ Load, Extract and modify ACB files
 Build USM files
 **NOTE:** FFmpeg must be installed and available in PATH for this to work.
 ---------------------------
+
 .. code-block:: python
+
     from PyCriCodecsEx.usm import USM, USMBuilder, ADXCodec, HCACodec
     builder = USMBuilder()
     builder.add_video('some_video.m1v')
@@ -50,7 +54,9 @@ Build USM files
 Load, extract then remux USM files into MP4
 **NOTE:** FFmpeg must be installed and available in PATH for this to work.
 ---------------------------
+
 .. code-block:: python
+
     usm = USM('build.usm')
     audio = usm.get_audios()
     video = usm.get_video()
@@ -85,7 +91,9 @@ Load, extract then remux USM files into MP4
 
 Extract CPK files
 ---------------------------
+
 .. code-block:: python
+
     import os
     from PyCriCodecsEx.cpk import CPK
     
@@ -105,7 +113,9 @@ Extract CPK files
 
 Pack files into a CPK
 ---------------------------
+
 .. code-block:: python
+    
     import os
     from PyCriCodecsEx.cpk import CPKBuilder
 

setup.py

@@ -20,11 +20,8 @@ def build_extensions(self):
                 compile_args += ['/O2']
         else:
             compile_args = ['-std=c++14']
-            if ENV_DEBUG:
-                # ASAN on Linux
-                # This only works with GCC - you also need to specify 
-                # LD_PRELOAD=$(gcc -print-file-name=libasan.so)
-                compile_args += ['-O0', '-g', '-fsanitize=address']
+            if ENV_DEBUG:                
+                compile_args += ['-O0', '-g']
             else:
                 compile_args += ['-O2']
         for ext in self.extensions: