Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 2 additions & 3 deletions .eslintrc.js
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,7 @@ module.exports = {
env: {
browser: true,
es6: true,
commonjs: true,
jest: true
commonjs: true
},
root: true,
extends: [
Expand Down Expand Up @@ -78,4 +77,4 @@ module.exports = {
}
]
}
};
};
8 changes: 1 addition & 7 deletions .github/workflows/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ jobs:
- name: Set up Node.js
uses: actions/setup-node@v2
with:
node-version: '16'
node-version: '16'

- name: Install dependencies
run: yarn install
Expand All @@ -51,9 +51,3 @@ jobs:
micromamba activate untarjs

yarn run eslint

- name: Run test
run: |
micromamba activate untarjs

yarn run test
4 changes: 3 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,9 @@ This package has 2 methods:

The example of using:
```sh
import untarjs from "@emscripten-forge/untarjs";
import { initUntarJS } from "@emscripten-forge/untarjs";

const untarjs = await initUntarJS();
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@AnastasiaSliusar note the change in the API here, this allows loading the WASM once.


const condaPackageUrl = 'https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2';
untarjs.extract(condaPackageUrl).then((files)=>{
Expand Down
7 changes: 0 additions & 7 deletions jest.config.js

This file was deleted.

7 changes: 1 addition & 6 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,7 @@
"prettier": "prettier --list-different --write \"src/**/*.ts\"",
"prettier:check": "prettier --check \"src/**/*.ts\"",
"eslint": "eslint --ext .ts --fix .",
"eslint:check": "eslint --ext .ts.",
"test": "jest"
"eslint:check": "eslint --ext .ts."
},
"files": [
"lib/"
Expand Down Expand Up @@ -45,20 +44,16 @@
},
"devDependencies": {
"@eslint/js": "^9.13.0",
"@types/jest": "^29.5.14",
"@types/node": "^22.8.1",
"@typescript-eslint/eslint-plugin": "~6.13.2",
"@typescript-eslint/parser": "~6.13.2",
"eslint": "~8.55.0",
"eslint-config-prettier": "~9.1.0",
"eslint-plugin-jest": "~27.6.0",
"eslint-plugin-prettier": "~5.0.1",
"eslint-plugin-react": "~7.33.2",
"globals": "^15.11.0",
"jest": "^29.7.0",
"prettier": "^3.3.3",
"rimraf": "^3.0.2",
"ts-jest": "^29.2.5",
"typescript": "^5",
"typescript-eslint": "^8.12.2"
}
Expand Down
24 changes: 10 additions & 14 deletions src/helper.ts
Original file line number Diff line number Diff line change
@@ -1,21 +1,17 @@
import unpack, { IWasmModule } from './unpack';
import unpackWasm from './unpack.wasm';

const initializeWasm = async (): Promise<IWasmModule | undefined> => {
try {
const wasmModule: IWasmModule = await unpack({
locateFile(path: string) {
if (path.endsWith('.wasm')) {
return unpackWasm;
}
return path;
const initializeWasm = async (): Promise<IWasmModule> => {
const wasmModule: IWasmModule = await unpack({
locateFile(path: string) {
if (path.endsWith('.wasm')) {
return unpackWasm;
}
});
return path;
}
});

return wasmModule;
} catch (err) {
console.error('Error initializing the WASM module:', err);
return undefined;
}
return wasmModule;
};

export default initializeWasm;
242 changes: 114 additions & 128 deletions src/index.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
import initializeWasm from './helper';
import { IFileData } from './types';
import { IWasmModule } from './unpack';
import { IFileData, IUnpackJSAPI } from './types';

const fetchByteArray = async (url: string): Promise<Uint8Array> => {
const response = await fetch(url);
Expand All @@ -11,138 +10,125 @@ const fetchByteArray = async (url: string): Promise<Uint8Array> => {
return new Uint8Array(arrayBuffer);
};

const init = async (): Promise<IWasmModule | null> => {
try {
const wasmModule = await initializeWasm();
return wasmModule as IWasmModule;
} catch (error) {
console.error('Error initializing WASM module:', error);
return null;
}
};

export const extractData = async (data: Uint8Array): Promise<IFileData[]> => {
const wasmModule = await init();
if (!wasmModule) {
console.error('WASM module not initialized.');
return [];
}

/**Since WebAssembly, memory is accessed using pointers
and the first parameter of extract_archive method from unpack.c, which is Uint8Array of file data, should be a pointer
so we have to allocate memory for file data
**/
const inputPtr = wasmModule._malloc(data.length);
wasmModule.HEAPU8.set(data, inputPtr);

// fileCountPtr is the pointer to 4 bytes of memory in WebAssembly's heap that holds fileCount value from the ExtractedArchive structure in unpack.c.
const fileCountPtr = wasmModule._malloc(4);

try {
const resultPtr = wasmModule._extract_archive(
inputPtr,
data.length,
fileCountPtr
);

/**
* Since extract_archive returns a pointer that refers to an instance of the ExtractedArchive in unpack.c
typedef struct {
FileData* files;
size_t fileCount;
int status;
char* error_message;
} ExtractedArchive;

its fields are laid out in memory sequentially. Based on this and types each field will take 4 bytes:

4 bytes 4 bytes 4 bytes 4 bytes
---------------|---------------|---------------|---------------
files fileCount status error_message

`resultPtr` points to the beginning of the ExtractedArchive structure in WebAssembly memory
and in order to get pointer of statusPtr we need to calculate it as: 0(offset of file pointer) + 4 (offset of fileCount) + 4 (offset for status)
'status' field and pointer of `error_message` are 32-bit signed integer
*/
const statusPtr = wasmModule.getValue(resultPtr + 8, 'i32');
const errorMessagePtr = wasmModule.getValue(resultPtr + 12, 'i32');
if (statusPtr !== 1) {
const errorMessage = wasmModule.UTF8ToString(errorMessagePtr);
console.error(
'Extraction failed with status:',
statusPtr,
'Error:',
errorMessage
export const initUntarJS = async (): Promise<IUnpackJSAPI> => {
const wasmModule = await initializeWasm();

const extractData = async (data: Uint8Array): Promise<IFileData[]> => {
/**Since WebAssembly, memory is accessed using pointers
and the first parameter of extract_archive method from unpack.c, which is Uint8Array of file data, should be a pointer
so we have to allocate memory for file data
**/
const inputPtr = wasmModule._malloc(data.length);
wasmModule.HEAPU8.set(data, inputPtr);

// fileCountPtr is the pointer to 4 bytes of memory in WebAssembly's heap that holds fileCount value from the ExtractedArchive structure in unpack.c.
const fileCountPtr = wasmModule._malloc(4);

try {
const resultPtr = wasmModule._extract_archive(
inputPtr,
data.length,
fileCountPtr
);

/**
* Since extract_archive returns a pointer that refers to an instance of the ExtractedArchive in unpack.c
typedef struct {
FileData* files;
size_t fileCount;
int status;
char* error_message;
} ExtractedArchive;

its fields are laid out in memory sequentially. Based on this and types each field will take 4 bytes:

4 bytes 4 bytes 4 bytes 4 bytes
---------------|---------------|---------------|---------------
files fileCount status error_message

`resultPtr` points to the beginning of the ExtractedArchive structure in WebAssembly memory
and in order to get pointer of statusPtr we need to calculate it as: 0(offset of file pointer) + 4 (offset of fileCount) + 4 (offset for status)
'status' field and pointer of `error_message` are 32-bit signed integer
*/
const statusPtr = wasmModule.getValue(resultPtr + 8, 'i32');
const errorMessagePtr = wasmModule.getValue(resultPtr + 12, 'i32');
if (statusPtr !== 1) {
const errorMessage = wasmModule.UTF8ToString(errorMessagePtr);
console.error(
'Extraction failed with status:',
statusPtr,
'Error:',
errorMessage
);
return [];
}
const filesPtr = wasmModule.getValue(resultPtr, 'i32');
const fileCount = wasmModule.getValue(resultPtr + 4, 'i32');

const files: IFileData[] = [];

/**
* FilesPtr is a pointer that refers to an instance of the FileData in unpack.c
typedef struct {
char* filename;
uint8_t* data;
size_t data_size;
} FileData;

and its fields are laid out in memory sequentially too so each field take 4 bytes:

4 bytes 4 bytes 4 bytes
---------------|---------------|---------------
filename data data_size

`filesPtr + i * 12` calculates the memory address of the i-th FileData element in the array
where `12` is the size of each FileData structure in memory in bytes: 4 + 4 + 4
*/

for (let i = 0; i < fileCount; i++) {
const fileDataPtr = filesPtr + i * 12;
const filenamePtr = wasmModule.getValue(fileDataPtr, 'i32');
const dataSize = wasmModule.getValue(fileDataPtr + 8, 'i32');
const dataPtr = wasmModule.getValue(fileDataPtr + 4, 'i32');
const filename = wasmModule.UTF8ToString(filenamePtr);
const fileData = new Uint8Array(
wasmModule.HEAPU8.buffer,
dataPtr,
dataSize
);

files.push({
filename: filename,
data: fileData
});
}

wasmModule._free(inputPtr);
wasmModule._free(fileCountPtr);
wasmModule._free(errorMessagePtr);
wasmModule._free(resultPtr);

return files;
} catch (error) {
console.error('Error during extraction:', error);
return [];
}
const filesPtr = wasmModule.getValue(resultPtr, 'i32');
const fileCount = wasmModule.getValue(resultPtr + 4, 'i32');

const files: IFileData[] = [];

/**
* FilesPtr is a pointer that refers to an instance of the FileData in unpack.c
typedef struct {
char* filename;
uint8_t* data;
size_t data_size;
} FileData;

and its fields are laid out in memory sequentially too so each field take 4 bytes:

4 bytes 4 bytes 4 bytes
---------------|---------------|---------------
filename data data_size

`filesPtr + i * 12` calculates the memory address of the i-th FileData element in the array
where `12` is the size of each FileData structure in memory in bytes: 4 + 4 + 4
*/

for (let i = 0; i < fileCount; i++) {
const fileDataPtr = filesPtr + i * 12;
const filenamePtr = wasmModule.getValue(fileDataPtr, 'i32');
const dataSize = wasmModule.getValue(fileDataPtr + 8, 'i32');
const dataPtr = wasmModule.getValue(fileDataPtr + 4, 'i32');
const filename = wasmModule.UTF8ToString(filenamePtr);
const fileData = new Uint8Array(
wasmModule.HEAPU8.buffer,
dataPtr,
dataSize
);

files.push({
filename: filename,
data: fileData
});
};

const extract = async (url: string): Promise<IFileData[]> => {
try {
const data = await fetchByteArray(url);
return await extractData(data);
} catch (error) {
console.error('Error during extracting:', error);
return [];
}

wasmModule._free(inputPtr);
wasmModule._free(fileCountPtr);
wasmModule._free(errorMessagePtr);
wasmModule._free(resultPtr);

return files;
} catch (error) {
console.error('Error during extraction:', error);
return [];
}
};

export const extract = async (url: string): Promise<IFileData[]> => {
try {
const data = await fetchByteArray(url);
return await extractData(data);
} catch (error) {
console.error('Error during extracting:', error);
return [];
}
return {
extract,
extractData
};
};


export * from './types';

export default {
extract,
extractData
};
5 changes: 5 additions & 0 deletions src/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,3 +2,8 @@ export interface IFileData {
filename: string;
data: Uint8Array;
}

export interface IUnpackJSAPI {
extractData: (data: Uint8Array) => Promise<IFileData[]>;
extract: (url: string) => Promise<IFileData[]>;
}
Loading
Loading