diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..4c0dbb4 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,26 @@ +name: WebARKitLib test + +on: push +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '20.x' + - name: Update Ubuntu and install libjpeg-dev + run: | + sudo apt-get update && sudo apt install libjpeg-dev + - name: Build and test WebARKitLib + run: | + cd tests && mkdir build && cd build && cmake -DEMSCRIPTEN_COMP=0 .. && make && ./webarkit_test + - name: Build WebARKitLib with Emscripten (Docker) + run: | + cd .. + ls + docker run -dit --name emscripten-webarkit-testing -v $(pwd):/src emscripten/emsdk:3.1.38 bash + docker exec emscripten-webarkit-testing emcmake cmake -B WebARKitLib/WebARKit/build -S WebARKitLib/WebARKit -DEMSCRIPTEN_COMP=1 .. + docker exec emscripten-webarkit-testing emmake make -C WebARKitLib/WebARKit/build + \ No newline at end of file diff --git a/.gitignore b/.gitignore index 4abc42f..b6fc9cf 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ -include/AR/config.h \ No newline at end of file +include/AR/config.h +tests/build +build \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..2f7d9cd --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,73 @@ +# Contributing to WebARKitLib + +Thanks for helping improve WebARKit! This is the C/C++ source of the tracker; most +changes are consumed by the [webarkit-testing](https://github.com/webarkit/webarkit-testing) +superproject as a submodule. + +## Workflow + +- **Branch from `dev`** and open your PR **against `dev`** (not `main`). +- Reference the related issue in the PR description. +- **Sign your commits** (`git commit -S …`). + +## Commit messages — Conventional Commits + +All commit messages **must** follow +[Conventional Commits](https://www.conventionalcommits.org/): + +``` +(): +``` + +Common types: `feat`, `fix`, `docs`, `refactor`, `test`, `chore`, `perf`. Examples: + +``` +feat(tracker): detect features on a downsampled frame +fix(#55): populate pose3d so getPoseMatrixCV() isn't zero +docs: document the pose pipeline +refactor(#50): remove dead computePose/invertPose +``` + +## What you may change + +- Active WebAR work lives under **`WebARKit/`** (the OCVT planar tracker). +- **Do not** modify the vendored **old ARToolKit5** sources under `lib/SRC/**` or + `include/AR/**` — those are upstream utilities used by + [jsartoolkitNFT](https://github.com/webarkit/jsartoolkitNFT) / + [ARnft](https://github.com/webarkit/ARnft) and are kept untouched. + +## Cross-repo changes (the PR pair) + +> **Applies to `WebARKit/` (this folder) ↔ the `webarkit-testing` repo.** This flow is +> for changes to the **`WebARKit/`** OCVT tracker, which the **webarkit-testing** +> superproject compiles to WASM. Changes to the vendored `lib/SRC/**` / `include/AR/**` +> (old ARToolKit5) do **not** use this flow — they reach the web through +> [jsartoolkitNFT](https://github.com/webarkit/jsartoolkitNFT), a different consumer. + +A change to `WebARKit/` that affects the WASM build is consumed by webarkit-testing via +the submodule. Coordinate both repos: + +1. Branch from `dev` in **both** WebARKitLib and webarkit-testing. +2. Make the C++ change here; in webarkit-testing bump the submodule pointer and + rebuild (`npm run build-docker` → `build-es6`). +3. Open a **PR pair** (WebARKitLib → `dev`, webarkit-testing → `dev`); cross-link them. +4. After the WebARKitLib PR merges, re-point the submodule to the merged `dev` tip. + +## Building & tests + +- The library is normally built **from webarkit-testing** (WASM) — see its README. + `WebARKit/CMakeLists.txt` can also build it as a static lib for **WASM** or **native + Linux** (selected via `EMSCRIPTEN_COMP`), pulling OpenCV from + [webarkit/opencv-em](https://github.com/webarkit/opencv-em). +- **Unit tests** (GoogleTest) live in `tests/` and run in CI on every PR: + + ```bash + cmake -S tests -B tests/build && cmake --build tests/build && ctest --test-dir tests/build + ``` + + If you change a config constant or the version, update the matching assertions in + `tests/webarkit_test.cc`. + +## License + +By contributing, you agree that your contributions are licensed under **LGPL-3.0**. diff --git a/README.md b/README.md index 9ce079d..42448ab 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,98 @@ +![github releases](https://flat.badgen.net/github/release/webarkit/WebARKitLib) +![github stars](https://flat.badgen.net/github/stars/webarkit/WebARKitLib) +![github forks](https://flat.badgen.net/github/forks/webarkit/WebARKitLib)[![Test](https://github.com/webarkit/WebARKitLib/actions/workflows/test.yml/badge.svg)](https://github.com/webarkit/WebARKitLib/actions/workflows/test.yml) + # WebARKitLib -The C/C++ source code of WebARKit, from Artoolkit5 and extended. -Updated to [Eigen](https://eigen.tuxfamily.org) 3.4.0 and Emscripten emsdk 3.1.26. +The C/C++ source of **WebARKit** — a browser-side Augmented Reality tracker. The +active code (`WebARKit/`) is an emscripten-friendly port of the **OCVT** planar +image-tracking design from [ArtoolkitX](https://github.com/artoolkitx/artoolkitx), +built on OpenCV. It is normally compiled to WebAssembly and consumed from the +[webarkit-testing](https://github.com/webarkit/webarkit-testing) superproject. + +> **Two lineages live here, keep them straight:** +> - **`WebARKit/`** — the WebAR OCVT planar tracker (the design reference is *ArtoolkitX*). This is the code under active development. +> - **`lib/SRC/**`, `include/AR/**`** — vendored **old ARToolKit5** utility sources/headers, kept only because [jsartoolkitNFT](https://github.com/webarkit/jsartoolkitNFT) needs them (and [ARnft](https://github.com/webarkit/ARnft) in turn consumes jsartoolkitNFT as an npm module). Treated as untouched upstream. (Updated to Eigen 3.4.0 / emsdk 3.1.26.) + +## Code layout (`WebARKit/`) + +| Component | Role | +|---|---| +| `WebARKitManager` | top-level façade — owns the tracker, drives init/process, exposes the output (homography, pose, GL-view, projection) | +| `WebARKitTrackers/WebARKitOpticalTracking/WebARKitTracker` | the OCVT tracker core: feature detect/match → homography → optical-flow + template-match tracking | +| `…/TrackingPointSelector` + `…/TrackedPoint` | bin-based selection of template tracking points (ArtoolkitX-derived) | +| `…/WebARKitHomographyInfo` + `WebARKitUtils.h` | RANSAC homography estimation/validation (`getHomographyInliers`) | +| `…/WebARKitConfig` | tuning constants (feature counts, match ratios, pyramid/template sizes, version) | +| `…/TrackerVisualization` | optional debug-overlay scaffolding (inert until wired) | +| `WebARKitCamera` | camera intrinsics from a diagonal-FOV estimate | +| `WebARKitGL` | CV→GL matrix conversion (`arglCameraViewRHf`) + GL projection | +| `WebARKitPattern` | the reference marker (`WebARKitPattern`) and per-frame pose state (`WebARKitPatternTrackingInfo`) | +| `WebARKitLog` | logging | + +## Pose pipeline (one glance) + +``` +solvePnP (cameraPoseFromPoints) // OpenCV-convention camera pose + -> getTrackablePose / updateTrackable // CV->GL handedness fix, D*R*D (see #42) + -> arglCameraViewRHf // right-handed GL modelview + -> matrixGL_RH // what the examples attach content to +``` + +Pose getters: **`getPoseMatrixCV()`** (raw 4×4 OpenCV pose) and **`getPoseMatrixGL()`** +(the GL/right-handed pose); projection via `getCameraProjectionMatrix()`. The full +derivation is in +[`docs/design-projection-and-pose-artoolkitx-alignment.md`](https://github.com/webarkit/webarkit-testing/blob/dev/docs/design-projection-and-pose-artoolkitx-alignment.md) +(webarkit-testing). + +## Building + +This repo isn't built as a single unit — each consumer compiles the **subset it +needs**: + +- **`WebARKit/` (the OCVT tracker)** is built by the + [webarkit-testing](https://github.com/webarkit/webarkit-testing) superproject, which + compiles it to WebAssembly with emscripten (emsdk **3.1.26**) inside Docker and wires + it to JS through `emscripten/WebARKitJS.{cpp,h}` + `bindings.cpp` (which live in + webarkit-testing). webarkit-testing currently drives that compile with the Node script + `tools/makem.js` (`npm run build-docker` → `build-es6`); migrating to the CMake config + (`WebARKit/CMakeLists.txt`) is planned. +- **The vendored `lib/SRC/**` + `include/AR/**` (old ARToolKit5)** is built by + [jsartoolkitNFT](https://github.com/webarkit/jsartoolkitNFT) as part of its build — + not by webarkit-testing. ([ARnft](https://github.com/webarkit/ARnft) consumes + jsartoolkitNFT as an npm module and doesn't build the emscripten code itself.) + +`WebARKit/CMakeLists.txt` can **already build WebARKitLib as a static library** +today — for both **WASM** (emscripten) and **native Linux**, selected via the +`EMSCRIPTEN_COMP` flag. It depends on a prebuilt **OpenCV** from +[webarkit/opencv-em](https://github.com/webarkit/opencv-em) (the emscripten +`opencv-js-…-emcc` build for WASM, or the native `opencv-…` build for Linux), +fetched automatically via CMake `FetchContent`. The "planned" part above is +webarkit-testing switching its WASM build over to this CMake config (away from +`tools/makem.js`). The same config also drives the unit tests (see below). + +## Tests + +C++ unit tests (GoogleTest) live in [`tests/`](tests/) (`webarkit_test.cc`, +`CMakeLists.txt`, `pinball.jpg`) and run in CI via +[`.github/workflows/test.yml`](https://github.com/webarkit/WebARKitLib/actions/workflows/test.yml). +Build them standalone with CMake: + +```bash +cmake -S tests -B tests/build && cmake --build tests/build && ctest --test-dir tests/build +``` + +## Contributing + +Please read [`./CONTRIBUTING.md`](./CONTRIBUTING.md) for the full guidelines. In short: + +- Branch from `dev`; PR back to `dev`; sign your commits and reference the issue. +- **Commit messages must follow [Conventional Commits](https://www.conventionalcommits.org/)** + (e.g. `feat:`, `fix:`, `docs:`, `refactor:`, `test:`, `chore:`). +- Library changes are usually paired with a build/bump in webarkit-testing — see that + repo's "cross-repo PR pair" flow. +- Don't modify the vendored `lib/SRC/**` / `include/AR/**` (old ARToolKit5) as part of + WebAR work. + +## License + +LGPL-3.0 (see `LICENSE.txt`). diff --git a/WebARKit/CMakeLists.txt b/WebARKit/CMakeLists.txt new file mode 100644 index 0000000..724e5cd --- /dev/null +++ b/WebARKit/CMakeLists.txt @@ -0,0 +1,83 @@ +cmake_minimum_required(VERSION 3.16) + +project(WebARKitLib) + +set(CMAKE_CXX_STANDARD 14) + +if(VERSION GREATER 3.24) + cmake_policy(SET CMP0135 NEW) +endif() + +include(FetchContent) + +if(${EMSCRIPTEN_COMP} EQUAL 1) + message("Fetching opencv for emscripten compilation from webarkit/opencv-em ...") + FetchContent_Declare( + build_opencv + URL https://github.com/webarkit/opencv-em/releases/download/0.1.6/opencv-js-4.10.0-emcc-3.1.38.zip + ) +else() + message("Fetching opencv from webarkit/opencv-em ...") + FetchContent_Declare( + build_opencv + URL https://github.com/webarkit/opencv-em/releases/download/0.1.6/opencv-4.10.0.zip + ) +endif() + +FetchContent_MakeAvailable(build_opencv) + +get_filename_component(PARENT_DIR ./ ABSOLUTE) + +set(WEBARKIT_HEADERS +${PARENT_DIR}/WebARKitTrackers/WebARKitOpticalTracking/include/WebARKitTrackers/WebARKitOpticalTracking/TrackedPoint.h +${PARENT_DIR}/WebARKitTrackers/WebARKitOpticalTracking/include/WebARKitTrackers/WebARKitOpticalTracking/TrackingPointSelector.h +${PARENT_DIR}/WebARKitTrackers/WebARKitOpticalTracking/include/WebARKitTrackers/WebARKitOpticalTracking/WebARKitConfig.h +${PARENT_DIR}/WebARKitTrackers/WebARKitOpticalTracking/include/WebARKitTrackers/WebARKitOpticalTracking/WebARKitEnums.h +${PARENT_DIR}/WebARKitTrackers/WebARKitOpticalTracking/include/WebARKitTrackers/WebARKitOpticalTracking/WebARKitHomographyInfo.h +${PARENT_DIR}/WebARKitTrackers/WebARKitOpticalTracking/include/WebARKitTrackers/WebARKitOpticalTracking/WebARKitTracker.h +${PARENT_DIR}/WebARKitTrackers/WebARKitOpticalTracking/include/WebARKitTrackers/WebARKitOpticalTracking/WebARKitUtils.h +${PARENT_DIR}/include/WebARKitCamera.h +${PARENT_DIR}/include/WebARKitLog.h +${PARENT_DIR}/include/WebARKitGL.h +${PARENT_DIR}/include/WebARKitManager.h +${PARENT_DIR}/include/WebARKitPattern.h +) + +set(SOURCE +${PARENT_DIR}/WebARKitTrackers/WebARKitOpticalTracking/TrackedPoint.cpp +${PARENT_DIR}/WebARKitTrackers/WebARKitOpticalTracking/TrackingPointSelector.cpp +${PARENT_DIR}/WebARKitTrackers/WebARKitOpticalTracking/WebARKitConfig.cpp +${PARENT_DIR}/WebARKitTrackers/WebARKitOpticalTracking/WebARKitHomographyInfo.cpp +${PARENT_DIR}/WebARKitTrackers/WebARKitOpticalTracking/WebARKitTracker.cpp +${PARENT_DIR}/WebARKitCamera.cpp +${PARENT_DIR}/WebARKitLog.cpp +${PARENT_DIR}/WebARKitGL.cpp +${PARENT_DIR}/WebARKitManager.cpp +${PARENT_DIR}/WebARKitPattern.cpp +) + +add_library( + WebARKitLib STATIC + ${WEBARKIT_HEADERS} + ${SOURCE} +) + +target_include_directories(WebARKitLib PRIVATE "${PARENT_DIR}/include") +target_include_directories(WebARKitLib PRIVATE "${PARENT_DIR}/WebARKitTrackers/WebARKitOpticalTracking/include") + +target_include_directories(WebARKitLib PRIVATE + "${build_opencv_SOURCE_DIR}" + "${build_opencv_SOURCE_DIR}/libs/opencv/include" + "${build_opencv_SOURCE_DIR}/libs/opencv/modules/calib3d/include" + "${build_opencv_SOURCE_DIR}/libs/opencv/modules/core/include" + "${build_opencv_SOURCE_DIR}/libs/opencv/modules/features2d/include" + "${build_opencv_SOURCE_DIR}/libs/opencv/modules/flann/include" + "${build_opencv_SOURCE_DIR}/libs/opencv/modules/imgproc/include" + "${build_opencv_SOURCE_DIR}/libs/opencv/modules/video/include" + "${build_opencv_SOURCE_DIR}/libs/opencv_contrib/modules/xfeatures2d/include" + ) + +target_link_libraries(WebARKitLib +jpeg +"${build_opencv_SOURCE_DIR}/3rdparty/lib/liblibopenjp2.a" +) \ No newline at end of file diff --git a/WebARKit/WebARKitCamera.cpp b/WebARKit/WebARKitCamera.cpp new file mode 100644 index 0000000..98b4006 --- /dev/null +++ b/WebARKit/WebARKitCamera.cpp @@ -0,0 +1,52 @@ +#include +#include +#include + +namespace webarkit { +WebARKitCamera::WebARKitCamera() : xsize(-1), ysize(-1), diagonal_fov_degrees(70.0) { cmat.fill(0.0); } + +WebARKitCamera::~WebARKitCamera() {} + +bool WebARKitCamera::setupCamera(int width, int height) { + if (width <= 0 || height <= 0) { + return false; + } + xsize = width; + ysize = height; + + setFocalLength(xsize, ysize); + + cmat.at(0) = focal_length; + cmat.at(2) = 0.5 * xsize; + cmat.at(4) = focal_length; + cmat.at(5) = 0.5 * ysize; + cmat.at(8) = 1.0; + kc.fill(0.0); + return true; +}; + +void WebARKitCamera::printSettings() { + WEBARKIT_LOGi("WebARKit: Camera Size %d , %d\n", xsize, ysize); + WEBARKIT_LOGi("WebARKit: camera matrix = [%.2f %.2f %.2f]\n", cmat[0], cmat[1], cmat[2]); + WEBARKIT_LOGi(" [%.2f %.2f %.2f]\n", cmat[3], cmat[4], cmat[5]); + WEBARKIT_LOGi(" [%.2f %.2f %.2f]\n", cmat[6], cmat[7], cmat[8]); + WEBARKIT_LOGi("WebARKit: kc = [%.4f %.4f %.4f %.4f %.4f %.4f]\n", kc[0], kc[1], kc[2], kc[3], kc[4], kc[5]); +}; + +std::array WebARKitCamera::getCameraData() const { + return cmat; +} + +std::array WebARKitCamera::getDistortionCoefficients() const { + return kc; +} + +void WebARKitCamera::setFocalLength(int width, int height) { + double diagonal_image_size; + double diagonal_fov_radians; + // simple routine to calculate focal length from diagonal field of view, and convert to camera matrix. + diagonal_image_size = std::pow(std::pow(width, 2.0) + std::pow(height, 2.0), 0.5); + diagonal_fov_radians = diagonal_fov_degrees * m_pi / 180.0; + focal_length = 0.5 * diagonal_image_size / std::tan(0.5 * diagonal_fov_radians); +} +} // namespace webarkit \ No newline at end of file diff --git a/WebARKit/WebARKitGL.cpp b/WebARKit/WebARKitGL.cpp new file mode 100644 index 0000000..3588777 --- /dev/null +++ b/WebARKit/WebARKitGL.cpp @@ -0,0 +1,89 @@ +#include + +namespace webarkit { + +void arglCameraViewRHf(float para[3][4], float m_modelview[16], const float scale) { + m_modelview[0 + 0 * 4] = para[0][0]; // R1C1 + m_modelview[0 + 1 * 4] = para[0][1]; // R1C2 + m_modelview[0 + 2 * 4] = para[0][2]; + m_modelview[0 + 3 * 4] = para[0][3]; + m_modelview[1 + 0 * 4] = -para[1][0]; // R2 + m_modelview[1 + 1 * 4] = -para[1][1]; + m_modelview[1 + 2 * 4] = -para[1][2]; + m_modelview[1 + 3 * 4] = -para[1][3]; + m_modelview[2 + 0 * 4] = -para[2][0]; // R3 + m_modelview[2 + 1 * 4] = -para[2][1]; + m_modelview[2 + 2 * 4] = -para[2][2]; + m_modelview[2 + 3 * 4] = -para[2][3]; + m_modelview[3 + 0 * 4] = 0.0f; + m_modelview[3 + 1 * 4] = 0.0f; + m_modelview[3 + 2 * 4] = 0.0f; + m_modelview[3 + 3 * 4] = 1.0f; + if (scale != 0.0f) { + m_modelview[12] *= scale; + m_modelview[13] *= scale; + m_modelview[14] *= scale; + } +} + +void arglCameraViewRHf(cv::Mat para, std::array& m_modelview, const double scale) { + m_modelview[0 + 0 * 4] = para.at(0, 0); // R1C1 + m_modelview[0 + 1 * 4] = para.at(0, 1); // R1C2 + m_modelview[0 + 2 * 4] = para.at(0, 2); + m_modelview[0 + 3 * 4] = para.at(0, 3); + m_modelview[1 + 0 * 4] = -para.at(1, 0); // R2 + m_modelview[1 + 1 * 4] = -para.at(1, 1); + m_modelview[1 + 2 * 4] = -para.at(1, 2); + m_modelview[1 + 3 * 4] = -para.at(1, 3); + m_modelview[2 + 0 * 4] = -para.at(2, 0); // R3 + m_modelview[2 + 1 * 4] = -para.at(2, 1); + m_modelview[2 + 2 * 4] = -para.at(2, 2); + m_modelview[2 + 3 * 4] = -para.at(2, 3); + m_modelview[3 + 0 * 4] = 0.0f; + m_modelview[3 + 1 * 4] = 0.0f; + m_modelview[3 + 2 * 4] = 0.0f; + m_modelview[3 + 3 * 4] = 1.0f; + if (scale != 0.0f) { + m_modelview[12] *= scale; + m_modelview[13] *= scale; + m_modelview[14] *= scale; + } +} + +void cameraProjectionMatrix(const std::array& calibration, double nearPlane, double farPlane, int screenWidth, int screenHeight, std::array& projectionMatrix) +{ + // Camera parameters + double f_x = calibration.at(0); // Focal length in x axis + double f_y = calibration.at(4); // Focal length in y axis (usually the same?) + double c_x = calibration.at(2); // Camera primary point x + double c_y = calibration.at(5); // Camera primary point y + + // WebARKitLib#42: standard GL projection with BOTH focal terms positive. + // The modelview (matrixGL_RH) already performs the CV->GL handedness flip by + // negating the Y and Z rows in arglCameraViewRHf, so the projection itself must + // be the plain pinhole form. The previous negative X focal (-2*f_x/w) mirrored + // the marker horizontally: a marker origin at camera -X projected to +X (right) + // instead of -X (left). Verified against the tracked pose: origin (tx,ty,tz)= + // (-1.996, 2.223, -9.094) projects to NDC (-0.39, +0.58) (upper-left, on the + // marker) only with +2*f_x/w; the negative sign put it upper-right. + projectionMatrix[0] = 2.0f * f_x / screenWidth; + projectionMatrix[1] = 0.0f; + projectionMatrix[2] = 0.0f; + projectionMatrix[3] = 0.0f; + + projectionMatrix[4] = 0.0f; + projectionMatrix[5] = 2.0f * f_y / screenHeight; + projectionMatrix[6] = 0.0f; + projectionMatrix[7] = 0.0f; + + projectionMatrix[8] = 2.0f * c_x / screenWidth - 1.0f; + projectionMatrix[9] = 2.0f * c_y / screenHeight - 1.0f; + projectionMatrix[10] = -( farPlane + nearPlane) / ( farPlane - nearPlane ); + projectionMatrix[11] = -1.0f; + + projectionMatrix[12] = 0.0f; + projectionMatrix[13] = 0.0f; + projectionMatrix[14] = -2.0f * farPlane * nearPlane / ( farPlane - nearPlane ); + projectionMatrix[15] = 0.0f; +} +} // namespace webarkit \ No newline at end of file diff --git a/WebARKit/WebARKitLog.cpp b/WebARKit/WebARKitLog.cpp new file mode 100644 index 0000000..9280853 --- /dev/null +++ b/WebARKit/WebARKitLog.cpp @@ -0,0 +1,212 @@ +/* + * WebARKitLog.cpp + * WebARKit + * + * This file is part of WebARKit. + * + * WebARKit is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * WebARKit is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with WebARKit. If not, see . + * + * As a special exception, the copyright holders of this library give you + * permission to link this library with independent modules to produce an + * executable, regardless of the license terms of these independent modules, and to + * copy and distribute the resulting executable under terms of your choice, + * provided that you also meet, for each linked independent module, the terms and + * conditions of the license of that module. An independent module is a module + * which is neither derived from nor based on this library. If you modify this + * library, you may extend this exception to your version of the library, but you + * are not obligated to do so. If you do not wish to do so, delete this exception + * statement from your version. + * + * Copyright 2015-2016 Daqri, LLC. + * Copyright 2003-2015 ARToolworks, Inc. + * Copyright 2023 WebARKit. + * + * Author(s): Hirokazu Kato, Philip Lamb, Walter Perdan + * + * This code was taken from Artoolkit5 https://github.com/artoolkitx/artoolkit5 + * with small modifications to adapt to existing WebARKIt code + * + */ + +#include + +#ifndef _WIN32 +# include // pthread_self(), pthread_equal() +# ifdef __APPLE__ +# include +# endif +#else +# include +# define snprintf _snprintf +#endif + +// +// Global required for logging functions. +// +int webarkitLogLevel = WEBARKIT_LOG_LEVEL_DEFAULT; +static WEBARKIT_LOG_LOGGER_CALLBACK webarkitLogLoggerCallback = NULL; +static int webarkitLogLoggerCallBackOnlyIfOnSameThread = 0; +#ifndef _WIN32 +static pthread_t webarkitLogLoggerThread; +#else +static DWORD webarkitLogLoggerThreadID; +#endif +#define WEBARKIT_LOG_WRONG_THREAD_BUFFER_SIZE 4096 +static char *webarkitLogWrongThreadBuffer = NULL; +static size_t webarkitLogWrongThreadBufferSize = 0; +static size_t webarkitLogWrongThreadBufferCount = 0; + + +void webarkitLogSetLogger(WEBARKIT_LOG_LOGGER_CALLBACK callback, int callBackOnlyIfOnSameThread) +{ + webarkitLogLoggerCallback = callback; + webarkitLogLoggerCallBackOnlyIfOnSameThread = callBackOnlyIfOnSameThread; + if (callback && callBackOnlyIfOnSameThread) { +#ifndef _WIN32 + webarkitLogLoggerThread = pthread_self(); +#else + webarkitLogLoggerThreadID = GetCurrentThreadId(); +#endif + if (!webarkitLogWrongThreadBuffer) { + if ((webarkitLogWrongThreadBuffer = static_cast(malloc(sizeof(char) * WEBARKIT_LOG_WRONG_THREAD_BUFFER_SIZE)))) { + webarkitLogWrongThreadBufferSize = WEBARKIT_LOG_WRONG_THREAD_BUFFER_SIZE; + } + } + } else { + if (webarkitLogWrongThreadBuffer) { + free(webarkitLogWrongThreadBuffer); + webarkitLogWrongThreadBuffer = NULL; + webarkitLogWrongThreadBufferSize = 0; + } + } +} + +void webarkitLog(const char *tag, const int logLevel, const char *format, ...) +{ + if (logLevel < webarkitLogLevel) return; + if (!format || !format[0]) return; + + va_list ap; + va_start(ap, format); + webarkitLogv(tag, logLevel, format, ap); + va_end(ap); +} + +void webarkitLogv(const char *tag, const int logLevel, const char *format, va_list ap) +{ + va_list ap2; + char *buf = NULL; + size_t len; + const char *logLevelStrings[] = { + "debug", + "info", + "warning", + "error" + }; + const size_t logLevelStringsCount = (sizeof(logLevelStrings)/sizeof(logLevelStrings[0])); + size_t logLevelStringLen; + + if (logLevel < webarkitLogLevel) return; + if (!format || !format[0]) return; + + // Count length required to unpack varargs. + va_copy(ap2, ap); +#ifdef _WIN32 + len = _vscprintf(format, ap); +#else + len = vsnprintf(NULL, 0, format, ap2); +#endif + va_end(ap2); + if (len < 1) return; + + // Add characters required for logLevelString. + if (logLevel >= 0 && logLevel < (int)logLevelStringsCount) { + logLevelStringLen = 3 + strlen(logLevelStrings[logLevel]); // +3 for brackets and a space, e.g. "[debug] ". + } else { + logLevelStringLen = 0; + } + + buf = (char *)malloc((logLevelStringLen + len + 1) * sizeof(char)); // +1 for nul-term. + + if (logLevelStringLen > 0) { + snprintf(buf, logLevelStringLen + 1, "[%s] ", logLevelStrings[logLevel]); + } + + vsnprintf(buf + logLevelStringLen, len + 1, format, ap); + len += logLevelStringLen; + + if (webarkitLogLoggerCallback) { + + if (!webarkitLogLoggerCallBackOnlyIfOnSameThread) { + (*webarkitLogLoggerCallback)(buf); + } else { +#ifndef _WIN32 + if (!pthread_equal(pthread_self(), webarkitLogLoggerThread)) +#else + if (GetCurrentThreadId() != webarkitLogLoggerThreadID) +#endif + { + // On non-log thread, put it into buffer if we can. + if (webarkitLogWrongThreadBuffer && (webarkitLogWrongThreadBufferCount < webarkitLogWrongThreadBufferSize)) { + if (len <= (webarkitLogWrongThreadBufferSize - (webarkitLogWrongThreadBufferCount + 4))) { // +4 to reserve space for "...\0". + strncpy(&webarkitLogWrongThreadBuffer[webarkitLogWrongThreadBufferCount], buf, len + 1); + webarkitLogWrongThreadBufferCount += len; + } else { + strncpy(&webarkitLogWrongThreadBuffer[webarkitLogWrongThreadBufferCount], "...", 4); + webarkitLogWrongThreadBufferCount = webarkitLogWrongThreadBufferSize; // Mark buffer as full. + } + } + } else { + // On log thread, print buffer if anything was in it, then the current message. + if (webarkitLogWrongThreadBufferCount > 0) { + (*webarkitLogLoggerCallback)(webarkitLogWrongThreadBuffer); + webarkitLogWrongThreadBufferCount = 0; + } + (*webarkitLogLoggerCallback)(buf); + } + } + + } else { +#if defined(__ANDROID__) + int logLevelA; + switch (logLevel) { + case WEBARKIT_LOG_LEVEL_REL_INFO: logLevelA = ANDROID_LOG_ERROR; break; + case WEBARKIT_LOG_LEVEL_ERROR: logLevelA = ANDROID_LOG_ERROR; break; + case WEBARKIT_LOG_LEVEL_WARN: logLevelA = ANDROID_LOG_WARN; break; + case WEBARKIT_LOG_LEVEL_INFO: logLevelA = ANDROID_LOG_INFO; break; + case WEBARKIT_LOG_LEVEL_DEBUG: default: logLevelA = ANDROID_LOG_DEBUG; break; + } + __android_log_write(logLevelA, (tag ? tag : "libAR"), buf); + //#elif defined(_WINRT) + // OutputDebugStringA(buf); +#elif defined(__APPLE__) + if (os_log_create == NULL) { // os_log only available macOS 10.12 / iOS 10.0 and later. + fprintf(stderr, "%s", buf); + } else { + os_log_type_t type; + switch (logLevel) { + case WEBARKIT_LOG_LEVEL_REL_INFO: type = OS_LOG_TYPE_DEFAULT; break; + case WEBARKIT_LOG_LEVEL_ERROR: type = OS_LOG_TYPE_ERROR; break; + case WEBARKIT_LOG_LEVEL_WARN: type = OS_LOG_TYPE_DEFAULT; break; + case WEBARKIT_LOG_LEVEL_INFO: type = OS_LOG_TYPE_INFO; break; + case WEBARKIT_LOG_LEVEL_DEBUG: default: type = OS_LOG_TYPE_DEBUG; break; + } + os_log_with_type(OS_LOG_DEFAULT, type, "%{public}s", buf); + } +#else + fprintf(stderr, "%s", buf); +#endif + } + free(buf); +} diff --git a/WebARKit/WebARKitManager.cpp b/WebARKit/WebARKitManager.cpp new file mode 100644 index 0000000..9b84f3f --- /dev/null +++ b/WebARKit/WebARKitManager.cpp @@ -0,0 +1,130 @@ +#include + +namespace webarkit { + +WebARKitManager::WebARKitManager() : state(NOTHING_INITIALISED), versionString("?"), m_trackerType(webarkit::AKAZE_TRACKER) {} + +WebARKitManager::~WebARKitManager() { + if (!versionString.empty()) { + versionString.clear(); + } +} + +std::string WebARKitManager::getWebARKitVersion() { + if (versionString.empty()) { + versionString = webarkitGetVersion(); + WEBARKIT_LOGi("Webarkit C++ lib version: %s.\n", versionString.c_str()); + } + return versionString; +} + +bool WebARKitManager::initialiseBase(webarkit::TRACKER_TYPE trackerType, int frameWidth, int frameHeight) { + WEBARKIT_LOGd("WebARKItManager::initialiseBase(...)\n"); + if (state != NOTHING_INITIALISED) { + WEBARKIT_LOGe("Initialise called while already initialised. Will finish first.\n"); + if (!shutdown()) { + return false; + } + } + + versionString = webarkitGetVersion(); + WEBARKIT_LOGi("Webarkit C++ lib v%s initalised.\n", versionString.c_str()); + + m_trackerType = trackerType; + + m_tracker = std::make_shared(); + m_tracker->initialize(m_trackerType, frameWidth, frameHeight); + + state = BASE_INITIALISED; + + WEBARKIT_LOGd("WebARKitManager::initialiseBase() done.\n"); + return true; +} + +bool WebARKitManager::initTracker(cv::Mat refData, size_t refCols, size_t refRows, ColorSpace colorSpace) { + WEBARKIT_LOGd("WebARKitManager::initTracker(...)\n"); + if (refData.empty() || refCols <= 0 || refRows <= 0) { + WEBARKIT_LOGe("Error initialising tracker.\n"); + return false; + } + m_tracker->initTracker(refData, refCols, refRows, colorSpace); + state = WAITING_FOR_VIDEO; + WEBARKIT_LOGd("WebARKitManager::initTracker() done.\n"); + return true; +} + +bool WebARKitManager::initTracker(uchar* refData, size_t refCols, size_t refRows, ColorSpace colorSpace) { + WEBARKIT_LOGd("WebARKitManager::initTracker(...)\n"); + if (!refData || refCols <= 0 || refRows <= 0) { + WEBARKIT_LOGe("Error initialising tracker.\n"); + return false; + } + m_tracker->initTracker(refData, refCols, refRows, colorSpace); + state = WAITING_FOR_VIDEO; + WEBARKIT_LOGd("WebARKitManager::initTracker() done.\n"); + return true; +} + +void WebARKitManager::setLogLevel(int logLevel) { + WEBARKIT_LOGd("WebARKitManager::setLogLevel(...)\n"); + if (logLevel >= 0) { + webarkitLogLevel = logLevel; + } + WEBARKIT_LOGd("WebARKitManager::setLogLevel() done.\n"); +} + +bool WebARKitManager::shutdown() { + m_tracker.reset(); + WEBARKIT_LOGd("WebARKitManager::shutdown(): Shutting down WebARKitManager."); + return true; +}; + +void WebARKitManager::processFrameData(uchar* frameData, size_t frameCols, size_t frameRows, ColorSpace colorSpace, BLUR_TYPE blurType) { + WEBARKIT_LOGd("WebARKitManager::processFrameData(...)\n"); + if (state < WAITING_FOR_VIDEO) { + WEBARKIT_LOGe("processFrameData called without init the tracker. Call first initTracker.\n"); + } + if (!frameData || frameCols <= 0 || frameRows <= 0) { + WEBARKIT_LOGe("Error initialising processFrameData.\n"); + //return false; + } + m_tracker->processFrameData(frameData, frameCols, frameRows, colorSpace, blurType); + state = DETECTION_RUNNING; + WEBARKIT_LOGd("WebARKitManager::processFrameData() done\n"); +} + +std::vector WebARKitManager::getOutputData() { + return m_tracker->getOutputData(); +}; + +cv::Mat WebARKitManager::getPoseMatrixCV() { + return m_tracker->getPoseMatrixCV(); +} + +float* WebARKitManager::getPoseMatrixGL() { + return m_tracker->getPoseMatrixGL(); +} + +cv::Mat WebARKitManager::getGLViewMatrix() { + return m_tracker->getGLViewMatrix(); +} + +std::array WebARKitManager::getTransformationMatrix() { + std::array transformationMatrix; + webarkit::arglCameraViewRHf((float (*)[4])m_tracker->getPoseMatrixGL(), (float*)transformationMatrix.data(), 1.0f); + return transformationMatrix; +} + +std::array WebARKitManager::getCameraProjectionMatrix() { + return m_tracker->getCameraProjectionMatrix(); +} + +bool WebARKitManager::isValid() { + return m_tracker->isValid(); +} + +void WebARKitManager::setOriginCentered(bool centered) { + m_tracker->setOriginCentered(centered); +} + +} // namespace webarkit \ No newline at end of file diff --git a/WebARKit/WebARKitPattern.cpp b/WebARKit/WebARKitPattern.cpp new file mode 100644 index 0000000..d82861d --- /dev/null +++ b/WebARKit/WebARKitPattern.cpp @@ -0,0 +1,70 @@ +#include +#include + +WebARKitPatternTrackingInfo::WebARKitPatternTrackingInfo() { + pose3d = cv::Mat::zeros(4, 4, CV_64FC1); + glViewMatrix = cv::Mat::zeros(4, 4, CV_64FC1); + m_scale = 1.0f; +} + +void WebARKitPatternTrackingInfo::cameraPoseFromPoints(cv::Mat& pose, const std::vector& objPts, + const std::vector& imgPts, const cv::Matx33f& caMatrix, + const cv::Mat& distCoeffs) { + cv::Mat rvec = cv::Mat::zeros(3, 1, CV_64FC1); // output rotation vector + cv::Mat tvec = cv::Mat::zeros(3, 1, CV_64FC1); // output translation vector + + cv::solvePnPRansac(objPts, imgPts, caMatrix, distCoeffs, rvec, tvec); + + // Assemble pose matrix from rotation and translation vectors. + cv::Mat rMat; + Rodrigues(rvec, rMat); + cv::hconcat(rMat, tvec, pose); + + // WebARKitLib#55: also populate pose3d -- the raw 4x4 OpenCV-convention camera + // pose returned by getPoseMatrixCV(). The live path previously left pose3d at its + // zero-initialised value, so getPoseMatrixCV() returned an all-zero matrix. Build + // it here from the same rMat/tvec the GL path uses, so the CV and GL getters both + // reflect the current frame. No handedness/scale correction is applied (that is + // getPoseMatrixGL's trans); pose3d stays in raw OpenCV convention. + for (int row = 0; row < 3; ++row) { + for (int col = 0; col < 3; ++col) { + pose3d.at(row, col) = rMat.at(row, col); + } + pose3d.at(row, 3) = tvec.at(row, 0); + } + pose3d.at(3, 3) = 1.0; +}; + +void WebARKitPatternTrackingInfo::getTrackablePose(cv::Mat& pose) { + //float transMat [3][4]; + cv::Mat poseOut; + pose.convertTo(poseOut, CV_32FC1); + //std::cout << "poseOut: " << poseOut << std::endl; + memcpy(transMat, poseOut.ptr(0), 3*4*sizeof(float)); +} + +void WebARKitPatternTrackingInfo::updateTrackable() { + if (transMat) { + // CV->GL conversion, marker/object side: negate the Y and Z rotation + // COLUMNS (matching ArtoolkitX ARTrackable2d::updateWithTwoDResults). + // Combined with the Y,Z ROW negation applied downstream by arglCameraViewRHf + // (when building matrixGL_RH / getGLViewMatrix), this yields the consistent + // similarity D*R*D (D = diag(1,-1,-1)) -- a proper right-handed marker frame + // with X=right, Y=up, Z=toward the viewer, so AR content placed at +Z pops + // up out of the marker as expected. Without this column negation the frame + // is the raw OpenCV handedness (Z into the marker / away from the camera). + // + // The translation (column 3) is NOT negated here -- it keeps its sign and + // the marker-size scale. arglCameraViewRHf negates the Y,Z translation rows + // downstream, so depth stays in front of the camera (no "behind camera" + // regression); only the rotation handedness is corrected. See WebARKitLib#42. + for (int j = 0; j < 3; j++) { + trans[j][0] = transMat[j][0]; + trans[j][1] = -transMat[j][1]; + trans[j][2] = -transMat[j][2]; + trans[j][3] = (transMat[j][3] * m_scale * 0.001f * 1.64f); + } + } +} + +void WebARKitPatternTrackingInfo::computeGLviewMatrix(cv::Mat &pose) { cv::transpose(pose, glViewMatrix); } \ No newline at end of file diff --git a/WebARKit/WebARKitTrackers/WebARKitOpticalTracking/TrackedPoint.cpp b/WebARKit/WebARKitTrackers/WebARKitOpticalTracking/TrackedPoint.cpp new file mode 100644 index 0000000..e2d6223 --- /dev/null +++ b/WebARKit/WebARKitTrackers/WebARKitOpticalTracking/TrackedPoint.cpp @@ -0,0 +1,60 @@ +/* + * TrackedPoint.cpp + * artoolkitX + * + * This file is part of artoolkitX. + * + * artoolkitX is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * artoolkitX is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with artoolkitX. If not, see . + * + * As a special exception, the copyright holders of this library give you + * permission to link this library with independent modules to produce an + * executable, regardless of the license terms of these independent modules, and to + * copy and distribute the resulting executable under terms of your choice, + * provided that you also meet, for each linked independent module, the terms and + * conditions of the license of that module. An independent module is a module + * which is neither derived from nor based on this library. If you modify this + * library, you may extend this exception to your version of the library, but you + * are not obligated to do so. If you do not wish to do so, delete this exception + * statement from your version. + * + * Copyright 2018 Realmax, Inc. + * Copyright 2015 Daqri, LLC. + * Copyright 2010-2015 ARToolworks, Inc. + * + * Author(s): Philip Lamb, Daniel Bell. + * Modified for WebARKit by @kalwalt - Walter Perdan - 2024 + * + */ + +#include + +bool TrackedPoint::IsTracking() +{ + return tracking; +} + +void TrackedPoint::SetTracking(bool newTracking) +{ + tracking = newTracking; +} + +bool TrackedPoint::IsSelected() +{ + return selected; +} + +void TrackedPoint::SetSelected(bool newSelected) +{ + selected = newSelected; +} diff --git a/WebARKit/WebARKitTrackers/WebARKitOpticalTracking/TrackingPointSelector.cpp b/WebARKit/WebARKitTrackers/WebARKitOpticalTracking/TrackingPointSelector.cpp new file mode 100644 index 0000000..8694c83 --- /dev/null +++ b/WebARKit/WebARKitTrackers/WebARKitOpticalTracking/TrackingPointSelector.cpp @@ -0,0 +1,201 @@ +/* + * TrackingPointSelector.cpp + * artoolkitX + * + * This file is part of artoolkitX. + * + * artoolkitX is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * artoolkitX is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with artoolkitX. If not, see . + * + * As a special exception, the copyright holders of this library give you + * permission to link this library with independent modules to produce an + * executable, regardless of the license terms of these independent modules, and to + * copy and distribute the resulting executable under terms of your choice, + * provided that you also meet, for each linked independent module, the terms and + * conditions of the license of that module. An independent module is a module + * which is neither derived from nor based on this library. If you modify this + * library, you may extend this exception to your version of the library, but you + * are not obligated to do so. If you do not wish to do so, delete this exception + * statement from your version. + * + * Copyright 2018 Realmax, Inc. + * Copyright 2015 Daqri, LLC. + * Copyright 2010-2015 ARToolworks, Inc. + * + * Author(s): Philip Lamb, Daniel Bell. + * Modified for WebARKit by @kalwalt - Walter Perdan - 2024 + * + */ + +#include + +TrackingPointSelector::TrackingPointSelector() +{ +} + +TrackingPointSelector::TrackingPointSelector(std::vector pts, int width, int height, int markerTemplateWidth) : + _reset(false), + _pts(pts) +{ + DistributeBins(width, height, markerTemplateWidth); +} + +/** + @brief Iterates over \_pts and for each one, provided it doesn't intersect the image border, + creates a TrackedPoint representing a template () with a serially-increasing id from 0, and puts + it into the trackingPointsBin structure (a vector of pairs of (binIndex, trackingPoint). + */ +void TrackingPointSelector::DistributeBins(int width, int height, int markerTemplateWidth) +{ + int numberOfBins = 10; + + // Split width and height dimensions into 10 bins each, for total of 100 bins. + int totalXBins = width/numberOfBins; + int totalYBins = height/numberOfBins; + // Init empty bins. + for (int i = 0; i < (numberOfBins * numberOfBins); i++) { + trackingPointBin.insert(std::pair >(i, std::vector())); + } + + // Iterate the points and add points to each bin. + for (int i = 0, id = 0; i < _pts.size(); i++) { + int bx = (int)_pts[i].x/totalXBins; + int by = (int)_pts[i].y/totalYBins; + int index = bx + (by * numberOfBins); + + cv::Rect templateRoi = cv::Rect(_pts[i].x - markerTemplateWidth, _pts[i].y - markerTemplateWidth, markerTemplateWidth*2, markerTemplateWidth*2); + bool is_inside = (templateRoi & cv::Rect(0, 0, width, height)) == templateRoi; // templateRoi must not intersect image boundary. + if (is_inside) { + TrackedPoint newPt; + newPt.id = id; + newPt.pt = _pts[i]; + newPt.pt3d = cv::Point3f(_pts[i].x, _pts[i].y, 0); + newPt.markerRoi = templateRoi; + trackingPointBin[index].push_back(newPt); + id++; + } + } +} + +void TrackingPointSelector::SetHomography(cv::Mat newHomography) +{ + _homography = newHomography; +} + +/// @return 3x3 cv::Mat (of type CV_64FC1, i.e. double) containing the homography. +cv::Mat TrackingPointSelector::GetHomography() +{ + return _homography; +} + +void TrackingPointSelector::UpdatePointStatus(std::vector status) +{ + int index = 0; + for (std::vector::iterator it = _selectedPts.begin(); it != _selectedPts.end(); ++it) { + if (it->tracking) { + it->SetTracking((int)status[index++]); + } + } +} + +void TrackingPointSelector::ResetSelection() +{ + _reset = true; +} + +std::vector TrackingPointSelector::GetInitialFeatures() +{ + if (!_reset) return GetTrackedFeatures(); + _reset = false; + + // Reset state of all points to not selected and not tracking. + _selectedPts.clear(); + for (auto &bin : trackingPointBin) { + for (auto &trackPt : bin.second) { + trackPt.SetSelected(false); + trackPt.SetTracking(false); + } + } + + // Selects a random template from each bin for tracking. + std::vector ret; + for (auto &bin : trackingPointBin) { + size_t pointCount = bin.second.size(); + if (pointCount > 0) { // If there are points in the bin. + // Select a random point from the bin. + int tIndex = pointCount > 1 ? rng.uniform(0, static_cast(bin.second.size())) : 0; + bin.second[tIndex].SetSelected(true); + bin.second[tIndex].SetTracking(true); + _selectedPts.push_back(bin.second[tIndex]); + + ret.push_back(bin.second[tIndex].pt); + } + } + return ret; +} + +std::vector TrackingPointSelector::GetTrackedFeatures() +{ + std::vector selectedPoints; + for (std::vector::iterator it = _selectedPts.begin(); it != _selectedPts.end(); ++it) { + if (it->IsTracking()) { + selectedPoints.push_back(it->pt); + } + } + return selectedPoints; +} + +std::vector TrackingPointSelector::GetTrackedFeatures3d() +{ + std::vector selectedPoints; + for (std::vector::iterator it = _selectedPts.begin(); it != _selectedPts.end(); ++it) { + if (it->IsTracking()) { + selectedPoints.push_back(it->pt3d); + } + } + return selectedPoints; +} + +std::vector TrackingPointSelector::GetTrackedFeaturesWarped() +{ + std::vector selectedPoints = GetTrackedFeatures(); + std::vector warpedPoints; + // cv::perspectiveTransform asserts on an empty input (and requires a valid + // 3x3 homography). This happens when a marker is detected on the very first + // frame, before the tracked-point selection has been populated. Return an + // empty result instead of throwing. + if (selectedPoints.empty() || _homography.empty()) { + return warpedPoints; + } + perspectiveTransform(selectedPoints, warpedPoints, _homography); + return warpedPoints; +} + +std::vector TrackingPointSelector::GetAllFeatures() +{ + std::vector allBinnedPoints; + for (auto &track : trackingPointBin) { + for (auto &trackPt : track.second) { + allBinnedPoints.push_back(trackPt.pt); + } + } + return allBinnedPoints; +} + +void TrackingPointSelector::CleanUp() +{ + _selectedPts.clear(); + _pts.clear(); + trackingPointBin.clear(); + _homography.release(); +} diff --git a/WebARKit/WebARKitTrackers/WebARKitOpticalTracking/WebARKitConfig.cpp b/WebARKit/WebARKitTrackers/WebARKitOpticalTracking/WebARKitConfig.cpp new file mode 100644 index 0000000..1cf6e1e --- /dev/null +++ b/WebARKit/WebARKitTrackers/WebARKitOpticalTracking/WebARKitConfig.cpp @@ -0,0 +1,49 @@ +#include + +extern const double DEFAULT_NN_MATCH_RATIO = 0.7f; +extern const double TEBLID_NN_MATCH_RATIO = 0.8f; +extern const int DEFAULT_MAX_FEATURES = 800; +extern const int TEBLID_MAX_FEATURES = 1000; +extern const int N = 10; +extern const int MIN_NUM_MATCHES = 8; +extern const int minRequiredDetectedFeatures = 50; ///< Minimum number of detected features required to consider a target matched. +extern const int markerTemplateWidth = 15; ///< Width in pixels of image patches used in template matching. +extern const int maxLevel = 3; ///< Maximum number of levels in optical flow image pyramid. +extern const cv::Size winSize(31, 31); +extern const cv::TermCriteria termcrit(cv::TermCriteria::COUNT | cv::TermCriteria::EPS, 20, 0.03); +extern const int searchRadius = 15; +extern const int match_method = cv::TM_SQDIFF_NORMED; +extern const cv::Size featureImageMinSize(640, 480); ///< Minimum size when downscaling incoming images used for feature tracking. +extern const double featureDetectPyramidLevel = + 1.05f; ///> Scale factor applied to image pyramid to determine image to perform feature matching upon. +extern const int featureBorder = 8; +extern const cv::Size blurSize(3, 3); +extern const double ransac_thresh = 2.5f; +extern cv::RNG rng( 0xFFFFFFFF ); +extern const double m_pi = 3.14159265358979323846; +extern const std::string WEBARKIT_HEADER_VERSION_STRING = "0.8.0"; +/*@ + The MAJOR version number defines non-backwards compatible + changes in the ARToolKit API. Range: [0-99]. + */ +extern const int WEBARKIT_HEADER_VERSION_MAJOR = 0; + +/*@ + The MINOR version number defines additions to the ARToolKit + API, or (occsasionally) other significant backwards-compatible + changes in runtime functionality. Range: [0-99]. + */ +extern const int WEBARKIT_HEADER_VERSION_MINOR = 8; + +/*@ + The TINY version number defines bug-fixes to existing + functionality. Range: [0-99]. + */ +extern const int WEBARKIT_HEADER_VERSION_TINY = 0; + +/*@ + The BUILD version number will always be zero in releases, + but may be non-zero in post-release development builds, + version-control repository-sourced code, or other. Range: [0-99]. + */ +extern const int WEBARKIT_HEADER_VERSION_DEV = 0; diff --git a/WebARKit/WebARKitTrackers/WebARKitOpticalTracking/WebARKitHomographyInfo.cpp b/WebARKit/WebARKitTrackers/WebARKitOpticalTracking/WebARKitHomographyInfo.cpp new file mode 100644 index 0000000..7d856fa --- /dev/null +++ b/WebARKit/WebARKitTrackers/WebARKitOpticalTracking/WebARKitHomographyInfo.cpp @@ -0,0 +1,20 @@ +#include + +namespace webarkit { +namespace homography { + +WebARKitHomographyInfo::WebARKitHomographyInfo() { validHomography = false; } + +WebARKitHomographyInfo::WebARKitHomographyInfo(cv::Mat hom, std::vector newStatus, + std::vector matches) { + homography = hom; + status = newStatus; + inlier_matches = matches; + if (matches.size() > 4) { + validHomography = true; + } +} + +} // namespace homography + +} // namespace webarkit diff --git a/WebARKit/WebARKitTrackers/WebARKitOpticalTracking/WebARKitTracker.cpp b/WebARKit/WebARKitTrackers/WebARKitOpticalTracking/WebARKitTracker.cpp new file mode 100644 index 0000000..64d5168 --- /dev/null +++ b/WebARKit/WebARKitTrackers/WebARKitOpticalTracking/WebARKitTracker.cpp @@ -0,0 +1,882 @@ +#include +#include +#include +#include +#include + + +namespace webarkit { + +class WebARKitTracker::WebARKitTrackerImpl { + public: + bool _trackVizActive; + TrackerVisualization _trackViz; + + WebARKitTrackerImpl() + : corners(4), initialized(false), output(17, 0.0), _valid(false), _maxNumberOfMarkersToTrack(1), + _currentlyTrackedMarkers(0), _frameCount(0), _frameSizeX(0), + _frameSizeY(0), + _isDetected(false), _isTracking(false), _centerOrigin(false), + _featureDetectPyrLevel(0), _featureDetectScaleFactor(cv::Vec2f(1.0f, 1.0f)), numMatches(0), + minNumMatches(MIN_NUM_MATCHES), _nn_match_ratio(0.7f), _trackVizActive(false), + _trackViz(TrackerVisualization()) { + m_camMatrix = cv::Matx33d::zeros(); + m_distortionCoeff = cv::Mat::zeros(4, 1, cv::DataType::type); + }; + + ~WebARKitTrackerImpl() = default; + + void initialize(webarkit::TRACKER_TYPE trackerType, int frameWidth, int frameHeight) { + _frameSizeX = frameWidth; + _frameSizeY = frameHeight; + + // WebARKitLib#44: feature detection runs on a pyrDown'd copy of the frame + // (the "detectionFrame" in processFrame). _featureDetectPyrLevel is how many + // pyrDown steps bring the frame down to >= featureImageMinSize; matched frame + // keypoints are scaled back up by _featureDetectScaleFactor. For frames at or + // below featureImageMinSize (e.g. 640x480) the level is 0 -> no downsampling, + // factor 1.0. The reference image is detected at full resolution (initTracker); + // only the live frame is downsampled (matches ArtoolkitX OCVT). + double xmin_log2 = std::log2(static_cast(featureImageMinSize.width)); + double ymin_log2 = std::log2(static_cast(featureImageMinSize.height)); + _featureDetectPyrLevel = static_cast( + std::min(std::floor(std::log2(static_cast(_frameSizeX)) - xmin_log2), + std::floor(std::log2(static_cast(_frameSizeY)) - ymin_log2))); + if (_featureDetectPyrLevel < 0) _featureDetectPyrLevel = 0; + // Exact scale factor, computed the same way cv::pyrDown rounds ((x+1)/2 per + // level) so the keypoint rescale is sub-pixel accurate. + int xScaled = _frameSizeX; + int yScaled = _frameSizeY; + for (int i = 1; i <= _featureDetectPyrLevel; i++) { + xScaled = (xScaled + 1) / 2; + yScaled = (yScaled + 1) / 2; + _featureDetectScaleFactor = cv::Vec2f((float)_frameSizeX / (float)xScaled, + (float)_frameSizeY / (float)yScaled); + } + WEBARKIT_LOGi("Feature detect pyramid level: %d (scale factor %.3f, %.3f)\n", + (int)_featureDetectPyrLevel, _featureDetectScaleFactor[0], _featureDetectScaleFactor[1]); + + setDetectorType(trackerType); + if (trackerType == webarkit::TEBLID_TRACKER) { + _nn_match_ratio = TEBLID_NN_MATCH_RATIO; + } else if (trackerType == webarkit::AKAZE_TRACKER) { + _nn_match_ratio = DEFAULT_NN_MATCH_RATIO; + minNumMatches = 40; + } else { + _nn_match_ratio = DEFAULT_NN_MATCH_RATIO; + minNumMatches = 15; + } + WEBARKIT_LOGd("Min Num Matches: %d\n", minNumMatches); + _camera->setupCamera(frameWidth, frameHeight); + _camera->printSettings(); + + std::array camData = _camera->getCameraData(); + for (auto i = 0; i < 3; i++) { + for (auto j = 0; j < 3; j++) { + m_camMatrix(i, j) = camData[i * 3 + j]; + } + } + + for (auto i = 0; i < 3; i++) { + for (auto j = 0; j < 3; j++) { + WEBARKIT_LOGi("Camera Matrix: %.2f\n", m_camMatrix(i, j)); + } + } + + for (auto i = 0; i < 4; i++) { + WEBARKIT_LOGi("Distortion coefficients: %.2f\n", m_distortionCoeff.at(i, 0)); + } + + webarkit::cameraProjectionMatrix(camData, 0.1, 1000.0, frameWidth, frameHeight, m_cameraProjectionMatrix); + + for (auto i = 0; i < 16; i++) { + + WEBARKIT_LOGi("Camera Proj Matrix: %.2f\n", m_cameraProjectionMatrix[i]); + + } + + //1.9102363924347978, 0, 0, 0, 0, 2.5377457054523322, 0, 0, -0.013943280545895442, -0.005830389685211879, -1.0000002000000199, -1, 0, 0, -0.00020000002000000202, 0 + + _pyramid.clear(); + _prevPyramid.clear(); + _currentlyTrackedMarkers = 0; + } + + template void initTracker(T refData, size_t refCols, size_t refRows, ColorSpace colorSpace) { + WEBARKIT_LOGi("Init Tracker!\n"); + + cv::Mat refGray = convert2Grayscale(refData, refCols, refRows, colorSpace); + refGray.copyTo(_image); + + cv::Mat trackerFeatureMask = createTrackerFeatureMask(refGray); + + if (!extractFeatures(refGray, trackerFeatureMask, refKeyPts, refDescr)) { + WEBARKIT_LOGe("No features detected!\n"); + return; + }; + + // Normalized dimensions : + const float maxSize = std::max(refCols, refRows); + const float unitW = refCols / maxSize; + const float unitH = refRows / maxSize; + + _pattern.size = cv::Size(refCols, refRows); + + WEBARKIT_LOGd("WebARKitPattern size ready!\n"); + + _pattern.points2d.push_back(cv::Point2f(0, 0)); + _pattern.points2d.push_back(cv::Point2f(refCols, 0)); + _pattern.points2d.push_back(cv::Point2f(refCols, refRows)); + _pattern.points2d.push_back(cv::Point2f(0, refRows)); + + WEBARKIT_LOGd("WebARKitPattern points2d ready!\n"); + + _pattern.points3d.push_back(cv::Point3f(-unitW, -unitH, 0)); + _pattern.points3d.push_back(cv::Point3f(unitW, -unitH, 0)); + _pattern.points3d.push_back(cv::Point3f(unitW, unitH, 0)); + _pattern.points3d.push_back(cv::Point3f(-unitW, unitH, 0)); + + WEBARKIT_LOGd("WebARKitPattern points3d ready!\n"); + + corners[0] = cvPoint(0, 0); + corners[1] = cvPoint(refCols, 0); + corners[2] = cvPoint(refCols, refRows); + corners[3] = cvPoint(0, refRows); + + _bBox.push_back(cv::Point2f(0, 0)); + _bBox.push_back(cv::Point2f(refCols, 0)); + _bBox.push_back(cv::Point2f(refCols, refRows)); + _bBox.push_back(cv::Point2f(0, refRows)); + + _trackSelection = TrackingPointSelector(Points(refKeyPts), refCols, refRows, markerTemplateWidth); + + initialized = true; + + WEBARKIT_LOGi("Tracker ready!\n"); + } + + bool extractFeatures(const cv::Mat& grayImage, cv::Mat& featureMask, std::vector& keypoints, + cv::Mat& descriptors) const { + assert(!grayImage.empty()); + assert(grayImage.channels() == 1); + + this->_featureDetector->detect(grayImage, keypoints, featureMask); + WEBARKIT_LOGd("keypoints size: %d\n", keypoints.size()); + if (keypoints.empty()) { + WEBARKIT_LOGe("No keypoints detected!\n"); + return false; + } + this->_featureDescriptor->compute(grayImage, keypoints, descriptors); + WEBARKIT_LOGd("descriptors size: %d\n", descriptors.size()); + if (descriptors.empty()) { + WEBARKIT_LOGe("No descriptors computed!\n"); + return false; + } + return true; + } + + void processFrameData(uchar* frameData, size_t frameCols, size_t frameRows, ColorSpace colorSpace, + BLUR_TYPE blurType) { + cv::Mat grayFrame = convert2Grayscale(frameData, frameCols, frameRows, colorSpace); + if (blurType == BLUR_TYPE::BOX_BLUR) { + cv::blur(grayFrame, grayFrame, blurSize); + } + else if (blurType == BLUR_TYPE::MEDIAN_BLUR) { + cv::medianBlur(grayFrame, grayFrame, blurSize.width); + } + processFrame(grayFrame); + grayFrame.release(); + }; + + std::vector getOutputData() { return output; }; + + cv::Mat getPoseMatrixCV() { return _patternTrackingInfo.pose3d; }; + + float* getPoseMatrixGL() { return (float*)_patternTrackingInfo.trans; } + + + cv::Mat getGLViewMatrix() { return _patternTrackingInfo.glViewMatrix; }; + + std::array getCameraProjectionMatrix() { return m_cameraProjectionMatrix; }; + + bool isValid() { return _valid; }; + + // WebARKitLib#38: when true, the pose origin is the marker centre instead of + // the reference image's top-left corner. Default false (ArtoolkitX parity). + void setOriginCentered(bool centered) { _centerOrigin = centered; }; + + protected: + bool RunTemplateMatching(cv::Mat frame, int trackableId) { + // std::cout << "Starting template match" << std::endl; + std::vector finalTemplatePoints, finalTemplateMatchPoints; + // Get a handle on the corresponding points from current image and the marker + std::vector trackablePoints = _trackSelection.GetTrackedFeatures(); + std::vector trackablePointsWarped = _trackSelection.GetTrackedFeaturesWarped(); + // Create an empty result image - May be able to pre-initialize this container + + int n = (int)trackablePointsWarped.size(); + if (_trackVizActive) { + _trackViz.templateMatching = {}; + _trackViz.templateMatching.templateMatchingCandidateCount = n; + } + + for (int j = 0; j < n; j++) { + auto pt = trackablePointsWarped[j]; + if (cv::pointPolygonTest(_bBoxTransformed, trackablePointsWarped[j], true) > 0) { + auto ptOrig = trackablePoints[j]; + + cv::Rect templateRoi = GetTemplateRoi(pt); + cv::Rect frameROI(0, 0, frame.cols, frame.rows); + if (IsRoiValidForFrame(frameROI, templateRoi)) { + cv::Rect markerRoi(0, 0, _image.cols, _image.rows); + + std::vector vertexPoints = GetVerticesFromPoint(ptOrig); + std::vector vertexPointsResults; + perspectiveTransform(vertexPoints, vertexPointsResults, _trackSelection.GetHomography()); + + cv::Rect srcBoundingBox = cv::boundingRect(cv::Mat(vertexPointsResults)); + + vertexPoints.clear(); + vertexPoints = GetVerticesFromTopCorner(srcBoundingBox.x, srcBoundingBox.y, srcBoundingBox.width, + srcBoundingBox.height); + perspectiveTransform(vertexPoints, vertexPointsResults, _trackSelection.GetHomography().inv()); + + std::vector testVertexPoints = FloorVertexPoints(vertexPointsResults); + std::vector finalWarpPoints = + GetVerticesFromTopCorner(0, 0, srcBoundingBox.width, srcBoundingBox.height); + cv::Mat templateHomography = + findHomography(testVertexPoints, finalWarpPoints, cv::RANSAC, ransac_thresh); + + if (!templateHomography.empty()) { + cv::Rect templateBoundingBox = cv::boundingRect(cv::Mat(vertexPointsResults)); + cv::Rect searchROI = InflateRoi(templateRoi, searchRadius); + if (IsRoiValidForFrame(frameROI, searchROI)) { + searchROI = searchROI & frameROI; + templateBoundingBox = templateBoundingBox & markerRoi; + + if (templateBoundingBox.area() > 0 && searchROI.area() > templateBoundingBox.area()) { + cv::Mat searchImage = frame(searchROI); + cv::Mat templateImage = _image(templateBoundingBox); + cv::Mat warpedTemplate; + + warpPerspective(templateImage, warpedTemplate, templateHomography, + srcBoundingBox.size()); + cv::Mat matchResult = MatchTemplateToImage(searchImage, warpedTemplate); + + if (!matchResult.empty()) { + double minVal; + double maxVal; + cv::Point minLoc, maxLoc, matchLoc; + minMaxLoc(matchResult, &minVal, &maxVal, &minLoc, &maxLoc, cv::Mat()); + if (minVal < 0.5) { + matchLoc = minLoc; + matchLoc.x += searchROI.x + (warpedTemplate.cols / 2); + matchLoc.y += searchROI.y + (warpedTemplate.rows / 2); + finalTemplatePoints.push_back(ptOrig); + finalTemplateMatchPoints.push_back(matchLoc); + } else { + if (_trackVizActive) + _trackViz.templateMatching.failedTemplateMinimumCorrelationCount++; + } + } else { + if (_trackVizActive) + _trackViz.templateMatching.failedTemplateMatchCount++; + } + } else { + if (_trackVizActive) + _trackViz.templateMatching.failedTemplateBigEnoughTestCount++; + } + } else { + if (_trackVizActive) + _trackViz.templateMatching.failedSearchROIInFrameTestCount++; + } + } else { + if (_trackVizActive) + _trackViz.templateMatching.failedGotHomogTestCount++; + } + } else { + if (_trackVizActive) + _trackViz.templateMatching.failedROIInFrameTestCount++; + } + } else { + if (_trackVizActive) + _trackViz.templateMatching.failedBoundsTestCount++; + } + } + bool gotHomography = updateTrackableHomography(trackableId, finalTemplatePoints, finalTemplateMatchPoints); + if (!gotHomography) { + _isTracking = false; + _isDetected = false; + // WebARKitLib#46: template matching is the appearance check -- if it fails + // (the tracked region no longer looks like the marker, e.g. the marker + // left the frame and optical flow drifted onto static background), the + // marker is lost. Clear _valid too so isValid() turns false and the + // tracker reports 'not found'. runOpticalFlow's failure path already does + // this; ArtoolkitX has no _valid and treats !isDetected && !isTracking as + // "not visible" (IsTrackableVisible). + this->_valid = false; + _currentlyTrackedMarkers--; + } + if (_trackVizActive) { + _trackViz.templateMatching.templateMatchingOK = gotHomography; + } + return gotHomography; + } + + void processFrame(cv::Mat& frame) { + buildImagePyramid(frame); + + if (!initialized) { + WEBARKIT_LOGe("Reference image not found. AR is unintialized!\n"); + assert(initialized == false); + } + + if (_trackVizActive) { + memset(_trackViz.bounds, 0, 8 * sizeof(float)); + _trackViz.opticalFlowTrackablePoints.clear(); + _trackViz.opticalFlowTrackedPoints.clear(); + _trackViz.opticalFlowOK = false; + } + + WEBARKIT_LOGd("Reset Tracking!\n"); + + clear_output(); + + _isDetected = false; + + cv::Mat frameDescr; + std::vector frameKeyPts; + + // WebARKitLib#44 part B: skip feature detection while the marker is already + // being tracked. Detection (extractFeatures + descriptor matching) is the + // dominant per-frame cost; once optical flow has a lock (_isTracking), it + // maintains the pose and re-detection is unnecessary. When optical flow or + // template matching loses the marker, _isTracking is cleared (see runOpticalFlow + // / RunTemplateMatching) and detection resumes on the next frame to re-acquire. + // + // The guard is on _isTracking, NOT on _currentlyTrackedMarkers < + // _maxNumberOfMarkersToTrack: _isDetected is reset to false every frame (above), + // and on the first detection frame optical flow is skipped (_frameCount == 0), so + // the marker is detected but not yet "tracking". A counter-based guard would then + // skip both detection AND optical flow on the next frame and freeze (this bites + // the static-image example, which feeds the same frame repeatedly). Gating on + // _isTracking re-detects until optical flow actually holds a lock. + if (!_isTracking) { + // WebARKitLib#44 part A: detect features on a pyrDown'd copy of the frame for + // performance on large frames. matched keypoints are scaled back to full-frame + // coordinates in MatchFeatures via _featureDetectScaleFactor. For level 0 + // (frame <= featureImageMinSize, e.g. 640x480) detectionFrame == frame, so the + // path is identical to full-res detection. + cv::Mat detectionFrame; + if (_featureDetectPyrLevel < 1) { + detectionFrame = frame; + } else { + cv::Mat srcFrame = frame; + for (int pyrLevel = 1; pyrLevel <= _featureDetectPyrLevel; pyrLevel++) { + cv::pyrDown(srcFrame, detectionFrame, cv::Size(0, 0)); + srcFrame = detectionFrame; + } + } + + cv::Mat featureMask = createFeatureMask(detectionFrame); + + if (!extractFeatures(detectionFrame, featureMask, frameKeyPts, frameDescr)) { + WEBARKIT_LOGe("No features detected in extractFeatures!\n"); + } + WEBARKIT_LOGd("frame KeyPoints size: %d\n", frameKeyPts.size()); + if (static_cast(frameKeyPts.size()) > minRequiredDetectedFeatures) { + MatchFeatures(frameKeyPts, frameDescr); + } + } + int i = 0; + // WebARKitLib#46: also run optical flow while tracking, not only on a fresh + // detection. When the marker leaves the frame, MatchFeatures fails + // (_isDetected = false) but _isTracking is still true; running optical flow + // here lets it fail on the now-absent marker and clear _isTracking/_valid + // (via updateTrackableHomography), so isValid() turns false and the tracker + // reports 'not found' instead of staying stuck on a stale pose. It also + // bridges momentary detection dropouts with a fresh pose while the marker is + // still present. + if (_isDetected || _isTracking) { + WEBARKIT_LOGd("Start tracking!\n"); + if (_frameCount > 0 && _prevPyramid.size() > 0) { + // if (_prevPyramid.size() > 0) { + // std::cout << "Starting Optical Flow" << std::endl; + std::vector trackablePoints = _trackSelection.GetInitialFeatures(); + std::vector trackablePointsWarped = _trackSelection.GetTrackedFeaturesWarped(); + if (!runOpticalFlow(i, trackablePoints, trackablePointsWarped)) { + WEBARKIT_LOGd("Optical flow failed.\n"); + } else { + if (_trackVizActive) { _trackViz.opticalFlowOK = true;} + // Refine optical flow with template match. + if (!RunTemplateMatching(frame, i)) { + WEBARKIT_LOGd("Template matching failed."); + } + // std::cout << "Optical flow ok." << std::endl; + } + } + } + + if (_isDetected || _isTracking) { + cv::Mat _pose; + std::vector imgPoints = _trackSelection.GetTrackedFeaturesWarped(); + std::vector objPoints = _trackSelection.GetTrackedFeatures3d(); + // WebARKitLib#38 (webarkit-testing#38): optional centered origin. When + // enabled, shift the 3D object points so the pose origin is the marker + // centre instead of the reference image's top-left corner -- content at + // (0,0,0) then sits in the middle of the marker. Offsetting the object + // points moves only the solved translation; the rotation is unchanged. + // Applied to the solvePnP points ONLY -- the 2D imgPoints and the + // matching/template/homography paths stay in raw image space. Default + // off (ArtoolkitX parity). See docs/design-center-origin-option.md. + if (_centerOrigin) { + const float cx = _pattern.size.width * 0.5f; + const float cy = _pattern.size.height * 0.5f; + for (auto& p : objPoints) { p.x -= cx; p.y -= cy; } + } + // Need at least 4 correspondences for solvePnP, and the two sets must + // match in size. On the very first frame a marker can be detected + // before the tracked-point selection is populated (the optical-flow + // branch that calls GetInitialFeatures() is gated on _frameCount > 0), + // leaving these empty. Skip pose estimation in that case. + if (imgPoints.size() >= 4 && imgPoints.size() == objPoints.size()) { + _patternTrackingInfo.cameraPoseFromPoints(_pose, objPoints, imgPoints, m_camMatrix, m_distortionCoeff); + _patternTrackingInfo.getTrackablePose(_pose); + _patternTrackingInfo.updateTrackable(); + _patternTrackingInfo.computeGLviewMatrix(_pose); + fill_output(m_H); + WEBARKIT_LOGi("Marker tracked ! Num. matches : %d\n", numMatches); + } + } + + swapImagePyramid(); + _frameCount++; + } + + void fill_output(cv::Mat& H) { + output[0] = H.at(0, 0); + output[1] = H.at(0, 1); + output[2] = H.at(0, 2); + output[3] = H.at(1, 0); + output[4] = H.at(1, 1); + output[5] = H.at(1, 2); + output[6] = H.at(2, 0); + output[7] = H.at(2, 1); + output[8] = H.at(2, 2); + + output[9] = _bBoxTransformed[0].x; + output[10] = _bBoxTransformed[0].y; + output[11] = _bBoxTransformed[1].x; + output[12] = _bBoxTransformed[1].y; + output[13] = _bBoxTransformed[2].x; + output[14] = _bBoxTransformed[2].y; + output[15] = _bBoxTransformed[3].x; + output[16] = _bBoxTransformed[3].y; + }; + + void clear_output() { std::fill(output.begin(), output.end(), 0); }; + + void buildImagePyramid(cv::Mat& frame) { cv::buildOpticalFlowPyramid(frame, _pyramid, winSize, maxLevel); } + + void swapImagePyramid() { _pyramid.swap(_prevPyramid); } + + void MatchFeatures(const std::vector& newFrameFeatures, cv::Mat newFrameDescriptors) { + int maxMatches = 0; + int bestMatchIndex = -1; + std::vector finalMatched1, finalMatched2; + if (!_isDetected) { + std::vector> matches = getMatches(newFrameDescriptors); + numMatches = matches.size(); + WEBARKIT_LOGd("Num Matches: %d\n", numMatches); + + if (matches.size() > minRequiredDetectedFeatures) { + std::vector matched1, matched2; + std::vector status; + int totalGoodMatches = 0; + for (unsigned int j = 0; j < matches.size(); j++) { + // Ratio Test for outlier removal, removes ambiguous matches. + if (matches[j][0].distance < _nn_match_ratio * matches[j][1].distance) { + matched1.push_back(newFrameFeatures[matches[j][0].queryIdx]); + matched2.push_back(refKeyPts[matches[j][0].trainIdx]); + status.push_back(1); + totalGoodMatches++; + } else { + status.push_back(0); + } + } + // Measure goodness of match by most number of matching features. + // This allows for maximum of a single marker to match each time. + // TODO: Would a better metric be percentage of marker features matching? + if (totalGoodMatches > maxMatches) { + finalMatched1 = matched1; + finalMatched2 = matched2; + maxMatches = totalGoodMatches; + // bestMatchIndex = i; + } + } + } + // } // end for cycle + + if (maxMatches > 0) { + // WebARKitLib#44: detection ran on the downsampled detectionFrame, so the + // matched FRAME keypoints (finalMatched1) are in downsampled coordinates -- + // scale them back up to full-frame coordinates before fitting the + // homography. Level 0 => factor 1.0 => no-op. The reference keypoints + // (finalMatched2) stay in reference coordinates. + for (size_t i = 0; i < finalMatched1.size(); i++) { + finalMatched1[i].pt.x *= _featureDetectScaleFactor[0]; + finalMatched1[i].pt.y *= _featureDetectScaleFactor[1]; + } + homography::WebARKitHomographyInfo homoInfo = + getHomographyInliers(Points(finalMatched2), Points(finalMatched1)); + if (homoInfo.validHomography) { + // std::cout << "New marker detected" << std::endl; + _isDetected = true; + // Since we've just detected the marker, make sure next invocation of + // GetInitialFeatures() for this marker makes a new selection. + _trackSelection.ResetSelection(); + _trackSelection.SetHomography(homoInfo.homography); + + // Use the homography to form the initial estimate of the bounding box. + // This will be refined by the optical flow pass. + perspectiveTransform(_bBox, _bBoxTransformed, homoInfo.homography); + if (_trackVizActive) { + for (int i = 0; i < 4; i++) { + _trackViz.bounds[i][0] = _bBoxTransformed[i].x; + _trackViz.bounds[i][1] = _bBoxTransformed[i].y; + } + } + _currentlyTrackedMarkers++; + } + } + } + + bool runOpticalFlow(int trackableId, const std::vector& trackablePoints, + const std::vector& trackablePointsWarped) { + std::vector flowResultPoints, trackablePointsWarpedResult; + std::vector statusFirstPass, statusSecondPass; + std::vector err; + cv::calcOpticalFlowPyrLK(_prevPyramid, _pyramid, trackablePointsWarped, flowResultPoints, statusFirstPass, err, + winSize, maxLevel, termcrit, 0, 0.001); + // By using bi-directional optical flow, we improve quality of detected points. + cv::calcOpticalFlowPyrLK(_pyramid, _prevPyramid, flowResultPoints, trackablePointsWarpedResult, + statusSecondPass, err, winSize, maxLevel, termcrit, 0, 0.001); + + // Keep only the points for which flow was found in both temporal directions. + int killed1 = 0; + std::vector filteredTrackablePoints, filteredTrackedPoints; + for (auto j = 0; j != flowResultPoints.size(); ++j) { + if (!statusFirstPass[j] || !statusSecondPass[j]) { + statusFirstPass[j] = (uchar)0; + killed1++; + continue; + } + filteredTrackablePoints.push_back(trackablePoints[j]); + filteredTrackedPoints.push_back(flowResultPoints[j]); + } + + if (_trackVizActive) { + _trackViz.opticalFlowTrackablePoints = filteredTrackablePoints; + _trackViz.opticalFlowTrackedPoints = filteredTrackedPoints; + } + // std::cout << "Optical flow discarded " << killed1 << " of " << flowResultPoints.size() << " points" << + // std::endl; + + if (!updateTrackableHomography(trackableId, filteredTrackablePoints, filteredTrackedPoints)) { + _isDetected = false; + _isTracking = false; + this->_valid = false; + _currentlyTrackedMarkers--; + return false; + } + + _isTracking = true; + return true; + } + + bool updateTrackableHomography(int trackableId, const std::vector& matchedPoints1, + const std::vector& matchedPoints2) { + if (matchedPoints1.size() > 4) { + homography::WebARKitHomographyInfo homoInfo = getHomographyInliers(matchedPoints1, matchedPoints2); + if (homoInfo.validHomography) { + _trackSelection.UpdatePointStatus(homoInfo.status); + _trackSelection.SetHomography(homoInfo.homography); + m_H = homoInfo.homography; + this->_valid = homoInfo.validHomography; + // Update the bounding box. + perspectiveTransform(_bBox, _bBoxTransformed, homoInfo.homography); + if (_trackVizActive) { + for (int i = 0; i < 4; i++) { + _trackViz.bounds[i][0] = _bBoxTransformed[i].x; + _trackViz.bounds[i][1] = _bBoxTransformed[i].y; + } + } + if (_frameCount > 1) { + _trackSelection.ResetSelection(); + } + return true; + } + } + return false; + } + + std::vector GetVerticesFromPoint(cv::Point ptOrig, int width = markerTemplateWidth, + int height = markerTemplateWidth) { + std::vector vertexPoints; + vertexPoints.push_back(cv::Point2f(ptOrig.x - width / 2, ptOrig.y - height / 2)); + vertexPoints.push_back(cv::Point2f(ptOrig.x + width / 2, ptOrig.y - height / 2)); + vertexPoints.push_back(cv::Point2f(ptOrig.x + width / 2, ptOrig.y + height / 2)); + vertexPoints.push_back(cv::Point2f(ptOrig.x - width / 2, ptOrig.y + height / 2)); + return vertexPoints; + } + + std::vector GetVerticesFromTopCorner(int x, int y, int width, int height) { + std::vector vertexPoints; + vertexPoints.push_back(cv::Point2f(x, y)); + vertexPoints.push_back(cv::Point2f(x + width, y)); + vertexPoints.push_back(cv::Point2f(x + width, y + height)); + vertexPoints.push_back(cv::Point2f(x, y + height)); + return vertexPoints; + } + + cv::Rect GetTemplateRoi(cv::Point2f pt) { + return cv::Rect(pt.x - (markerTemplateWidth / 2), pt.y - (markerTemplateWidth / 2), markerTemplateWidth, + markerTemplateWidth); + } + + cv::Mat MatchTemplateToImage(cv::Mat searchImage, cv::Mat warpedTemplate) { + int result_cols = searchImage.cols - warpedTemplate.cols + 1; + int result_rows = searchImage.rows - warpedTemplate.rows + 1; + if (result_cols > 0 && result_rows > 0) { + cv::Mat result; + result.create(result_rows, result_cols, CV_32FC1); + + double minVal; + double maxVal; + minMaxLoc(warpedTemplate, &minVal, &maxVal, 0, 0, cv::noArray()); + + cv::Mat normSeatchROI; + normalize(searchImage, normSeatchROI, minVal, maxVal, cv::NORM_MINMAX, -1, cv::Mat()); + /// Do the Matching and Normalize + matchTemplate(normSeatchROI, warpedTemplate, result, match_method); + return result; + } else { + // std::cout << "Results image too small" << std::endl; + return cv::Mat(); + } + } + + bool IsRoiValidForFrame(cv::Rect frameRoi, cv::Rect roi) { return (roi & frameRoi) == roi; } + + cv::Rect InflateRoi(cv::Rect roi, int inflationFactor) { + cv::Rect newRoi = roi; + newRoi.x -= inflationFactor; + newRoi.y -= inflationFactor; + newRoi.width += 2 * inflationFactor; + newRoi.height += 2 * inflationFactor; + return newRoi; + } + + std::vector FloorVertexPoints(const std::vector& vertexPoints) { + std::vector testVertexPoints = vertexPoints; + float minX = std::numeric_limits::max(); + float minY = std::numeric_limits::max(); + for (int k = 0; k < testVertexPoints.size(); k++) { + if (testVertexPoints[k].x < minX) { + minX = testVertexPoints[k].x; + } + if (testVertexPoints[k].y < minY) { + minY = testVertexPoints[k].y; + } + } + for (int k = 0; k < testVertexPoints.size(); k++) { + testVertexPoints[k].x -= minX; + testVertexPoints[k].y -= minY; + } + return testVertexPoints; + } + + std::vector> getMatches(const cv::Mat& frameDescr) { + std::vector> knnMatches; + _matcher->knnMatch(frameDescr, refDescr, knnMatches, 2); + return knnMatches; + } + + cv::Mat createTrackerFeatureMask(cv::Mat& frame) { + cv::Mat featureMask; + if (featureMask.empty()) { + // Only create mask if we have something to draw in it. + featureMask = cv::Mat::zeros(frame.size(), frame.type()); + } + cv::Rect innerRegion(featureBorder, featureBorder, frame.cols - (featureBorder * 2), + frame.rows - (featureBorder * 2)); + cv::Mat maskRoi(featureMask, innerRegion); + maskRoi.setTo(cv::Scalar(255)); + return featureMask; + } + + cv::Mat createFeatureMask(cv::Mat& frame) { + cv::Mat featureMask; + if (_isDetected) { + if (featureMask.empty()) { + // Only create mask if we have something to draw in it. + featureMask = cv::Mat::ones(frame.size(), frame.type()); + } + std::vector> contours(1); + // WebARKitLib#44: the mask is built on the downsampled detectionFrame + // (frame.size() here), but _bBoxTransformed is in full-frame coordinates, + // so divide by _featureDetectScaleFactor to bring the tracked-marker + // region into downsampled coordinates matching the mask. Level 0 => + // factor 1.0 => identity. + for (int j = 0; j < 4; j++) { + contours[0].push_back(cv::Point(_bBoxTransformed[j].x / _featureDetectScaleFactor[0], + _bBoxTransformed[j].y / _featureDetectScaleFactor[1])); + } + drawContours(featureMask, contours, 0, cv::Scalar(0), -1, 8); + } + return featureMask; + } + + cv::Mat _image; + + int _currentlyTrackedMarkers; + + int _frameCount; + + int _frameSizeX; + int _frameSizeY; + + bool _valid; + + bool _isDetected; + + bool _isTracking; + + // WebARKitLib#38: pose origin = marker centre when true (default false). + bool _centerOrigin; + + // WebARKitLib#44: detection runs on a pyrDown'd frame; matched keypoints are + // scaled back up by _featureDetectScaleFactor. Level 0 / factor 1 => no downsampling. + int _featureDetectPyrLevel; + cv::Vec2f _featureDetectScaleFactor; + + std::vector corners; + + cv::Mat m_H; + + cv::Mat prevIm; + + int numMatches; + + int minNumMatches; + + std::vector framePts; + + bool initialized; + + WebARKitCamera* _camera = new WebARKitCamera(); + + WebARKitPattern _pattern; + + WebARKitPatternTrackingInfo _patternTrackingInfo; + + TrackingPointSelector _trackSelection; + + cv::Matx33d m_camMatrix; + cv::Mat m_distortionCoeff; + + std::array m_cameraProjectionMatrix; + private: + int _maxNumberOfMarkersToTrack; + + std::vector output; // 9 from homography matrix, 8 from warped corners*/ + + cv::Ptr _featureDetector; + + cv::Ptr _featureDescriptor; + + cv::Ptr _matcher; + + cv::Mat refGray, refDescr; + + std::vector refKeyPts; + + webarkit::TRACKER_TYPE _trackerType; + + double _nn_match_ratio; + + std::vector _pyramid, _prevPyramid; + + std::vector _bBoxTransformed; + + std::vector _bBox; + + void setDetectorType(webarkit::TRACKER_TYPE trackerType) { + _trackerType = trackerType; + if (trackerType == webarkit::TRACKER_TYPE::AKAZE_TRACKER) { + const double akaze_thresh = 3e-4; // AKAZE detection threshold set to locate about 1000 keypoints + cv::Ptr akaze = cv::AKAZE::create(); + akaze->setThreshold(akaze_thresh); + this->_featureDetector = akaze; + this->_featureDescriptor = akaze; + } else if (trackerType == webarkit::TRACKER_TYPE::ORB_TRACKER) { + this->_featureDetector = cv::ORB::create(DEFAULT_MAX_FEATURES); + this->_featureDescriptor = cv::ORB::create(DEFAULT_MAX_FEATURES); + } else if (trackerType == webarkit::TRACKER_TYPE::FREAK_TRACKER) { + this->_featureDetector = cv::ORB::create(DEFAULT_MAX_FEATURES); + this->_featureDescriptor = cv::xfeatures2d::FREAK::create(); + } else if (trackerType == webarkit::TRACKER_TYPE::TEBLID_TRACKER) { + this->_featureDetector = cv::ORB::create(TEBLID_MAX_FEATURES); + this->_featureDescriptor = cv::xfeatures2d::TEBLID::create(1.00f); + } + if (trackerType == webarkit::TRACKER_TYPE::AKAZE_TRACKER) { + _matcher = cv::BFMatcher::create(); + } else { + _matcher = cv::BFMatcher::create(cv::NORM_HAMMING); + } + }; +}; + +WebARKitTracker::WebARKitTracker() : _trackerImpl(new WebARKitTrackerImpl()) {} + +WebARKitTracker::~WebARKitTracker() = default; // destructor + +WebARKitTracker::WebARKitTracker(WebARKitTracker&&) = default; // copy constructor + +WebARKitTracker& WebARKitTracker::operator=(WebARKitTracker&&) = default; // move assignment operator + +void WebARKitTracker::initialize(webarkit::TRACKER_TYPE trackerType, int frameWidth, int frameHeight) { + _trackerImpl->initialize(trackerType, frameWidth, frameHeight); +} + +void WebARKitTracker::initTracker(cv::Mat refData, size_t refCols, size_t refRows, ColorSpace colorSpace) { + _trackerImpl->initTracker(refData, refCols, refRows, colorSpace); +} + +void WebARKitTracker::initTracker(uchar* refData, size_t refCols, size_t refRows, ColorSpace colorSpace) { + _trackerImpl->initTracker(refData, refCols, refRows, colorSpace); +} + +void WebARKitTracker::processFrameData(uchar* frameData, size_t frameCols, size_t frameRows, ColorSpace colorSpace, + BLUR_TYPE blurType) { + _trackerImpl->processFrameData(frameData, frameCols, frameRows, colorSpace, blurType); +} + +std::vector WebARKitTracker::getOutputData() { return _trackerImpl->getOutputData(); } + +cv::Mat WebARKitTracker::getPoseMatrixCV() { return _trackerImpl->getPoseMatrixCV(); } + +float* WebARKitTracker::getPoseMatrixGL() { return _trackerImpl->getPoseMatrixGL(); } + + +cv::Mat WebARKitTracker::getGLViewMatrix() { return _trackerImpl->getGLViewMatrix(); } + +std::array WebARKitTracker::getCameraProjectionMatrix() { + return _trackerImpl->getCameraProjectionMatrix(); +} + +bool WebARKitTracker::isValid() { return _trackerImpl->isValid(); } + +void WebARKitTracker::setOriginCentered(bool centered) { _trackerImpl->setOriginCentered(centered); } + +} // namespace webarkit \ No newline at end of file diff --git a/WebARKit/WebARKitTrackers/WebARKitOpticalTracking/include/WebARKitTrackers/WebARKitOpticalTracking/TrackedPoint.h b/WebARKit/WebARKitTrackers/WebARKitOpticalTracking/include/WebARKitTrackers/WebARKitOpticalTracking/TrackedPoint.h new file mode 100644 index 0000000..8d6965c --- /dev/null +++ b/WebARKit/WebARKitTrackers/WebARKitOpticalTracking/include/WebARKitTrackers/WebARKitOpticalTracking/TrackedPoint.h @@ -0,0 +1,61 @@ +/* + * TrackedPoint.h + * artoolkitX + * + * This file is part of artoolkitX. + * + * artoolkitX is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * artoolkitX is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with artoolkitX. If not, see . + * + * As a special exception, the copyright holders of this library give you + * permission to link this library with independent modules to produce an + * executable, regardless of the license terms of these independent modules, and to + * copy and distribute the resulting executable under terms of your choice, + * provided that you also meet, for each linked independent module, the terms and + * conditions of the license of that module. An independent module is a module + * which is neither derived from nor based on this library. If you modify this + * library, you may extend this exception to your version of the library, but you + * are not obligated to do so. If you do not wish to do so, delete this exception + * statement from your version. + * + * Copyright 2018 Realmax, Inc. + * Copyright 2015 Daqri, LLC. + * Copyright 2010-2015 ARToolworks, Inc. + * + * Author(s): Philip Lamb, Daniel Bell. + * Modified for WebARKit by @kalwalt - Walter Perdan - 2024 + * + */ + +#ifndef TRACKED_POINT_H +#define TRACKED_POINT_H + +#include + +class TrackedPoint +{ +public: + int id; + cv::Point2f pt; + cv::Point3f pt3d; + cv::Rect markerRoi; + bool selected; + bool tracking; + + bool IsTracking(); + void SetTracking(bool newTracking); + bool IsSelected(); + void SetSelected(bool newSelected); +}; + +#endif //TRACKED_POINT diff --git a/WebARKit/WebARKitTrackers/WebARKitOpticalTracking/include/WebARKitTrackers/WebARKitOpticalTracking/TrackerVisualization.h b/WebARKit/WebARKitTrackers/WebARKitOpticalTracking/include/WebARKitTrackers/WebARKitOpticalTracking/TrackerVisualization.h new file mode 100644 index 0000000..84d8800 --- /dev/null +++ b/WebARKit/WebARKitTrackers/WebARKitOpticalTracking/include/WebARKitTrackers/WebARKitOpticalTracking/TrackerVisualization.h @@ -0,0 +1,64 @@ +/* + * TrackerVisualization.h + * artoolkitX + * + * This file is part of artoolkitX. + * + * artoolkitX is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * artoolkitX is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with artoolkitX. If not, see . + * + * As a special exception, the copyright holders of this library give you + * permission to link this library with independent modules to produce an + * executable, regardless of the license terms of these independent modules, and to + * copy and distribute the resulting executable under terms of your choice, + * provided that you also meet, for each linked independent module, the terms and + * conditions of the license of that module. An independent module is a module + * which is neither derived from nor based on this library. If you modify this + * library, you may extend this exception to your version of the library, but you + * are not obligated to do so. If you do not wish to do so, delete this exception + * statement from your version. + * + * Copyright 2024 Eden Networks Ltd. + * + * Author(s): Philip Lamb. + * + */ + +#ifndef TRACKER_VISUALIZATION_H +#define TRACKER_VISUALIZATION_H + +#include + +class TrackerVisualization +{ +public: + int id; + float bounds[4][2]; + std::vector opticalFlowTrackablePoints; + std::vector opticalFlowTrackedPoints; + bool opticalFlowOK; + struct templateMatching { + int templateMatchingCandidateCount; + int failedBoundsTestCount; + int failedROIInFrameTestCount; + int failedGotHomogTestCount; + int failedSearchROIInFrameTestCount; + int failedTemplateBigEnoughTestCount; + int failedTemplateMatchCount; + int failedTemplateMinimumCorrelationCount; + bool templateMatchingOK; + }; + templateMatching templateMatching; +}; + +#endif // TRACKER_VISUALIZATION_H diff --git a/WebARKit/WebARKitTrackers/WebARKitOpticalTracking/include/WebARKitTrackers/WebARKitOpticalTracking/TrackingPointSelector.h b/WebARKit/WebARKitTrackers/WebARKitOpticalTracking/include/WebARKitTrackers/WebARKitOpticalTracking/TrackingPointSelector.h new file mode 100644 index 0000000..c6f0094 --- /dev/null +++ b/WebARKit/WebARKitTrackers/WebARKitOpticalTracking/include/WebARKitTrackers/WebARKitOpticalTracking/TrackingPointSelector.h @@ -0,0 +1,94 @@ +/* + * TrackingPointSelector.h + * artoolkitX + * + * This file is part of artoolkitX. + * + * artoolkitX is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * artoolkitX is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with artoolkitX. If not, see . + * + * As a special exception, the copyright holders of this library give you + * permission to link this library with independent modules to produce an + * executable, regardless of the license terms of these independent modules, and to + * copy and distribute the resulting executable under terms of your choice, + * provided that you also meet, for each linked independent module, the terms and + * conditions of the license of that module. An independent module is a module + * which is neither derived from nor based on this library. If you modify this + * library, you may extend this exception to your version of the library, but you + * are not obligated to do so. If you do not wish to do so, delete this exception + * statement from your version. + * + * Copyright 2018 Realmax, Inc. + * Copyright 2015 Daqri, LLC. + * Copyright 2010-2015 ARToolworks, Inc. + * + * Author(s): Philip Lamb, Daniel Bell. + * Modified for WebARKit by @kalwalt - Walter Perdan - 2024 + * + */ + +#ifndef TRACKINGPOINTSELECTOR_H +#define TRACKINGPOINTSELECTOR_H +#include +#include +#include +#include + +/** + @brief Class used to manage selection of tracking points based on image templates (i.e. unique pixel patches). + */ +class TrackingPointSelector +{ +public: + TrackingPointSelector(); + + TrackingPointSelector(std::vector pts, int width, int height, int markerTemplateWidth); + + void DistributeBins(int width, int height, int markerTemplateWidth); + + void SetHomography(cv::Mat newHomography); + + cv::Mat GetHomography(); + + void UpdatePointStatus(std::vector status); + + /** + @brief Signal that the next call to GetInitialFeatures should return a new selection. + */ + void ResetSelection(); + + /** + @brief If reset, then selects an initial random template from each bin for tracking, + and returns this set. If not reset then returns the same set as GetTrackedFeatures. + */ + std::vector GetInitialFeatures(); + + std::vector GetTrackedFeatures(); + + std::vector GetTrackedFeatures3d(); + + std::vector GetTrackedFeaturesWarped(); + + /// Get all points from all bins that are candidates for selection. + std::vector GetAllFeatures(); + + void CleanUp(); + +private: + bool _reset; + std::vector _pts; + std::map > trackingPointBin; + cv::Mat _homography; + std::vector _selectedPts; +}; +#endif //TRACKINGPOINTSELECTOR diff --git a/WebARKit/WebARKitTrackers/WebARKitOpticalTracking/include/WebARKitTrackers/WebARKitOpticalTracking/WebARKitConfig.h b/WebARKit/WebARKitTrackers/WebARKitOpticalTracking/include/WebARKitTrackers/WebARKitOpticalTracking/WebARKitConfig.h new file mode 100644 index 0000000..d038a64 --- /dev/null +++ b/WebARKit/WebARKitTrackers/WebARKitOpticalTracking/include/WebARKitTrackers/WebARKitOpticalTracking/WebARKitConfig.h @@ -0,0 +1,36 @@ +#ifndef WEBARKIT_CONFIG_H +#define WEBARKIT_CONFIG_H + +#include +#include +#include +#include +#include +#include + +extern const double DEFAULT_NN_MATCH_RATIO; +extern const double TEBLID_NN_MATCH_RATIO; +extern const int DEFAULT_MAX_FEATURES; +extern const int TEBLID_MAX_FEATURES; +extern const int N; +extern const int MIN_NUM_MATCHES; +extern const int minRequiredDetectedFeatures; ///< Minimum number of detected features required to consider a target matched. +extern const int markerTemplateWidth; ///< Width in pixels of image patches used in template matching. +extern const int maxLevel; ///< Maximum number of levels in optical flow image pyramid. +extern const cv::Size winSize; +extern const cv::TermCriteria termcrit; +extern const int searchRadius; +extern const int match_method; +extern const cv::Size featureImageMinSize; ///< Minimum size when downscaling incoming images used for feature tracking. +extern const double featureDetectPyramidLevel; ///> Scale factor applied to image pyramid to determine image to perform feature matching upon. +extern const int featureBorder; +extern const cv::Size blurSize; +extern const double ransac_thresh; +extern cv::RNG rng; +extern const double m_pi; +extern const std::string WEBARKIT_HEADER_VERSION_STRING; +extern const int WEBARKIT_HEADER_VERSION_MAJOR; +extern const int WEBARKIT_HEADER_VERSION_MINOR; +extern const int WEBARKIT_HEADER_VERSION_TINY; +extern const int WEBARKIT_HEADER_VERSION_DEV; +#endif diff --git a/WebARKit/WebARKitTrackers/WebARKitOpticalTracking/include/WebARKitTrackers/WebARKitOpticalTracking/WebARKitEnums.h b/WebARKit/WebARKitTrackers/WebARKitOpticalTracking/include/WebARKitTrackers/WebARKitOpticalTracking/WebARKitEnums.h new file mode 100644 index 0000000..05d759b --- /dev/null +++ b/WebARKit/WebARKitTrackers/WebARKitOpticalTracking/include/WebARKitTrackers/WebARKitOpticalTracking/WebARKitEnums.h @@ -0,0 +1,27 @@ +#ifndef WEBARKIT_ENUMS_H +#define WEBARKIT_ENUMS_H + +namespace webarkit { + +enum TRACKER_TYPE { + AKAZE_TRACKER = 0, + ORB_TRACKER = 1, + FREAK_TRACKER = 2, + TEBLID_TRACKER = 3 +}; + +enum ColorSpace { + RGB = 0, + RGBA = 1, + GRAY = 2 +}; + +enum BLUR_TYPE { + MEDIAN_BLUR = 0, + BOX_BLUR = 1, + NONE_BLUR = 2 +}; + +} // namespace webarkit + +#endif \ No newline at end of file diff --git a/WebARKit/WebARKitTrackers/WebARKitOpticalTracking/include/WebARKitTrackers/WebARKitOpticalTracking/WebARKitHomographyInfo.h b/WebARKit/WebARKitTrackers/WebARKitOpticalTracking/include/WebARKitTrackers/WebARKitOpticalTracking/WebARKitHomographyInfo.h new file mode 100644 index 0000000..7cfe6d4 --- /dev/null +++ b/WebARKit/WebARKitTrackers/WebARKitOpticalTracking/include/WebARKitTrackers/WebARKitOpticalTracking/WebARKitHomographyInfo.h @@ -0,0 +1,24 @@ +#ifndef WEBARKIT_HOMOGRAPHY_INFO_H +#define WEBARKIT_HOMOGRAPHY_INFO_H +#include "WebARKitConfig.h" + +namespace webarkit { +namespace homography { + +class WebARKitHomographyInfo { + public: + WebARKitHomographyInfo(); + + WebARKitHomographyInfo(cv::Mat hom, std::vector newStatus, std::vector matches); + + bool validHomography; + cv::Mat homography; + std::vector status; + std::vector inlier_matches; +}; + +} // namespace homography + +} // namespace webarkit + +#endif \ No newline at end of file diff --git a/WebARKit/WebARKitTrackers/WebARKitOpticalTracking/include/WebARKitTrackers/WebARKitOpticalTracking/WebARKitTracker.h b/WebARKit/WebARKitTrackers/WebARKitOpticalTracking/include/WebARKitTrackers/WebARKitOpticalTracking/WebARKitTracker.h new file mode 100644 index 0000000..a7e4def --- /dev/null +++ b/WebARKit/WebARKitTrackers/WebARKitOpticalTracking/include/WebARKitTrackers/WebARKitOpticalTracking/WebARKitTracker.h @@ -0,0 +1,56 @@ +#ifndef WEBARKIT_TRACKER_H +#define WEBARKIT_TRACKER_H + +#include "WebARKitEnums.h" +#include +#include +#include +#include +#include +#include + +namespace webarkit { + +class WebARKitTracker { + public: + WebARKitTracker(); + + ~WebARKitTracker(); + + WebARKitTracker(WebARKitTracker&&); + + WebARKitTracker& operator=(WebARKitTracker&&); + + void initialize(webarkit::TRACKER_TYPE trackerType, int frameWidth, int frameHeight); + + void initTracker(cv::Mat refData, size_t refCols, size_t refRows, ColorSpace colorSpace); + + void initTracker(uchar* refData, size_t refCols, size_t refRows, ColorSpace colorSpace); + + void processFrameData(uchar* frameData, size_t frameCols, size_t frameRows, ColorSpace colorSpace, BLUR_TYPE blurType); + + std::vector getOutputData(); + + cv::Mat getPoseMatrixCV(); + + float* getPoseMatrixGL(); + + + cv::Mat getGLViewMatrix(); + + std::array getCameraProjectionMatrix(); + + bool isValid(); + + // WebARKitLib#38: pose origin = marker centre when true (default false). + void setOriginCentered(bool centered); + + private: + class WebARKitTrackerImpl; + + std::shared_ptr _trackerImpl; +}; + +} // namespace webarkit + +#endif \ No newline at end of file diff --git a/WebARKit/WebARKitTrackers/WebARKitOpticalTracking/include/WebARKitTrackers/WebARKitOpticalTracking/WebARKitUtils.h b/WebARKit/WebARKitTrackers/WebARKitOpticalTracking/include/WebARKitTrackers/WebARKitOpticalTracking/WebARKitUtils.h new file mode 100644 index 0000000..8aa2588 --- /dev/null +++ b/WebARKit/WebARKitTrackers/WebARKitOpticalTracking/include/WebARKitTrackers/WebARKitOpticalTracking/WebARKitUtils.h @@ -0,0 +1,183 @@ +#ifndef WEBARKIT_UTILS_H +#define WEBARKIT_UTILS_H + +// #include +#include +#include +#include + +namespace webarkit { + +static std::vector Points(std::vector keypoints) { + std::vector res; + for (unsigned i = 0; i < keypoints.size(); i++) { + res.push_back(keypoints[i].pt); + } + return res; +} + +/// Method for calculating and validating a homography matrix from a set of corresponding points. +/// pts1 and pts must have the same dimensionality. +/// @returns An WebARKitHomographyInfo instance, with its status vector of the same dimensionality as the pts1 and pts2 +/// vectors. +static homography::WebARKitHomographyInfo getHomographyInliers(std::vector pts1, + std::vector pts2) { + if (pts1.size() < 4) { + return homography::WebARKitHomographyInfo(); + } + + cv::Mat inlier_mask, homography; + homography = findHomography(pts1, pts2, cv::RANSAC, ransac_thresh, inlier_mask); + if (homography.empty()) { + // Failed to find a homography. + return homography::WebARKitHomographyInfo(); + } + + const double det = homography.at(0, 0) * homography.at(1, 1) - + homography.at(1, 0) * homography.at(0, 1); + if (det < 0) { + return homography::WebARKitHomographyInfo(); + } + + const double N1 = sqrt(homography.at(0, 0) * homography.at(0, 0) + + homography.at(1, 0) * homography.at(1, 0)); + if (N1 > 4 || N1 < 0.1) { + return homography::WebARKitHomographyInfo(); + } + + const double N2 = sqrt(homography.at(0, 1) * homography.at(0, 1) + + homography.at(1, 1) * homography.at(1, 1)); + if (N2 > 4 || N2 < 0.1) { + return homography::WebARKitHomographyInfo(); + } + + const double N3 = sqrt(homography.at(2, 0) * homography.at(2, 0) + + homography.at(2, 1) * homography.at(2, 1)); + if (N3 > 0.002) { + return homography::WebARKitHomographyInfo(); + } + + std::vector status; + std::vector inlier_matches; + int linliers = 0; + for (int i = 0; i < pts1.size(); i++) { + if ((int)inlier_mask.at(i, 0) == 1) { + status.push_back((uchar)1); + inlier_matches.push_back(cv::DMatch(i, i, 0)); + linliers++; + } else { + status.push_back((uchar)0); + } + } + // Return homography and corresponding inlier point sets + return homography::WebARKitHomographyInfo(homography, status, inlier_matches); +} + +static auto convert2Grayscale(cv::Mat& refData, size_t refCols, size_t refRows, ColorSpace colorSpace) { + cv::Mat refGray; + + switch (colorSpace) { + case ColorSpace::RGBA: { + refGray.create(refRows, refCols, CV_8UC1); + cv::cvtColor(refData, refGray, cv::COLOR_RGBA2GRAY); + } break; + case ColorSpace::RGB: { + refGray.create(refRows, refCols, CV_8UC1); + cv::cvtColor(refData, refGray, cv::COLOR_RGB2GRAY); + } break; + case ColorSpace::GRAY: { + refGray = refData; + } break; + default: { + refGray = refData; + } + } + + return refGray; +} + +static auto convert2Grayscale(uchar* refData, size_t refCols, size_t refRows, ColorSpace colorSpace) { + cv::Mat refGray; + + switch (colorSpace) { + case ColorSpace::RGBA: { + cv::Mat colorFrame(refRows, refCols, CV_8UC4, refData); + refGray.create(refRows, refCols, CV_8UC1); + cv::cvtColor(colorFrame, refGray, cv::COLOR_RGBA2GRAY); + WEBARKIT_LOGd("convert to GRAY from RGBA !!\n"); + } break; + case ColorSpace::RGB: { + cv::Mat colorFrame(refRows, refCols, CV_8UC3, refData); + refGray.create(refRows, refCols, CV_8UC1); + cv::cvtColor(colorFrame, refGray, cv::COLOR_RGB2GRAY); + WEBARKIT_LOGd("convert to GRAY from RGB !!\n"); + } break; + case ColorSpace::GRAY: { + refGray = cv::Mat(refRows, refCols, CV_8UC1, refData); + WEBARKIT_LOGd("no need to convert to GRAY!!\n"); + } break; + default: { + refGray = cv::Mat(refRows, refCols, CV_8UC1, refData); + WEBARKIT_LOGd("Default: no need to convert to GRAY!!\n"); + } + } + + return refGray; +} + +static cv::Mat grayscale(uchar data[], size_t cols, size_t rows, ColorSpace colorType) { + int cn; + switch (colorType) { + case ColorSpace::RGBA: + cn = 4; + break; + case ColorSpace::RGB: + cn = 3; + break; + case ColorSpace::GRAY: + + std::cout << "Grayscale input is not allowed with grayscale !" << std::endl; + + break; + default: + cn = 4; + } + auto size = cols * rows; + auto q = 0; + std::vector gray; + uchar r; + uchar g; + uchar b; + for (auto p = 0; p < size; p++) { + r = data[q + 0], g = data[q + 1], b = data[q + 2]; + // https://stackoverflow.com/a/596241/5843642 + gray.push_back((r + r + r + b + g + g + g + g) >> 3); + q += cn; + } + return cv::Mat(cols, rows, CV_8UC1, gray.data()); +} + +std::string inline webarkitGetVersion() { return WEBARKIT_HEADER_VERSION_STRING; } + +unsigned int inline webarkitGetVersion(char** versionStringRef) { + std::string version = WEBARKIT_HEADER_VERSION_STRING; + + if (versionStringRef) { + *versionStringRef = const_cast(version.data()); + } + + // Represent full version number (major, minor, tiny, build) in + // binary coded decimal. N.B: Integer division. + return (0x10000000u * ((unsigned int)WEBARKIT_HEADER_VERSION_MAJOR / 10u) + + 0x01000000u * ((unsigned int)WEBARKIT_HEADER_VERSION_MAJOR % 10u) + + 0x00100000u * ((unsigned int)WEBARKIT_HEADER_VERSION_MINOR / 10u) + + 0x00010000u * ((unsigned int)WEBARKIT_HEADER_VERSION_MINOR % 10u) + + 0x00001000u * ((unsigned int)WEBARKIT_HEADER_VERSION_TINY / 10u) + + 0x00000100u * ((unsigned int)WEBARKIT_HEADER_VERSION_TINY % 10u) + + 0x00000010u * ((unsigned int)WEBARKIT_HEADER_VERSION_DEV / 10u) + + 0x00000001u * ((unsigned int)WEBARKIT_HEADER_VERSION_DEV % 10u)); +} + +} // namespace webarkit + +#endif // WEBARKIT_UTILS_H \ No newline at end of file diff --git a/WebARKit/include/WebARKitCamera.h b/WebARKit/include/WebARKitCamera.h new file mode 100644 index 0000000..33b65f6 --- /dev/null +++ b/WebARKit/include/WebARKitCamera.h @@ -0,0 +1,33 @@ +#ifndef WEBARKITCAMERA_H +#define WEBARKITCAMERA_H + +#include + +namespace webarkit { +class WebARKitCamera { + public: + WebARKitCamera(); + ~WebARKitCamera(); + + bool setupCamera(int width, int height); + + void printSettings(); + + std::array getCameraData() const; + + std::array getDistortionCoefficients() const; + + double getFocalLength() const { return focal_length; } + + private: + int xsize, ysize; + std::array cmat; + std::array kc; + double focal_length; + double diagonal_fov_degrees; + + void setFocalLength(int width, int height); +}; +} // namespace webarkit + +#endif // WEBARKITCAMERA_H \ No newline at end of file diff --git a/WebARKit/include/WebARKitGL.h b/WebARKit/include/WebARKitGL.h new file mode 100644 index 0000000..dd1f0d7 --- /dev/null +++ b/WebARKit/include/WebARKitGL.h @@ -0,0 +1,15 @@ +#ifndef WEBARKIT_GL_H +#define WEBARKIT_GL_H + +#include + +namespace webarkit { + +void arglCameraViewRHf(float para[3][4], float m_modelview[16], const float scale); + +void arglCameraViewRHf(cv::Mat para, std::array& m_modelview, const double scale); + +void cameraProjectionMatrix(const std::array& calibration, double nearPlane, double farPlane, int screenWidth, int screenHeight, std::array& projectionMatrix); +} // namespace webarkit + +#endif // WEBARKITGL_H \ No newline at end of file diff --git a/WebARKit/include/WebARKitLog.h b/WebARKit/include/WebARKitLog.h new file mode 100644 index 0000000..fb25dd7 --- /dev/null +++ b/WebARKit/include/WebARKitLog.h @@ -0,0 +1,161 @@ +/* + * WebARKitLog.h + * WebARKit + * + * This file is part of WebARKit. + * + * WebARKit is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * WebARKit is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with WebARKit. If not, see . + * + * As a special exception, the copyright holders of this library give you + * permission to link this library with independent modules to produce an + * executable, regardless of the license terms of these independent modules, and to + * copy and distribute the resulting executable under terms of your choice, + * provided that you also meet, for each linked independent module, the terms and + * conditions of the license of that module. An independent module is a module + * which is neither derived from nor based on this library. If you modify this + * library, you may extend this exception to your version of the library, but you + * are not obligated to do so. If you do not wish to do so, delete this exception + * statement from your version. + * + * Copyright 2015-2016 Daqri, LLC. + * Copyright 2003-2015 ARToolworks, Inc. + * Copyright 2023 WebARKit. + * + * Author(s): Hirokazu Kato, Philip Lamb, Walter Perdan + * + * This code was taken from Artoolkit5 https://github.com/artoolkitx/artoolkit5 + * with small modifications to adapt to existing WebARKIt code + * + */ + +/*! + @file log.h + @brief Logging utilities. + @details + Various routines to format and redirect log output. + @Copyright 2015-2017 Daqri, LLC. + */ + +#ifndef WEBARKIT_LOG_H +#define WEBARKIT_LOG_H + +#include +#include +#include +#include +#ifndef _WIN32 // errno is defined in stdlib.h on Windows. +# ifdef EMSCRIPTEN // errno is not in sys/ +# include +# else +# include +# endif +#endif +#ifdef __ANDROID__ +# include +#endif +#ifdef _WIN32 +# define ARUTIL_CALLBACK __stdcall +#else +# define ARUTIL_CALLBACK +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +enum { + WEBARKIT_LOG_LEVEL_DEBUG = 0, + WEBARKIT_LOG_LEVEL_INFO, + WEBARKIT_LOG_LEVEL_WARN, + WEBARKIT_LOG_LEVEL_ERROR, + WEBARKIT_LOG_LEVEL_REL_INFO +}; +#define WEBARKIT_LOG_LEVEL_DEFAULT WEBARKIT_LOG_LEVEL_INFO + +/*! + @var int webarkitLogLevel + @brief Sets the severity level. Log messages below the set severity level are not logged. + @details + All calls to WebARKIt's logging facility include a "log level" parameter, which specifies + the severity of the log message. (The severities are defined in <WebARKitLog.h>.) + Setting this global allows for filtering of log messages. All log messages lower than + the set level will not be logged by webarkitLog(). + Note that debug log messages created using the WEBARKIT_LOGd() macro will be logged only in + debug builds, irrespective of the log level. + @see webarkitLog +*/ +extern int webarkitLogLevel; + +/*! + @brief Write a string to the current logging facility. + @details + The default logging facility varies by platform, but on Unix-like platforms is typically + the standard error file descriptor. However, logging may be redirected to some other + facility by webarkitLogSetLogger. + + Newlines are not automatically appended to log output. + @param tag A tag to supply to an OS-specific logging function to specify the source + of the error message. May be NULL, in which case "libwebarkit" will be used. + @param logLevel The severity of the log message. Defined in %lt;WebARKitLog.h>. + Log output is written to the logging facility provided the logLevel meets or + exceeds the minimum level specified in global webarkitLogLevel. + @param format Log format string, in the form of printf(). + @see webarkitLogLevel + @see webarkitLogSetLogger +*/ + +void webarkitLog(const char *tag, const int logLevel, const char *format, ...); +void webarkitLogv(const char *tag, const int logLevel, const char *format, va_list ap); + +typedef void (ARUTIL_CALLBACK *WEBARKIT_LOG_LOGGER_CALLBACK)(const char *logMessage); + +/*! + @brief Divert logging to a callback, or revert to default logging. + @details + The default logging facility varies by platform, but on Unix-like platforms is typically + the standard error file descriptor. However, logging may be redirected to some other + facility by this function. + @param callback The function which will be called with the log output, or NULL to + cancel redirection. + @param callBackOnlyIfOnSameThread If non-zero, then the callback will only be called + if the call to webarkitLog is made on the same thread as the thread which called this function, + and if the webarkitLog call is made on a different thread, log output will be buffered until + the next call to webarkitLog on the original thread. + + The purpose of this is to prevent logging from secondary threads in cases where the + callback model of the target platform precludes this. + @see webarkitLog +*/ +void webarkitLogSetLogger(WEBARKIT_LOG_LOGGER_CALLBACK callback, int callBackOnlyIfOnSameThread); + +#ifdef WEBARKIT_DEBUG +# define WEBARKIT_LOGd(...) webarkitLog(NULL, WEBARKIT_LOG_LEVEL_DEBUG, __VA_ARGS__) +#else +# define WEBARKIT_LOGd(...) +#endif +#define WEBARKIT_LOGi(...) webarkitLog(NULL, WEBARKIT_LOG_LEVEL_INFO, __VA_ARGS__) +#define WEBARKIT_LOGw(...) webarkitLog(NULL, WEBARKIT_LOG_LEVEL_WARN, __VA_ARGS__) +#define WEBARKIT_LOGe(...) webarkitLog(NULL, WEBARKIT_LOG_LEVEL_ERROR, __VA_ARGS__) +#define WEBARKIT_LOGperror(s) webarkitLog(NULL, WEBARKIT_LOG_LEVEL_ERROR, ((s != NULL) ? "%s: %s\n" : "%s%s\n"), ((s != NULL) ? s : ""), strerror(errno)) + +#ifdef __ANDROID__ +# define WEBARKIT_LOG(...) __android_log_print(ANDROID_LOG_INFO, "WebARKit", __VA_ARGS__) +#else +# define WEBARKIT_LOG(...) printf(__VA_ARGS__) +#endif + +#ifdef __cplusplus +} +#endif +#endif //#ifndef WEBARKIT_LOG_H diff --git a/WebARKit/include/WebARKitManager.h b/WebARKit/include/WebARKitManager.h new file mode 100644 index 0000000..4d5669c --- /dev/null +++ b/WebARKit/include/WebARKitManager.h @@ -0,0 +1,121 @@ +/* + * WebARKitManager.h + * WebARKit + * + * This file is part of WebARKit. + * + * WebARKit is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * WebARKit is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with WebARKit. If not, see . + * + * As a special exception, the copyright holders of this library give you + * permission to link this library with independent modules to produce an + * executable, regardless of the license terms of these independent modules, and to + * copy and distribute the resulting executable under terms of your choice, + * provided that you also meet, for each linked independent module, the terms and + * conditions of the license of that module. An independent module is a module + * which is neither derived from nor based on this library. If you modify this + * library, you may extend this exception to your version of the library, but you + * are not obligated to do so. If you do not wish to do so, delete this exception + * statement from your version. + * + * Copyright 2023 WebARKit. + * + * Author(s): Walter Perdan + * + * + */ + +#ifndef WEBARKIT_MANAGER_H +#define WEBARKIT_MANAGER_H + +#include +#include +#include +#include + +namespace webarkit{ + +class WebARKitManager { + private: + typedef enum { + NOTHING_INITIALISED, ///< No initialisation yet and no resources allocated. + BASE_INITIALISED, ///< Trackable management initialised, trackables can be added. + WAITING_FOR_VIDEO, ///< Waiting for video source to become ready. + DETECTION_RUNNING ///< Video running, additional initialisation occurred, tracking running. + } WebARKitState; + + WebARKitState state; ///< Current state of operation, progress through initialisation + std::string versionString; + std::shared_ptr m_tracker; + webarkit::TRACKER_TYPE m_trackerType; + + public: + /** + * Constructor. + */ + WebARKitManager(); + + /** + * Destructor. + */ + ~WebARKitManager(); + + /** + * Returns a string containing the WebARKit version, such as "1.0.0". + * @return The WebARKit version as a std::string + */ + std::string getWebARKitVersion(); + + /** + * Start trackable management so trackables can be added and removed. + * @return true if initialisation was OK, false if an error occured. + */ + bool initialiseBase(webarkit::TRACKER_TYPE trackerType, int frameWidth, int frameHeight); + + /** + * Return the current tracker object. + * @return the WebARKitTracker object. + */ + std::shared_ptr getTracker() { return m_tracker; }; + + bool initTracker(cv::Mat refData, size_t refCols, size_t refRows, ColorSpace colorSpace); + + bool initTracker(uchar* refData, size_t refCols, size_t refRows, ColorSpace colorSpace); + + void setLogLevel(int logLevel); + + bool shutdown(); + + void processFrameData(uchar* frameData, size_t frameCols, size_t frameRows, ColorSpace colorSpace, BLUR_TYPE blurType); + + std::vector getOutputData(); + + cv::Mat getPoseMatrixCV(); + + float* getPoseMatrixGL(); + + cv::Mat getGLViewMatrix(); + + std::array getTransformationMatrix(); + + std::array getCameraProjectionMatrix(); + + bool isValid(); + + /// WebARKitLib#38: pose origin = marker centre when true (default false). + void setOriginCentered(bool centered); +}; + +} // namespace webarkit + +#endif // WEBARKIT_MANAGER_H \ No newline at end of file diff --git a/WebARKit/include/WebARKitPattern.h b/WebARKit/include/WebARKitPattern.h new file mode 100644 index 0000000..632fa5e --- /dev/null +++ b/WebARKit/include/WebARKitPattern.h @@ -0,0 +1,48 @@ +#ifndef WEBARKITPATTERN_H +#define WEBARKITPATTERN_H + +#include + +struct WebARKitPattern { + cv::Size size; + + // cv::Mat grayImg; + + // std::vector keypoints; + // cv::Mat descriptors; + + std::vector points2d; + std::vector points3d; +}; + +/** + * Intermediate pattern tracking info structure + */ +class WebARKitPatternTrackingInfo { + public: + WebARKitPatternTrackingInfo(); + + cv::Mat homography; + std::vector points2d; + cv::Mat pose3d; + float transMat [3][4]; + float trans [3][4]; + cv::Mat glViewMatrix; + + void setScale(const float scale) { m_scale = scale; } + + float getScale() { return m_scale; } + + void cameraPoseFromPoints(cv::Mat& pose, const std::vector& objPts, const std::vector& imgPts, const cv::Matx33f& caMatrix, const cv::Mat& distCoeffs); + + void getTrackablePose(cv::Mat& pose); + + void updateTrackable(); + + void computeGLviewMatrix(cv::Mat &pose); + + private: + float m_scale; +}; + +#endif // WEBARKITPATTERN_H \ No newline at end of file diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt new file mode 100644 index 0000000..503a7fb --- /dev/null +++ b/tests/CMakeLists.txt @@ -0,0 +1,78 @@ +cmake_minimum_required(VERSION 3.22) +project(Webarkit_tests) + +# GoogleTest requires at least C++14 +set(CMAKE_CXX_STANDARD 14) + +if(POLICY CMP0135) + cmake_policy(SET CMP0026 NEW) +endif() + +# Fetch googletest v1.13.0 commit b796f7d44681514f58a683a3a71ff17c94edb0c1 +include(FetchContent) +FetchContent_Declare( + googletest + URL https://github.com/google/googletest/archive/b796f7d44681514f58a683a3a71ff17c94edb0c1.zip +) +# For Windows: Prevent overriding the parent project's compiler/linker settings +set(gtest_force_shared_crt ON CACHE BOOL "" FORCE) + +FetchContent_Declare( + build_opencv + URL https://github.com/webarkit/opencv-em/releases/download/0.1.6/opencv-4.10.0.zip +) + +FetchContent_MakeAvailable(googletest build_opencv) + +enable_testing() + +add_subdirectory(../WebARKit ../WebARKit/build) + +add_executable( + webarkit_test + webarkit_test.cc +) + +get_filename_component(PARENT_DIR ../ ABSOLUTE) + +target_include_directories(webarkit_test PRIVATE "${PARENT_DIR}/WebARKit/include") +target_include_directories(webarkit_test PRIVATE "${PARENT_DIR}/WebARKit/WebARKitTrackers/WebARKitOpticalTracking/include") + +target_include_directories(webarkit_test PRIVATE + "${build_opencv_SOURCE_DIR}" + "${build_opencv_SOURCE_DIR}/3rdparty/include/libjpeg" + "${build_opencv_SOURCE_DIR}/libs/opencv/include" + "${build_opencv_SOURCE_DIR}/libs/opencv/modules/calib3d/include" + "${build_opencv_SOURCE_DIR}/libs/opencv/modules/core/include" + "${build_opencv_SOURCE_DIR}/libs/opencv/modules/features2d/include" + "${build_opencv_SOURCE_DIR}/libs/opencv/modules/flann/include" + "${build_opencv_SOURCE_DIR}/libs/opencv/modules/imgcodecs/include" + "${build_opencv_SOURCE_DIR}/libs/opencv/modules/imgproc/include" + "${build_opencv_SOURCE_DIR}/libs/opencv/modules/video/include" + "${build_opencv_SOURCE_DIR}/libs/opencv_contrib/modules/xfeatures2d/include" + ) + +set(webarkit_test +${PARENT_DIR}/WebARKit/WebARKitManager.cpp +) + +# The order of the libs make the difference! + +target_link_libraries( + webarkit_test + WebARKitLib + "${build_opencv_SOURCE_DIR}/lib/libopencv_calib3d.a" + "${build_opencv_SOURCE_DIR}/lib/libopencv_features2d.a" + "${build_opencv_SOURCE_DIR}/lib/libopencv_flann.a" + "${build_opencv_SOURCE_DIR}/lib/libopencv_imgcodecs.a" + "${build_opencv_SOURCE_DIR}/lib/libopencv_video.a" + "${build_opencv_SOURCE_DIR}/lib/libopencv_xfeatures2d.a" + "${build_opencv_SOURCE_DIR}/lib/libopencv_imgproc.a" + "${build_opencv_SOURCE_DIR}/lib/libopencv_core.a" + "${build_opencv_SOURCE_DIR}/3rdparty/lib/liblibopenjp2.a" + "${build_opencv_SOURCE_DIR}/3rdparty/lib/libzlib.a" + GTest::gtest_main +) + +include(GoogleTest) +gtest_discover_tests(webarkit_test) \ No newline at end of file diff --git a/tests/pinball.jpg b/tests/pinball.jpg new file mode 100644 index 0000000..d1444ad Binary files /dev/null and b/tests/pinball.jpg differ diff --git a/tests/webarkit_test.cc b/tests/webarkit_test.cc new file mode 100644 index 0000000..7ac2964 --- /dev/null +++ b/tests/webarkit_test.cc @@ -0,0 +1,220 @@ +#include +#include +#include +#include +#include + +class WebARKitEnumTest : public testing::TestWithParam> {}; + + +TEST_P(WebARKitEnumTest, TestEnumValues) { + webarkit::TRACKER_TYPE tracker_value = std::get<0>(GetParam()); + webarkit::ColorSpace color_value = std::get<1>(GetParam()); + EXPECT_TRUE(tracker_value == webarkit::TRACKER_TYPE::AKAZE_TRACKER || + tracker_value == webarkit::TRACKER_TYPE::ORB_TRACKER || + tracker_value == webarkit::TRACKER_TYPE::FREAK_TRACKER || + tracker_value == webarkit::TRACKER_TYPE::TEBLID_TRACKER); + EXPECT_TRUE(color_value == webarkit::ColorSpace::RGB || + color_value == webarkit::ColorSpace::RGBA || + color_value == webarkit::ColorSpace::GRAY); +} + +INSTANTIATE_TEST_SUITE_P(WebARKitEnumTestSuite, WebARKitEnumTest, + testing::Combine(testing::ValuesIn({webarkit::TRACKER_TYPE::AKAZE_TRACKER, + webarkit::TRACKER_TYPE::ORB_TRACKER, + webarkit::TRACKER_TYPE::FREAK_TRACKER, + webarkit::TRACKER_TYPE::TEBLID_TRACKER}), + testing::ValuesIn({webarkit::ColorSpace::RGB, + webarkit::ColorSpace::RGBA, + webarkit::ColorSpace::GRAY}))); + +TEST(WebARKitConfigTest, TestConfigValues) { + EXPECT_EQ(DEFAULT_NN_MATCH_RATIO, 0.7f); + EXPECT_EQ(TEBLID_NN_MATCH_RATIO, 0.8f); + EXPECT_EQ(DEFAULT_MAX_FEATURES, 800); + EXPECT_EQ(TEBLID_MAX_FEATURES, 1000); + EXPECT_EQ(N, 10); + EXPECT_EQ(MIN_NUM_MATCHES, 8); + EXPECT_EQ(maxLevel, 3); + EXPECT_EQ(featureDetectPyramidLevel, 1.05f); + EXPECT_EQ(featureBorder, 8); + EXPECT_EQ(WEBARKIT_HEADER_VERSION_STRING, "0.8.0"); +} + +TEST(WebARKitConfigTest, TestWinSize) { + cv::Size expected_size(31, 31); + EXPECT_EQ(expected_size.width, winSize.width); + EXPECT_EQ(expected_size.height, winSize.height); +} + +TEST(WebARKitConfigTest, TestTermCriteria) { + int expected_type = cv::TermCriteria::COUNT | cv::TermCriteria::EPS; + int expected_max_count = 20; + double expected_epsilon = 0.03; + EXPECT_EQ(expected_type, termcrit.type); + EXPECT_EQ(expected_max_count, termcrit.maxCount); + EXPECT_EQ(expected_epsilon, termcrit.epsilon); +} + +TEST(WebARKitConfigTest, TestBlurSize) { + cv::Size expected_blur_size(3, 3); + EXPECT_EQ(expected_blur_size.width, blurSize.width); + EXPECT_EQ(expected_blur_size.height, blurSize.height); +} + +TEST(WebARKitConfigTest, TestPIConstant) { + double internal_m_pi = 3.14159265358979323846; + EXPECT_EQ(internal_m_pi, m_pi); +} + +TEST(WebARKitCameraTest, TestCamera) { + int width = 640; + int height = 480; + webarkit::WebARKitCamera camera; + EXPECT_TRUE(camera.setupCamera(width, height)); + std::array camera_mat = camera.getCameraData(); + EXPECT_EQ(camera_mat[0], 571.25920269684582); + EXPECT_EQ(camera_mat[2], 320.0); + EXPECT_EQ(camera_mat[4], 571.25920269684582); + EXPECT_EQ(camera_mat[5], 240.0); + EXPECT_EQ(camera_mat[8], 1.0); + EXPECT_EQ(camera.getFocalLength(), 571.25920269684582); + camera.printSettings(); +} + +TEST(WebARKitGLTest, TestCameraProjectionMatrix) { + int width = 640; + int height = 480; + webarkit::WebARKitCamera camera; + camera.setupCamera(width, height); + std::array camera_mat = camera.getCameraData(); + std::array projectionMatrix = {0.0}; + webarkit::cameraProjectionMatrix(camera_mat, 0.01, 100.0, width, height, projectionMatrix); + EXPECT_EQ(projectionMatrix[0], 1.7851850084276433); + EXPECT_EQ(projectionMatrix[5], 2.3802466779035241); + EXPECT_EQ(projectionMatrix[10], -1.0002000200020003); + EXPECT_EQ(projectionMatrix[11], -1.0); + EXPECT_EQ(projectionMatrix[14], -0.020002000200020003); +} + +// Check WebARKitManager initialisation. +TEST(WebARKitTest, InitialiseBaseAkazeTest) { + // Create a WebARKitManager object + webarkit::WebARKitManager manager; + // Check if the WebARKitManager initialisation is successful + EXPECT_TRUE(manager.initialiseBase(webarkit::TRACKER_TYPE::AKAZE_TRACKER, 640, 480)); +} + +// Check WebARKitManager initialisation. +TEST(WebARKitTest, InitialiseBaseFreakTest) { + // Create a WebARKitManager object + webarkit::WebARKitManager manager; + // Check if the WebARKitManager initialisation is successful + EXPECT_TRUE(manager.initialiseBase(webarkit::TRACKER_TYPE::FREAK_TRACKER, 640, 480)); +} + +// Check WebARKitManager initialisation. +TEST(WebARKitTest, InitialiseBaseOrbTest) { + // Create a WebARKitManager object + webarkit::WebARKitManager manager; + // Check if the WebARKitManager initialisation is successful + EXPECT_TRUE(manager.initialiseBase(webarkit::TRACKER_TYPE::ORB_TRACKER, 640, 480)); +} + +// Check WebARKitManager initialisation. +TEST(WebARKitTest, InitialiseBaseTeblidTest) { + // Create a WebARKitManager object + webarkit::WebARKitManager manager; + // Check if the WebARKitManager initialisation is successful + EXPECT_TRUE(manager.initialiseBase(webarkit::TRACKER_TYPE::TEBLID_TRACKER, 640, 480)); +} + +// Check WebARKit version +TEST(WebARKitTest, CheckWebARKitVersion) { + // Create a WebARKitManager object + webarkit::WebARKitManager manager; + // Init the manager with the Akaze tracker + manager.initialiseBase(webarkit::TRACKER_TYPE::AKAZE_TRACKER, 640, 480); + // Check if the WebARKit version is correct + EXPECT_STREQ(manager.getWebARKitVersion().c_str(), "0.8.0"); +} + +// Check cameraProjectionMatrix from manager +TEST(WebARKitTest, CheckCameraProjectionMatrix) { + // Create a WebARKitManager object + webarkit::WebARKitManager manager; + // Init the manager with the Akaze tracker + manager.initialiseBase(webarkit::TRACKER_TYPE::AKAZE_TRACKER, 640, 480); + // Check if the cameraProjectionMatrix is correct + std::array camProjectionMatrix = manager.getCameraProjectionMatrix(); + EXPECT_EQ(camProjectionMatrix[0], 1.7851850084276433); + EXPECT_EQ(camProjectionMatrix[5], 2.3802466779035241); + EXPECT_EQ(camProjectionMatrix[10], -1.0002000200020003); + EXPECT_EQ(camProjectionMatrix[11], -1.0); + EXPECT_EQ(camProjectionMatrix[14], -0.20002000200020004); +} + +TEST(WebARKitTest, InitTrackerTest) { + // Create a WebARKitManager object + webarkit::WebARKitManager manager; + // Init the manager with the Akaze tracker + manager.initialiseBase(webarkit::TRACKER_TYPE::AKAZE_TRACKER, 640, 480); + // Load the test image + cv::Mat image = cv::imread("../pinball.jpg", cv::IMREAD_GRAYSCALE); + + if(image.data == NULL) { + std::cout << "Something wrong while reading the image!" << std::endl; + } + + if(image.empty()) { + image = cv::Mat::zeros(2048, 1637, CV_8UC4); + } + + ASSERT_FALSE(image.empty()); + + int width = image.cols; + int height = image.rows; + unsigned char* data = image.data; + EXPECT_EQ(image.cols, 1637); + EXPECT_EQ(image.rows, 2048); + // Check if initTracker returns sucessfully + EXPECT_TRUE(manager.initTracker(data, width, height, webarkit::ColorSpace::GRAY)); +} + +TEST(WebARKitTest, InitTrackerTest2) { + // Create a WebARKitManager object + webarkit::WebARKitManager manager; + // Init the manager with the Akaze tracker + manager.initialiseBase(webarkit::TRACKER_TYPE::AKAZE_TRACKER, 640, 480); + // Load the test image + cv::Mat image = cv::imread("../pinball.jpg", cv::IMREAD_GRAYSCALE); + + if(image.data == NULL) { + std::cout << "Something wrong while reading the image!" << std::endl; + } + + if(image.empty()) { + image = cv::Mat::zeros(2048, 1637, CV_8UC4); + } + + ASSERT_FALSE(image.empty()); + + int width = image.cols; + int height = image.rows; + //unsigned char* data = image.data; + EXPECT_EQ(image.cols, 1637); + EXPECT_EQ(image.rows, 2048); + // Check if initTracker returns sucessfully + EXPECT_TRUE(manager.initTracker(image, width, height, webarkit::ColorSpace::GRAY)); +} + + +// Check WebARKit version +TEST(WebARKitTest, CheckShutDown) { + // Create a WebARKitManager object + webarkit::WebARKitManager manager; + // Init the manager with the Akaze tracker + manager.initialiseBase(webarkit::TRACKER_TYPE::AKAZE_TRACKER, 640, 480); + // Check if the WebARKit went down successfully + EXPECT_TRUE(manager.shutdown()); +} \ No newline at end of file