Skip to content

Commit 0cb145a

Browse files
roamzhenroamzhen
committed
(feat)vk: ocr-fix 3dgs camera-jpg
1 parent 4bc5fc5 commit 0cb145a

File tree

24 files changed

+2227
-183
lines changed

24 files changed

+2227
-183
lines changed

‎miniprogram/app.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,7 @@
9797
"pages/ar/hand-detect-3d/hand-detect-3d",
9898
"pages/ar/depth-detect/depth-detect",
9999
"pages/ar/cameraBuffer-detect/cameraBuffer-detect",
100+
"pages/ar/cameraBuffer-jpg/cameraBuffer-jpg",
100101
"pages/ar/ocr-detect/ocr-detect",
101102
"pages/ar/shoe-detect/shoe-detect",
102103
"pages/ar/photo-shoe-detect/photo-shoe-detect",

‎miniprogram/packageAPI/pages/ar/behavior/behavior-ar.js

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,11 @@ module.exports = Behavior({
66
xrScene: undefined, // xr-frame 的场景
77
xrCamera: undefined, // xr-frame 的相机
88
xrFrameReady: undefined, // xr-frame初始化完毕
9+
// WebGL相关
10+
camera: undefined, // 主要相机
911
// ThreeJs 相关变量
1012
gl: undefined, // 全局gl对象
1113
THREE: undefined, // THREE 对象
12-
camera: undefined, // Three相机，主要相机
1314
// 全局 data
1415
data: {
1516
domWidth: 0,

‎miniprogram/packageAPI/pages/ar/cameraBuffer-jpg/cameraBuffer-jpg.js

Lines changed: 225 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,225 @@
1+
import arBehavior from '../behavior/behavior-ar'
2+
import threeBehavior from '../behavior/behavior-three'
3+
4+
// VK 投影矩阵参数定义
5+
const NEAR = 0.01
6+
const FAR = 1000
7+
8+
Component({
9+
behaviors: [arBehavior, threeBehavior],
10+
data: {
11+
theme: 'light',
12+
widthScale: 1, // canvas宽度缩放值
13+
heightScale: 0.57, // canvas高度缩放值
14+
jpgUrl: '',
15+
},
16+
useLoopLog: false, // 是否开启循环log
17+
imgIndex: 0,
18+
lifetimes: {
19+
/**
20+
* 生命周期函数--监听页面加载
21+
*/
22+
detached() {
23+
console.log("页面detached")
24+
if (wx.offThemeChange) {
25+
wx.offThemeChange()
26+
}
27+
},
28+
ready() {
29+
console.log("页面准备完全")
30+
this.setData({
31+
theme: wx.getSystemInfoSync().theme || 'light'
32+
})
33+
34+
if (wx.onThemeChange) {
35+
wx.onThemeChange(({theme}) => {
36+
this.setData({theme})
37+
})
38+
}
39+
},
40+
},
41+
42+
methods: {
43+
// 对应案例的初始化逻辑，由统一的 behavior 触发
44+
init() {
45+
// 初始化 Three.js，用于模型相关的渲染
46+
this.initTHREE()
47+
48+
// 初始化 GL，基于 Three.js 的 Context，用于相机YUV渲染
49+
this.initYUV()
50+
51+
// 初始化VK
52+
// start完毕后，进行更新渲染循环
53+
this.initVK();
54+
55+
this.useLoopLog = false;
56+
this.imgIndex = 0;
57+
},
58+
initVK() {
59+
// VKSession 配置
60+
const session = this.session = wx.createVKSession({
61+
track: {
62+
plane: {
63+
mode: 1
64+
},
65+
},
66+
version: 'v2',
67+
gl: this.gl
68+
});
69+
70+
session.start(err => {
71+
if (err) return console.error('VK error: ', err)
72+
73+
console.log('@@@@@@@@ VKSession.version', session.version)
74+
75+
// VKSession EVENT resize
76+
session.on('resize', () => {
77+
this.calcCanvasSize();
78+
})
79+
80+
// VKSession EVENT addAnchors
81+
session.on('addAnchors', anchors => {
82+
})
83+
84+
// VKSession EVENT updateAnchors
85+
session.on('updateAnchors', anchors => {
86+
87+
})
88+
89+
// VKSession removeAnchors
90+
// 识别目标丢失时，会触发一次
91+
session.on('removeAnchors', anchors => {
92+
// console.log('removeAnchors', anchors)
93+
});
94+
95+
96+
console.log('ready to initloop')
97+
// start 初始化完毕后，进行更新渲染循环
98+
this.initLoop();
99+
100+
// 绘制双面，以及去掉清屏，用于显示yuv
101+
this.renderer.state.setCullFace(this.THREE.CullFaceNone)
102+
103+
});
104+
},
105+
loop() {
106+
107+
// 获取 VKFrame
108+
const frame = this.session.getVKFrame(this.canvas.width, this.canvas.height)
109+
110+
// 成功获取 VKFrame 才进行
111+
if(!frame) { return; }
112+
113+
// 更新相机 YUV 数据
114+
this.renderYUV(frame)
115+
116+
// 获取 VKCamera
117+
const VKCamera = frame.camera
118+
119+
if (this.useLoopLog) {
120+
// let transformStr = '';
121+
// for (let i = 0; i < 16; i++)
122+
// transformStr += VKCamera.transform[i] + ' ';
123+
// console.log('VKCamera.transform', transformStr);
124+
125+
console.log('timeStamp', frame.timestamp);
126+
127+
128+
let viewMatrixStr = '';
129+
for (let i = 0; i < 16; i++)
130+
viewMatrixStr += VKCamera.viewMatrix[i] + ' ';
131+
// console.log('VKCamera.viewMatrix', viewMatrixStr);
132+
if (this.preTimestamp && this.preTimestamp === frame.timestamp) {
133+
if (this.preViewMatrixStr && this.preViewMatrixStr !== viewMatrixStr) {
134+
console.log('preViewMatrixStr', this.preViewMatrixStr)
135+
console.log('viewMatrixStr', viewMatrixStr)
136+
console.log('Timestamp is same. But viewMatrix is not same');
137+
}
138+
}
139+
this.preTimestamp = frame.timestamp;
140+
this.preViewMatrixStr = viewMatrixStr;
141+
}
142+
143+
},
144+
getJpgImg() {
145+
console.log('Function getJpgImg');
146+
147+
// 按需写入获取 jpg 的 大小 和质量
148+
const width = 640;
149+
const height = 480;
150+
const quality = 90;
151+
152+
// 获取 VKFrame
153+
const frame = this.session.getVKFrame(width, height)
154+
155+
// 成功获取 VKFrame 才进行
156+
if(!frame) { return; }
157+
158+
console.log('getCameraJpgBuffer: ', width, height, quality)
159+
160+
const t1 = new Date().getTime()
161+
const jpgBuffer = frame.getCameraJpgBuffer(width, height, quality);
162+
const t2 = new Date().getTime()
163+
164+
console.log(`getCameraJpgBuffer cost ${t2 - t1}ms`)
165+
166+
// console.log('jpgBuffer', jpgBuffer);
167+
168+
const jpgUrl = this.saveLocalJPG(jpgBuffer, 'cameraJPG');
169+
170+
console.log('jpgUrl', jpgUrl)
171+
172+
this.setData({
173+
'jpgUrl': jpgUrl
174+
})
175+
176+
177+
},
178+
saveLocalJPG(bufferContent, name) {
179+
var url = `${wx.env.USER_DATA_PATH}/${name + this.imgIndex + '.jpg'}`
180+
181+
const fs = wx.getFileSystemManager()
182+
try {
183+
// 存在即删除
184+
const unlinkRes = fs.unlinkSync(url)
185+
// console.log('unlinkSync', unlinkRes)
186+
187+
this.imgIndex++;
188+
const newUrl = `${wx.env.USER_DATA_PATH}/${name + this.imgIndex + '.jpg'}`;
189+
190+
// console.log('write newUrl', newUrl)
191+
// 写入，新图片
192+
const writeRes = fs.writeFileSync(
193+
newUrl,
194+
bufferContent,
195+
'utf8'
196+
)
197+
198+
return newUrl
199+
200+
} catch(e) {
201+
// 利用catch实现，此时，为新写入
202+
try {
203+
// console.log('write url', url)
204+
205+
// 写入
206+
const writeRes = fs.writeFileSync(
207+
url,
208+
bufferContent,
209+
'utf8'
210+
)
211+
}catch(e) {
212+
console.error(e);
213+
}
214+
}
215+
216+
return url
217+
},
218+
getLog() {
219+
console.log('Function getLog');
220+
221+
this.useLoopLog = !this.useLoopLog;
222+
},
223+
224+
},
225+
})

‎miniprogram/packageAPI/pages/ar/cameraBuffer-jpg/cameraBuffer-jpg.json

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
{
2+
"usingComponents": {},
3+
"disableScroll": true,
4+
"renderer": "webview",
5+
"navigationBarTitleText": "相机帧获取jpg图片"
6+
}

‎miniprogram/packageAPI/pages/ar/cameraBuffer-jpg/cameraBuffer-jpg.wxml

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
2+
<view class="page wrap-fix" data-weui-theme="{{theme}}">
3+
<view class="canvas-wrap" style="width: {{widthScale * 100}}%; height: {{heightScale * 100}}%">
4+
<canvas type="webgl" id="canvas"></canvas>
5+
</view>
6+
7+
<view class="hint-words">提示：点击获取，会将摄像头资源转成贴图</view>
8+
9+
<view class="wrap-option">
10+
<view class="row">
11+
<button type="primary" bindtap="getJpgImg">获取jpg图像</button>
12+
<button type="primary" bindtap="getLog">开关调试log</button>
13+
</view>
14+
15+
</view>
16+
17+
<view class="hint-img-wrap">
18+
<view class="hint-img-box">
19+
<image class="hint-img" wx:if="{{jpgUrl !== ''}}" src="{{jpgUrl}}" mode="aspectFit" />
20+
</view>
21+
</view>
22+
23+
24+
</view>
25+

‎miniprogram/packageAPI/pages/ar/cameraBuffer-jpg/cameraBuffer-jpg.wxss

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
.canvas-wrap {
2+
position: relative;
3+
width: 100%;
4+
background-color: #000;
5+
}
6+
7+
.canvas-wrap #canvas {
8+
position: absolute;
9+
left: 0;
10+
top: 0;
11+
width: 100%;
12+
height: 100%;
13+
}
14+
15+
.hint-box {
16+
position: absolute;
17+
left: 0%;
18+
top: 0%;
19+
width: 0;
20+
height: 0;
21+
}
22+
.hint-id {
23+
position: absolute;
24+
left: -6rpx;
25+
right: -6rpx;
26+
bottom: 100%;
27+
color: #fff;
28+
font-size: 20rpx;
29+
line-height: 40rpx;
30+
text-align: center;
31+
text-overflow: ellipsis;
32+
background-color: #07c160;
33+
border-radius: 8rpx;
34+
35+
}
36+
37+
.wrap-fix {
38+
position: absolute;
39+
left: 0;
40+
top: 0;
41+
width: 100%;
42+
height: 100%;
43+
}
44+
45+
.hint-words {
46+
position: absolute;
47+
left: 20rpx;
48+
top: 20rpx;
49+
right: 20rpx;
50+
padding: 10rpx 20rpx;
51+
line-height: 40rpx;
52+
text-align: center;
53+
border-radius: 6rpx;
54+
background-color: rgba(0, 0, 0, .3);
55+
font-size: 26rpx;
56+
color: #fff;
57+
}
58+
59+
60+
.hint-img-wrap {
61+
position: absolute;
62+
left: 25%;
63+
bottom: 0%;
64+
width: 50%;
65+
height: 30%;
66+
}
67+
68+
.hint-img-box {
69+
position: absolute;
70+
left: 5%;
71+
top: 5%;
72+
right: 5%;
73+
bottom: 5%;
74+
background-color: #fafafa;
75+
border: 4rpx solid;
76+
}
77+
78+
.hint-img {
79+
position: absolute;
80+
left: 4%;
81+
top: 4%;
82+
width: 92%;
83+
height: 92%;
84+
}
85+
86+
.wrap-option {
87+
position: absolute;
88+
left: 20rpx;
89+
right: 0;
90+
bottom: 30%;
91+
height: 10%;
92+
flex-direction: column;
93+
display: flex;
94+
}
95+
96+
.wrap-option .row {
97+
flex: 1;
98+
display: flex;
99+
flex-direction: row;
100+
text-align: center;
101+
}
102+
.wrap-option button {
103+
flex: 1;
104+
margin: 5rpx 20rpx 20rpx 0;
105+
font-size: 30rpx;
106+
line-height: 1.2;
107+
}

0 commit comments

Comments
 (0)