Skip to content

Commit c19cb56

Browse files
NeutreeNeutree
committed
add KPU and mobilenet demo, fix LCD bug
1 parent eb16336 commit c19cb56

File tree

17 files changed

+1535
-10
lines changed

17 files changed

+1535
-10
lines changed

‎libraries/Camera/src/Camera.cpp‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,14 +36,22 @@ static const int resolution[][2] = {
3636
{1600, 1200}, /* UXGA */
3737
};
3838

39-
Camera::Camera(framesize_t frameSize = FRAMESIZE_QVGA, pixformat_t pixFormat = PIXFORMAT_RGB565)
39+
Camera::Camera(framesize_t frameSize, pixformat_t pixFormat)
4040
{
4141
_frameSize = frameSize;
4242
_pixFormat = pixFormat;
4343
_width = resolution[frameSize][0];
4444
_height = resolution[frameSize][1];
4545
}
4646

47+
Camera::Camera(int16_t width, uint16_t height, pixformat_t pixFormat)
48+
{
49+
_frameSize = FRAMESIZE_CUSTOM;
50+
_pixFormat = pixFormat;
51+
_width = width;
52+
_height = height;
53+
}
54+
4755
Camera::~Camera()
4856
{
4957

‎libraries/Camera/src/Camera.h‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -44,13 +44,15 @@ typedef enum {
4444
FRAMESIZE_SVGA, // 800x600
4545
FRAMESIZE_SXGA, // 1280x1024
4646
FRAMESIZE_UXGA, // 1600x1200
47+
FRAMESIZE_CUSTOM,
4748
} framesize_t;
4849

4950

5051
class Camera{
5152

5253
public:
5354
Camera(framesize_t frameSize, pixformat_t pixFormat);
55+
Camera(int16_t width, uint16_t height, pixformat_t pixFormat);
5456
~Camera();
5557
virtual bool begin( ) = 0;
5658
virtual void end() = 0;
@@ -64,8 +66,16 @@ class Camera{
6466
* If pixels format is RGB565: return RGB565 pixels with every uint16_t one pixel, e.g. RED: 0xF800
6567
*/
6668
virtual uint8_t* snapshot() = 0;
67-
virtual uint8_t* getRGB565(){ return 0; };
68-
virtual uint8_t* getRGB888(){ return 0; };
69+
/**
70+
* @return pixels with RGB565 format, every uint16_t one pixel, e.g. RED: 0xF800, so two pixels: {0xF800, 0xF800}
71+
*/
72+
virtual uint16_t* getRGB565(){ return nullptr; };
73+
/**
74+
*
75+
* @return pixels with RGB888 format, for n pixels: {{R0,R1,...,Rn-1,},{G0,G1,...,Gn-1},{B0,B1,...,Bn-1}}
76+
* e.g. two RED pixel: {0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00}
77+
*/
78+
virtual uint8_t* getRGB888(){ return nullptr; };
6979
virtual void setRotaion(uint8_t rotation) = 0;
7080
virtual void setInvert(bool invert) = 0;
7181

‎libraries/KPU/examples/mobilenet_v1/MBNet_1000.cpp‎

Lines changed: 208 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,208 @@
1+
/**
2+
* @file MBNet_1000.cpp
3+
* @brief Detect object type
4+
* @author [email protected]
5+
*/
6+
7+
8+
#include "MBNet_1000.h"
9+
#include "names.h"
10+
#include "stdlib.h"
11+
12+
13+
MBNet_1000::MBNet_1000(KPUClass& kpu, Sipeed_ST7789& lcd, Sipeed_OV2640& camera)
14+
:_kpu(kpu),_lcd(lcd), _camera(camera),
15+
_count(0), _result(nullptr)
16+
{
17+
_names = mbnet_label_name;
18+
memset(_statistics, 0, sizeof(_statistics));
19+
}
20+
21+
MBNet_1000::~MBNet_1000()
22+
{
23+
24+
}
25+
26+
27+
int MBNet_1000::begin(const char* kmodel_name)
28+
{
29+
File myFile;
30+
if(!_camera.begin())
31+
return -1;
32+
if(!_lcd.begin(15000000, COLOR_RED))
33+
return -2;
34+
_camera.run(true);
35+
36+
if (!SD.begin())
37+
return -3;
38+
39+
myFile = SD.open(kmodel_name);
40+
if (!myFile)
41+
return -4;
42+
uint32_t fSize = myFile.size();
43+
_lcd.setTextSize(2);
44+
_lcd.setTextColor(COLOR_WHITE);
45+
_lcd.setCursor(100,100);
46+
_lcd.print("Loading ... ");
47+
long ret = myFile.read(_model, fSize);
48+
myFile.close();
49+
if(ret != fSize)
50+
return -5;
51+
52+
if(_kpu.begin(_model) != 0)
53+
return -6;
54+
return 0;
55+
}
56+
57+
int MBNet_1000::detect()
58+
{
59+
uint8_t* img;
60+
uint8_t* img888;
61+
62+
img = _camera.snapshot();
63+
if(img == nullptr || img==0)
64+
return -1;
65+
img888 = _camera.getRGB888();
66+
if(_kpu.forward(img888) != 0)
67+
{
68+
return -2;
69+
}
70+
while( !_kpu.isForwardOk() );
71+
if( _kpu.getResult((uint8_t**)&_result, &_count) != 0)
72+
{
73+
return -3;
74+
}
75+
return 0;
76+
}
77+
78+
void MBNet_1000::show()
79+
{
80+
float prob;
81+
const char* name;
82+
uint8_t i, j;
83+
uint16_t* img;
84+
85+
_count /= sizeof(float);
86+
label_init();
87+
label_sort();
88+
89+
for(j=0; j<STATISTICS_NUM; ++j)
90+
_statistics[j].updated = false;
91+
for ( i = 0; i < 5; i++)
92+
{
93+
label_get(i, &prob, &name);
94+
for(j=0; j<STATISTICS_NUM; ++j)
95+
{
96+
if(_statistics[j].name == NULL)
97+
{
98+
_statistics[j].name = name;
99+
_statistics[j].sum = prob;
100+
_statistics[j].updated = true;
101+
break;
102+
}
103+
else if( _statistics[j].name == name )
104+
{
105+
_statistics[j].sum += prob;
106+
_statistics[j].updated = true;
107+
break;
108+
}
109+
else
110+
{
111+
}
112+
}
113+
if( j == STATISTICS_NUM)
114+
{
115+
float min = _statistics[0].sum;
116+
j = 0;
117+
for(i=1; i<STATISTICS_NUM; ++i)
118+
{
119+
if(_statistics[i].name)
120+
{
121+
if(_statistics[i].sum <= min)
122+
{
123+
min = _statistics[i].sum;
124+
j = i;
125+
}
126+
}
127+
}
128+
_statistics[j].name = name;
129+
_statistics[j].sum = prob;
130+
_statistics[j].updated = true;
131+
}
132+
}
133+
float max = _statistics[0].sum;
134+
float second = 0;
135+
uint8_t index1=0, index2 = 0;
136+
for(i=0; i<STATISTICS_NUM; ++i)
137+
{
138+
if(_statistics[i].name)
139+
{
140+
if(_statistics[i].sum > max)
141+
{
142+
max = _statistics[i].sum;
143+
index1 = i;
144+
}
145+
else if(_statistics[i].sum > second && _statistics[i].sum<max)
146+
{
147+
index2 = i;
148+
}
149+
}
150+
if( !_statistics[i].updated )
151+
{
152+
float tmp = _statistics[i].sum - _statistics[i].sum*2/5;
153+
if( tmp < 0)
154+
tmp = 0;
155+
_statistics[i].sum = tmp;
156+
}
157+
}
158+
img = _camera.getRGB565();
159+
_lcd.fillRect(224,0, _lcd.width()-224, _lcd.height(), COLOR_RED);
160+
_lcd.drawImage(0, 0, _camera.width(), _camera.height(), img);
161+
_lcd.setTextSize(2);
162+
_lcd.setTextColor(COLOR_WHITE);
163+
_lcd.setCursor(0,0);
164+
_lcd.println(_statistics[index1].name);
165+
_lcd.println("-----");
166+
_lcd.println(_statistics[index2].name);
167+
// _lcd.println("=======");
168+
// _lcd.println(_names[_index[0]]);
169+
// _lcd.println(_names[_index[1]]);
170+
// _lcd.println(_names[_index[2]]);
171+
// _lcd.println(_names[_index[3]]);
172+
// _lcd.println(_names[_index[4]]);
173+
}
174+
175+
176+
void MBNet_1000::label_init( )
177+
{
178+
int i;
179+
for(i = 0; i < _count; i++)
180+
_index[i] = i;
181+
}
182+
183+
void MBNet_1000::label_sort(void)
184+
{
185+
int i,j;
186+
float tmp_prob;
187+
uint16_t tmp_index;
188+
for(j=0; j<_count; j++)
189+
for(i=0; i<_count-1-j; i++)
190+
if(_result[i]<_result[i+1])
191+
{
192+
tmp_prob=_result[i];
193+
_result[i]=_result[i+1];
194+
_result[i+1]=tmp_prob;
195+
196+
tmp_index=_index[i];
197+
_index[i]=_index[i+1];
198+
_index[i+1]=tmp_index;
199+
}
200+
}
201+
202+
203+
void MBNet_1000::label_get(uint16_t index, float* prob, const char** name)
204+
{
205+
*prob = _result[index];
206+
*name = _names[_index[index]];
207+
}
208+

‎libraries/KPU/examples/mobilenet_v1/MBNet_1000.h‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
/**
2+
* @file MBNet_1000.h
3+
* @brief Detect object type
4+
* @author [email protected]
5+
*/
6+
7+
8+
#ifndef __MBNET_1000_H
9+
#define __MBNET_1000_H
10+
11+
#include "Sipeed_OV2640.h"
12+
#include "Sipeed_ST7789.h"
13+
#include <SD.h>
14+
#include <KPU.h>
15+
16+
#define KMODEL_SIZE (4220 * 1024)
17+
#define STATISTICS_NUM 5
18+
19+
typedef struct{
20+
const char* name;
21+
float sum;
22+
bool updated;
23+
} statistics_t;
24+
25+
26+
class MBNet_1000{
27+
28+
public:
29+
MBNet_1000(KPUClass& kpu, Sipeed_ST7789& lcd, Sipeed_OV2640& camera);
30+
~MBNet_1000();
31+
int begin(const char* kmodel_name);
32+
int detect();
33+
void show();
34+
const char** _names;
35+
36+
private:
37+
KPUClass& _kpu;
38+
Sipeed_ST7789& _lcd;
39+
Sipeed_OV2640& _camera;
40+
uint8_t _model[KMODEL_SIZE] __attribute__((aligned(128)));
41+
size_t _count;
42+
statistics_t _statistics[STATISTICS_NUM];
43+
float* _result;
44+
uint16_t _index[1000];
45+
46+
void label_init();
47+
void label_get(uint16_t index, float* prob, const char** name);
48+
void label_sort(void);
49+
};
50+
51+
52+
#endif
53+

‎libraries/KPU/examples/mobilenet_v1/mobilenet_v1.ino‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
2+
/**
3+
*
4+
*
5+
* Download model here:
6+
* http://dl.sipeed.com/MAIX/MaixPy/model/mobilenet_0x300000.kfpkg
7+
* Unpack it(zip format), get m.kmodel, change name to a short name "m" for example,
8+
* put it in SD card at root path
9+
*
10+
*
11+
*
12+
*/
13+
14+
15+
#include <Sipeed_OV2640.h>
16+
#include <Sipeed_ST7789.h>
17+
#include "MBNet_1000.h"
18+
19+
20+
SPIClass spi_(SPI0); // MUST be SPI0 for Maix series on board LCD
21+
Sipeed_ST7789 lcd(320, 240, spi_);
22+
Sipeed_OV2640 camera(224, 224, PIXFORMAT_RGB565);
23+
KPUClass KPU;
24+
MBNet_1000 mbnet(KPU, lcd, camera);
25+
26+
const char* kmodel_name = "m";
27+
28+
29+
void setup()
30+
{
31+
Serial.begin(115200);
32+
while (!Serial) {
33+
; // wait for serial port to connect. Needed for native USB port only
34+
}
35+
36+
Serial.println("init mobile net, load kmodel from SD card, it may takes a long time");
37+
if( mbnet.begin(kmodel_name) != 0)
38+
{
39+
Serial.println("mobile net init fail");
40+
while(1);
41+
}
42+
43+
}
44+
45+
void loop()
46+
{
47+
if(mbnet.detect() != 0)
48+
{
49+
Serial.println("detect object fail");
50+
return;
51+
}
52+
mbnet.show();
53+
}
54+

0 commit comments

Comments
 (0)