Skip to content

Commit b5b73c0

Browse files
1cadumagalhaes1cadumagalhaes
committed
build: build com release
1 parent 6acf0d3 commit b5b73c0

File tree

11 files changed

+456
-109
lines changed

11 files changed

+456
-109
lines changed

‎dist/convert-pdf-to-json.zip‎

138 Bytes
Binary file not shown.

‎dist/extract-events.zip‎

128 Bytes
Binary file not shown.

‎dist/hub-receive-message.zip‎

138 Bytes
Binary file not shown.

‎functions/convert-pdf-to-json/firestore.js‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
const { Firestore } = require('@google-cloud/firestore');
2+
const firestore = new Firestore();
3+
4+
async function get(path) {
5+
try {
6+
const document = firestore.doc(path);
7+
const data = await document.get();
8+
return data.data();
9+
} catch (error) {
10+
console.error(error);
11+
}
12+
}
13+
14+
15+
async function set(path, data) {
16+
try {
17+
const document = firestore.doc(path);
18+
return await document.set(data);
19+
} catch (error) {
20+
console.error(error);
21+
}
22+
}
23+
24+
async function update(path, data) {
25+
try {
26+
const document = firestore.doc(path);
27+
return await document.update(data);
28+
} catch (error) {
29+
console.error(error);
30+
}
31+
}
32+
33+
module.exports = {
34+
get, set, update
35+
};

‎functions/convert-pdf-to-json/hub.js‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
const { PubSub } = require('@google-cloud/pubsub');
2+
const { insertRowsAsStream } = require('./bigquery');
3+
const { get } = require('./firestore');
4+
const schemas = require('./schemas.json');
5+
6+
/**
7+
* Wrapper that sends message to pub/sub
8+
* @param {Object} data - Object to be sent
9+
* @param {String} topic - Pub/Sub topic name
10+
* @returns
11+
*/
12+
async function sendMessage(data, topic) {
13+
const pubSubClient = new PubSub();
14+
try {
15+
const dataBuffer = Buffer.from(JSON.stringify(data));
16+
const messageId = await pubSubClient.topic(topic).publish(dataBuffer);
17+
return messageId;
18+
} catch (error) {
19+
console.error("HUB: Erro ao enviar mensagem para o pub/sub");
20+
return error;
21+
}
22+
}
23+
24+
25+
/**
26+
* Sends a data quality alert to the pub/sub topic with the log format
27+
* @param {Number} code - A http code with the status
28+
* @param {String} message - The message to be sent as alert
29+
*
30+
*/
31+
async function publishAlert({ jobId, code, message, document, page, version }) {
32+
const { topicName } = await get("raft-suite/config");
33+
const deployVersion = process.env['K_REVISION'],
34+
functionName = process.env['K_SERVICE'],
35+
project = process.env.GCLOUD_PROJECT;
36+
37+
let log = {
38+
date: new Date(),
39+
jobId,
40+
project,
41+
module: functionName,
42+
deploy: deployVersion,
43+
version,//: 'beta', //pegar do bq ou datastore ,
44+
document,
45+
page,
46+
status:
47+
{
48+
code,
49+
message
50+
}
51+
}
52+
53+
try {
54+
const messageid = await sendMessage(log, topicName);
55+
console.info(`Message ${messageid} published to the topic`);
56+
} catch (error) {
57+
console.error(error);
58+
}
59+
}
60+
61+
async function receiveMessage(event, context) {
62+
let message = event.data ? Buffer.from(event.data, 'base64').toString() : undefined;
63+
try {
64+
await insertRowsAsStream(message, datasetId, tableId, schemas.hub);
65+
} catch (error) {
66+
console.error(error);
67+
}
68+
}
69+
70+
71+
72+
exports = {
73+
publishAlert,
74+
sendMessage
75+
}
76+
77+
78+
let teste = {
79+
"event": "teste",
80+
"context": "teste"
81+
};

‎functions/convert-pdf-to-json/pdf2text.js‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,16 @@ function getPdfData(path) {
2727

2828

2929
function formatJson(pdfData) {
30-
let json = JSON.parse(decodeURIComponent(JSON.stringify(pdfData, null, 2)));
30+
let string = decodeURIComponent(JSON.stringify(pdfData, null, 2));
31+
const regex = /"T": "(.*)("(.*)")(.*)"/g, replaceText = '"T": "$1\'$3\'$4"',
32+
regexSimple = /"T": "(.*)(")(.*)"/g, replaceTextSimple = '"T": "$1\'$3"';
33+
while (regex.test(string)) {
34+
string = string.replace(regex, replaceText);
35+
}
36+
while (regexSimple.test(string)) {
37+
string = string.replace(regexSimple, replaceTextSimple);
38+
}
39+
let json = JSON.parse(string);
3140
let pages = json['formImage']['Pages'];
3241
pages = pages
3342
.map((page) => {

‎functions/extract-events/extractEvents.js‎

Lines changed: 89 additions & 76 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,10 @@
1-
const config_file = require('./config.json');
1+
const { get } = require('./firestore');
22

33
exports.extractEvents = extractEventsFromJson;
4-
function extractEventsFromJson(content, pageNumber, nomeMapa, config = config_file) {
4+
async function extractEventsFromJson(content, pageNumber, nomeMapa, config = "") {
5+
if (config === "")
6+
config = await get("raft-suite/config");
57
let pages = JSON.parse(content);
6-
78
let groups = groupTexts(pages);
89
let { info, events } = groupEvents(groups, pageNumber, nomeMapa, config);
910
return { info, events };
@@ -45,7 +46,7 @@ function mergeRow(group, limit = 0) {
4546
let copy = [];
4647
let temp = null;
4748
const size = (56.67 - 54.955) / 6;
48-
for (i in group) {
49+
for (var i in group) {
4950
if (temp === null) temp = group[i];
5051
const item = temp;
5152
if (i == group.length - 1) {
@@ -69,82 +70,94 @@ function mergeRow(group, limit = 0) {
6970
}
7071

7172
function groupEvents(groups, pageNumber, nomeMapa, { event, customDimension }) {
72-
let regex = /(V\d+)\s-\s(T\d+)/;
73-
let regexTitle = new RegExp(event.title.join('|'), 'i');
74-
let regexCd = new RegExp(customDimension.title.join('|'), 'i');
75-
let pagina = groups[0][0].text || '';
76-
let infos_mapa = null;
77-
groups.forEach(
78-
(group) => group.forEach(
79-
(item) => {
80-
if (regex.test(item.text)) infos_mapa = item.text
81-
}
73+
try {
74+
let regex = /(V\d+)\s-\s(T\d+)/;
75+
let regexTitle = new RegExp(event.title.join('|'), 'i');
76+
let infos_mapa = null;
77+
groups.forEach(
78+
(group) => group.forEach(
79+
(item) => {
80+
if (regex.test(item.text)) infos_mapa = item.text
81+
}
82+
)
8283
)
83-
)
84-
let [, versao, tela] = infos_mapa != null ? infos_mapa.match(regex)
85-
: [0, 'VX', 'TX'];
86-
87-
let info = {
88-
name: nomeMapa,
89-
pageNumber: pageNumber,
90-
version: versao,
91-
screen: tela,
92-
};
93-
if (versao == 'VX') return { info, events: [] };
94-
95-
let pageviewRegex = new RegExp([event.title[1]], 'i');
96-
97-
let index = groups.findIndex(group => group.length > 1 && (pageviewRegex.test(group[0].text) || /pag/i.test(group[1].text)));
98-
/*var indx = 0;
99-
for (var x of groups) {
100-
if (x[0].text == "Pageview:") {
101-
break;
102-
} else {
103-
indx++
104-
}
105-
}*/
106-
107-
let page = groups.slice(index).sort((a, b) => a[0].x - b[0].x);
108-
page = page.map(group => mergeRow(group));
109-
110-
let events = [],
111-
e = [];
112-
113-
for (item of page) {
114-
if (item.length == event.numParams) {
115-
116-
item = {
117-
key: item[event.key].text,
118-
value: item[event.value].text,
119-
x: item[event.key].x,
120-
y: item[event.key].y,
121-
};
122-
} else if (item.length == customDimension.numParams) {
123-
item = {
124-
key: item[customDimension.key].text,
125-
value: item[customDimension.value].text,
126-
x: item[customDimension.key].x,
127-
y: item[customDimension.key].y,
128-
};
129-
} else continue;
130-
131-
if (regexTitle.test(item.key)) {
84+
let [, versao, tela] = infos_mapa != null ? infos_mapa.match(regex)
85+
: [0, 'VX', 'TX'];
86+
versao = formatDigits(versao);
87+
tela = formatDigits(tela);
88+
let info = {
89+
name: nomeMapa,
90+
pageNumber: pageNumber,
91+
version: versao,
92+
screen: tela,
93+
};
94+
if (versao == 'VX') return { info, events: [] };
95+
96+
let pageviewRegex = new RegExp([event.title[1]], 'i');
97+
groups = groups.map(group => mergeRow(group));
98+
let index = groups.findIndex(group => group.length > 1 && (pageviewRegex.test(group[0].text) || /pag/i.test(group[1].text)));
99+
index = index === -1 ? 0 : index;
100+
let page = groups.slice(index).sort((a, b) => a[0].x - b[0].x);
101+
102+
let events = [],
132103
e = [];
133-
if (page.indexOf(item) != 0) events.push(e);
134-
//else if (events.length == 0) events.push(item);
135-
}
136104

137-
if (customDimension.title.indexOf(item.key) == -1) e.push(item);
105+
for (let item of page) {
106+
if (item.length == event.numParams) {
107+
108+
item = {
109+
key: item[event.key].text,
110+
value: item[event.value].text,
111+
x: item[event.key].x,
112+
y: item[event.key].y,
113+
};
114+
} else if (item.length == customDimension.numParams) {
115+
item = {
116+
key: item[customDimension.key].text,
117+
value: item[customDimension.value].text,
118+
x: item[customDimension.key].x,
119+
y: item[customDimension.key].y,
120+
};
121+
} else continue;
122+
123+
if (regexTitle.test(item.key)) {
124+
e = [];
125+
if (page.indexOf(item) != 0) events.push(e);
126+
//else if (events.length == 0) events.push(item);
127+
}
128+
129+
if (customDimension.title.indexOf(item.key) == -1) e.push(item);
130+
}
131+
events = events
132+
.sort((a, b) => a[0].y - b[0].y)
133+
.map((item) =>
134+
item.map(({ key, value }) => {
135+
return { key, value };
136+
})
137+
);
138+
//console.log("=========",events, "=============");
139+
return { info, events };
140+
} catch (error) {
141+
console.error(error);
142+
publishAlert({
143+
jobId: "",
144+
code: 504,
145+
message: "Erro ao estruturar a lista de eventos.",
146+
document: nomeMapa,
147+
page: pageNumber,
148+
version: '-'
149+
});
138150
}
139-
events = events
140-
.sort((a, b) => a[0].y - b[0].y)
141-
.map((item) =>
142-
item.map(({ key, value }) => {
143-
return { key, value };
144-
})
145-
);
146-
//console.log("=========",events, "=============");
147-
return { info, events };
148151
}
149152

150153
//array = [array[index], ...array.slice(0, index), ...array.slice(index + 1, array.length)]
154+
155+
function formatDigits(text) {
156+
const r = /(\w)(\d{1,2})/;
157+
if (!r.test(text))
158+
return text
159+
let [letter, number] = text.match(r).slice(1);
160+
161+
number = (+number).toLocaleString(undefined, { minimumIntegerDigits: 2 });
162+
return letter + number;
163+
}

‎functions/extract-events/firestore.js‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
const { Firestore } = require('@google-cloud/firestore');
2+
const firestore = new Firestore();
3+
4+
async function get(path) {
5+
try {
6+
const document = firestore.doc(path);
7+
const data = await document.get();
8+
return data.data();
9+
} catch (error) {
10+
console.error(error);
11+
}
12+
}
13+
14+
15+
async function set(path, data) {
16+
try {
17+
const document = firestore.doc(path);
18+
return await document.set(data);
19+
} catch (error) {
20+
console.error(error);
21+
}
22+
}
23+
24+
async function update(path, data) {
25+
try {
26+
const document = firestore.doc(path);
27+
return await document.update(data);
28+
} catch (error) {
29+
console.error(error);
30+
}
31+
}
32+
33+
module.exports = {
34+
get, set, update
35+
};

0 commit comments

Comments
 (0)