Skip to content

Commit 60c8feb

Browse files
sadiqkhojaktuite
sadiqkhoja
and
ktuite
authored
new: OData endpoint for entities (getodk#740)
* new: OData endpoint for entities supports , and only * return property data * Change createdBy to creatorId and include in entity odata * Added $select and $filter for entities OData * return empty string for missing properties * fix 406 error for endpoint * fixed test case * remove identity from login function * sanitize property names * correct property value assignment remove duplicate code * pass filter criteria to the count function --------- Co-authored-by: Kathleen Tuite <[email protected]>
1 parent 7279088 commit 60c8feb

File tree

17 files changed

+919
-125
lines changed

17 files changed

+919
-125
lines changed

‎lib/data/entity.js‎

Lines changed: 118 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,13 @@ const { Transform } = require('stream');
1616
const { PartialPipe } = require('../util/stream');
1717
const Problem = require('../util/problem');
1818
const { submissionXmlToFieldStream } = require('./submission');
19+
const { nextUrlFor, getServiceRoot, jsonDataFooter, extractPaging } = require('../util/odata');
20+
const { sanitizeOdataIdentifier } = require('../util/util');
21+
22+
const odataToColumnMap = new Map([
23+
['__system/createdAt', 'entities.createdAt'],
24+
['__system/creatorId', 'entities.creatorId']
25+
]);
1926

2027
////////////////////////////////////////////////////////////////////////////
2128
// ENTITY PARSING
@@ -124,5 +131,115 @@ const streamEntityCsvs = (inStream, properties) => {
124131
return PartialPipe.of(inStream, rootStream, csv());
125132
};
126133

134+
const selectFields = (entity, properties, selectedProperties) => {
135+
const result = {};
136+
137+
// Handle ID properties
138+
if (!selectedProperties || selectedProperties.has('__id')) {
139+
result.__id = entity.uuid;
140+
}
141+
142+
if (!selectedProperties || selectedProperties.has('name')) {
143+
result.name = entity.uuid;
144+
}
145+
146+
if (!selectedProperties || selectedProperties.has('label')) {
147+
result.label = entity.label;
148+
}
149+
150+
// Handle System properties
151+
const systemProperties = {
152+
createdAt: entity.createdAt,
153+
creatorId: entity.aux.creator.id.toString(),
154+
creatorName: entity.aux.creator.displayName
155+
};
156+
157+
if (!selectedProperties || selectedProperties.has('__system')) {
158+
result.__system = systemProperties;
159+
} else {
160+
const selectedSysProp = {};
161+
for (const p of selectedProperties) {
162+
if (p.startsWith('__system')) {
163+
const sysProp = p.replace('__system/', '');
164+
selectedSysProp[sysProp] = systemProperties[sysProp];
165+
}
166+
}
167+
168+
if (Object.keys(selectedSysProp).length > 0) {
169+
result.__system = selectedSysProp;
170+
}
171+
}
172+
173+
// Handle user defined properties
174+
properties.forEach(p => {
175+
if (!selectedProperties || selectedProperties.has(p.name)) {
176+
result[sanitizeOdataIdentifier(p.name)] = entity.def.data[p.name] ?? '';
177+
}
178+
});
179+
180+
return result;
181+
};
182+
183+
// Validates the '$select'ed properties
184+
// Returns a set of properties or null if $select is null or *
185+
// Note: this function doesn't expand __system properties
186+
const extractSelectedProperties = (query, properties) => {
187+
if (query.$select && query.$select !== '*') {
188+
const selectedProperties = query.$select.split(',').map(p => p.trim());
189+
const propertiesSet = new Set(properties.map(p => p.name));
190+
selectedProperties.forEach(p => {
191+
if (p !== '__id' && p !== '__system' && !odataToColumnMap.has(p) && !propertiesSet.has(p)) {
192+
throw Problem.user.propertyNotFound({ property: p });
193+
}
194+
});
195+
return new Set(selectedProperties);
196+
}
127197

128-
module.exports = { parseSubmissionXml, validateEntity, streamEntityCsvs };
198+
return null;
199+
};
200+
201+
// Pagination is done at the database level
202+
const streamEntityOdata = (inStream, properties, domain, originalUrl, query, tableCount) => {
203+
const serviceRoot = getServiceRoot(originalUrl);
204+
const { limit, offset, shouldCount } = extractPaging(query);
205+
const selectedProperties = extractSelectedProperties(query, properties);
206+
207+
let isFirstEntity = true;
208+
const rootStream = new Transform({
209+
writableObjectMode: true, // we take a stream of objects from the db, but
210+
readableObjectMode: false, // we put out a stream of text.
211+
transform(entity, _, done) {
212+
try {
213+
if (isFirstEntity) { // header or fencepost.
214+
this.push('{"value":[');
215+
isFirstEntity = false;
216+
} else {
217+
this.push(',');
218+
}
219+
220+
this.push(JSON.stringify(selectFields(entity, properties, selectedProperties)));
221+
222+
done();
223+
} catch (ex) {
224+
done(ex);
225+
}
226+
}, flush(done) {
227+
this.push((isFirstEntity) ? '{"value":[],' : '],'); // open object or close row array.
228+
229+
// @odata.count and nextUrl.
230+
const nextUrl = nextUrlFor(limit, offset, tableCount, originalUrl);
231+
232+
this.push(jsonDataFooter({ table: 'Entities', domain, serviceRoot, nextUrl, count: (shouldCount ? tableCount.toString() : null) }));
233+
done();
234+
}
235+
});
236+
237+
return PartialPipe.of(inStream, rootStream);
238+
};
239+
240+
module.exports = {
241+
parseSubmissionXml, validateEntity,
242+
streamEntityCsvs, streamEntityOdata,
243+
odataToColumnMap,
244+
extractSelectedProperties, selectFields
245+
};

‎lib/data/odata-filter.js‎

Lines changed: 62 additions & 66 deletions
Original file line numberDiff line numberDiff line change
@@ -13,76 +13,72 @@ const odataParser = require('odata-v4-parser');
1313
const Problem = require('../util/problem');
1414

1515
////////////////////////////////////////
16-
// AST NODE TRANSFORMATION
16+
// MAIN ENTRY POINT
1717

18-
const extractFunctions = [ 'year', 'month', 'day', 'hour', 'minute', 'second' ];
19-
const methodCall = (fn, params) => {
20-
// n.b. odata-v4-parser appears to already validate function name and arity.
21-
const lowerName = fn.toLowerCase();
22-
if (extractFunctions.includes(lowerName))
23-
return sql`extract(${raw(lowerName)} from ${op(params[0])})`; // eslint-disable-line no-use-before-define
24-
else if (fn === 'now')
25-
return sql`now()`;
26-
};
27-
const binaryOp = (left, right, operator) =>
28-
// always use parens to ensure the original AST op precedence.
29-
sql`(${op(left)} ${raw(operator)} ${op(right)})`; // eslint-disable-line no-use-before-define
18+
const odataFilter = (expr, odataToColumnMap) => {
19+
if (expr == null) return sql`true`;
3020

31-
const op = (node) => {
32-
if (node.type === 'FirstMemberExpression') {
33-
if (node.raw === '__system/submissionDate') {
34-
return sql.identifier([ 'submissions', 'createdAt' ]); // TODO: HACK HACK
35-
} else if (node.raw === '__system/updatedAt') {
36-
return sql.identifier([ 'submissions', 'updatedAt' ]); // TODO: HACK HACK
37-
} else if (node.raw === '__system/submitterId') {
38-
return sql.identifier([ 'submissions', 'submitterId' ]); // TODO: HACK HACK
39-
} else if (node.raw === '__system/reviewState') {
40-
return sql.identifier([ 'submissions', 'reviewState' ]); // TODO: HACK HACK
41-
} else {
42-
throw Problem.internal.unsupportedODataField({ at: node.position, text: node.raw });
43-
}
44-
} else if (node.type === 'Literal') {
45-
// for some reason string literals come with their quotes
46-
// TODO: we don't unencode single quotes encoded doubly ('') but we don't support
47-
// any values w quotes in them yet anyway.
48-
return (node.raw === 'null') ? null
49-
: (/^'.*'$/.test(node.raw)) ? node.raw.slice(1, node.raw.length - 1)
50-
: node.raw; // eslint-disable-line indent
51-
} else if (node.type === 'MethodCallExpression') {
52-
return methodCall(node.value.method, node.value.parameters);
53-
} else if (node.type === 'EqualsExpression') {
54-
return binaryOp(node.value.left, node.value.right, 'is not distinct from');
55-
} else if (node.type === 'NotEqualsExpression') {
56-
return binaryOp(node.value.left, node.value.right, 'is distinct from');
57-
} else if (node.type === 'LesserThanExpression') {
58-
return binaryOp(node.value.left, node.value.right, '<');
59-
} else if (node.type === 'LesserOrEqualsExpression') {
60-
return binaryOp(node.value.left, node.value.right, '<=');
61-
} else if (node.type === 'GreaterThanExpression') {
62-
return binaryOp(node.value.left, node.value.right, '>');
63-
} else if (node.type === 'GreaterOrEqualsExpression') {
64-
return binaryOp(node.value.left, node.value.right, '>=');
65-
} else if (node.type === 'AndExpression') {
66-
return binaryOp(node.value.left, node.value.right, 'and');
67-
} else if (node.type === 'OrExpression') {
68-
return binaryOp(node.value.left, node.value.right, 'or');
69-
} else if (node.type === 'NotExpression') {
70-
return sql`(not ${op(node.value)})`;
71-
} else if (node.type === 'BoolParenExpression') {
72-
// Because we add parentheses elsewhere, we don't need to add another set of
73-
// parentheses here. The main effect of a BoolParenExpression is the way it
74-
// restructures the AST.
75-
return op(node.value);
76-
} else {
77-
throw Problem.internal.unsupportedODataExpression({ at: node.position, type: node.type, text: node.raw });
78-
}
79-
};
21+
////////////////////////////////////////
22+
// AST NODE TRANSFORMATION
23+
// These functions are defined inside odataFilter() so that they can access odataToColumnMap
24+
// I don't want to pass it to all of them.
8025

81-
////////////////////////////////////////
82-
// MAIN ENTRY POINT
26+
const extractFunctions = ['year', 'month', 'day', 'hour', 'minute', 'second'];
27+
const methodCall = (fn, params) => {
28+
// n.b. odata-v4-parser appears to already validate function name and arity.
29+
const lowerName = fn.toLowerCase();
30+
if (extractFunctions.includes(lowerName))
31+
return sql`extract(${raw(lowerName)} from ${op(params[0])})`; // eslint-disable-line no-use-before-define
32+
else if (fn === 'now')
33+
return sql`now()`;
34+
};
35+
const binaryOp = (left, right, operator) =>
36+
// always use parens to ensure the original AST op precedence.
37+
sql`(${op(left)} ${raw(operator)} ${op(right)})`; // eslint-disable-line no-use-before-define
8338

84-
const odataFilter = (expr) => {
85-
if (expr == null) return sql`true`;
39+
const op = (node) => {
40+
if (node.type === 'FirstMemberExpression') {
41+
if (odataToColumnMap.has(node.raw)) {
42+
return sql.identifier(odataToColumnMap.get(node.raw).split('.'));
43+
} else {
44+
throw Problem.internal.unsupportedODataField({ at: node.position, text: node.raw });
45+
}
46+
} else if (node.type === 'Literal') {
47+
// for some reason string literals come with their quotes
48+
// TODO: we don't unencode single quotes encoded doubly ('') but we don't support
49+
// any values w quotes in them yet anyway.
50+
return (node.raw === 'null') ? null
51+
: (/^'.*'$/.test(node.raw)) ? node.raw.slice(1, node.raw.length - 1)
52+
: node.raw; // eslint-disable-line indent
53+
} else if (node.type === 'MethodCallExpression') {
54+
return methodCall(node.value.method, node.value.parameters);
55+
} else if (node.type === 'EqualsExpression') {
56+
return binaryOp(node.value.left, node.value.right, 'is not distinct from');
57+
} else if (node.type === 'NotEqualsExpression') {
58+
return binaryOp(node.value.left, node.value.right, 'is distinct from');
59+
} else if (node.type === 'LesserThanExpression') {
60+
return binaryOp(node.value.left, node.value.right, '<');
61+
} else if (node.type === 'LesserOrEqualsExpression') {
62+
return binaryOp(node.value.left, node.value.right, '<=');
63+
} else if (node.type === 'GreaterThanExpression') {
64+
return binaryOp(node.value.left, node.value.right, '>');
65+
} else if (node.type === 'GreaterOrEqualsExpression') {
66+
return binaryOp(node.value.left, node.value.right, '>=');
67+
} else if (node.type === 'AndExpression') {
68+
return binaryOp(node.value.left, node.value.right, 'and');
69+
} else if (node.type === 'OrExpression') {
70+
return binaryOp(node.value.left, node.value.right, 'or');
71+
} else if (node.type === 'NotExpression') {
72+
return sql`(not ${op(node.value)})`;
73+
} else if (node.type === 'BoolParenExpression') {
74+
// Because we add parentheses elsewhere, we don't need to add another set of
75+
// parentheses here. The main effect of a BoolParenExpression is the way it
76+
// restructures the AST.
77+
return op(node.value);
78+
} else {
79+
throw Problem.internal.unsupportedODataExpression({ at: node.position, type: node.type, text: node.raw });
80+
}
81+
};
8682

8783
let ast; // still hate this.
8884
try { ast = odataParser.filter(expr); } // eslint-disable-line brace-style

‎lib/data/submission.js‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -328,11 +328,18 @@ const diffSubmissions = (structurals, versions) => {
328328
return byVersion;
329329
};
330330

331+
const odataToColumnMap = new Map([
332+
['__system/submissionDate', 'submissions.createdAt'],
333+
['__system/updatedAt', 'submissions.updatedAt'],
334+
['__system/submitterId', 'submissions.submitterId'],
335+
['__system/reviewState', 'submissions.reviewState']
336+
]);
337+
331338
module.exports = {
332339
submissionXmlToFieldStream,
333340
getSelectMultipleResponses,
334341
_hashedTree, _diffObj, _diffArray,
335-
diffSubmissions,
342+
diffSubmissions, odataToColumnMap,
336343
_symbols: { subhash, subhashes, keys, score }
337344
};
338345

0 commit comments

Comments
 (0)