Skip to content

Commit db38d71

Browse files
ktuitektuite
authored
Return dataset metadata properties and entities in order (getodk#780)
* Return dataset metadata properties in order * Adding another test, but one that shows off another bug * Publish individual dataset properties as needed * Export entities in csv with newest at top * Ordering by published time, adding a test, and fixing publish query * Still exploring ds prop order with deleted forms
1 parent d84662b commit db38d71

File tree

5 files changed

+335
-6
lines changed

5 files changed

+335
-6
lines changed

‎lib/model/query/datasets.js‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,14 @@ insert_property_fields AS (
107107
INSERT INTO ds_property_fields ("dsPropertyId", "formDefId", "schemaId", "path")
108108
SELECT "dsPropertyId", "formDefId"::integer, "schemaId"::integer, path FROM all_properties
109109
)
110+
${publish ? sql`
111+
,
112+
update_ds_properties AS (
113+
UPDATE ds_properties SET "publishedAt" = clock_timestamp()
114+
FROM all_properties
115+
WHERE ds_properties.id = all_properties."dsPropertyId" AND ds_properties."publishedAt" IS NULL
116+
)
117+
` : sql``}
110118
`;
111119

112120
const _createOrMerge = (dataset, fields, acteeId, publish) => sql`
@@ -137,9 +145,12 @@ const _getByNameSql = ((fields, datasetName, projectId, includeForms) => sql`
137145
) stats on stats."datasetId" = datasets.id
138146
LEFT OUTER JOIN ds_properties ON
139147
datasets.id = ds_properties."datasetId" AND ds_properties."publishedAt" IS NOT NULL
140-
${includeForms ? sql`
141148
LEFT OUTER JOIN ds_property_fields ON
142149
ds_properties.id = ds_property_fields."dsPropertyId"
150+
LEFT OUTER JOIN form_fields ON
151+
ds_property_fields.path = form_fields.path
152+
AND ds_property_fields."schemaId" = form_fields."schemaId"
153+
${includeForms ? sql`
143154
LEFT OUTER JOIN forms ON
144155
ds_property_fields."formDefId" = forms."currentDefId"
145156
LEFT JOIN form_defs ON
@@ -148,6 +159,7 @@ const _getByNameSql = ((fields, datasetName, projectId, includeForms) => sql`
148159
WHERE datasets.name = ${datasetName}
149160
AND datasets."projectId" = ${projectId}
150161
AND datasets."publishedAt" IS NOT NULL
162+
ORDER BY ds_properties."publishedAt", form_fields.order, ds_properties.id
151163
`);
152164

153165
const _getLinkedForms = (datasetName, projectId) => sql`
@@ -333,12 +345,14 @@ WITH properties_update as (
333345
JOIN form_defs fd ON fs."id" = fd."schemaId"
334346
WHERE fd."id" = ${formDefId}
335347
AND dpf."dsPropertyId" = dp.id
348+
AND dp."publishedAt" IS NULL
336349
RETURNING dp.*
337350
), datasets_update as (
338351
UPDATE datasets ds SET "publishedAt" = ${publishedAt}
339352
FROM dataset_form_defs dfd
340353
WHERE dfd."formDefId" = ${formDefId}
341354
AND dfd."datasetId" = ds.id
355+
AND ds."publishedAt" IS NULL
342356
RETURNING *
343357
)
344358
-- selecting following for publish.audit

‎lib/model/query/entities.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -147,7 +147,7 @@ WHERE
147147
entities."datasetId" = ${datasetId}
148148
AND entity_defs.current=true
149149
AND ${odataFilter(options.filter, odataToColumnMap)}
150-
ORDER BY entities.id
150+
ORDER BY entities."createdAt" DESC, entities.id DESC
151151
${page(options)}`)
152152
.then(stream.map(_exportUnjoiner));
153153

‎test/data/xml.js‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -368,6 +368,31 @@ module.exports = {
368368
</h:head>
369369
</h:html>`,
370370

371+
multiPropertyEntity: `<?xml version="1.0"?>
372+
<h:html xmlns="http://www.w3.org/2002/xforms" xmlns:h="http://www.w3.org/1999/xhtml" xmlns:jr="http://openrosa.org/javarosa" xmlns:entities="http://www.opendatakit.org/xforms">
373+
<h:head>
374+
<model entities:entities-version="2022.1.0">
375+
<instance>
376+
<data id="multiPropertyEntity" orx:version="1.0">
377+
<q1/>
378+
<q2/>
379+
<q3/>
380+
<q4/>
381+
<meta>
382+
<entity dataset="foo" id="" create="">
383+
<label/>
384+
</entity>
385+
</meta>
386+
</data>
387+
</instance>
388+
<bind entities:saveto="a_q3" nodeset="/data/q3" type="string"/>
389+
<bind entities:saveto="b_q1" nodeset="/data/q1" type="string"/>
390+
<bind entities:saveto="c_q4" nodeset="/data/q4" type="string"/>
391+
<bind entities:saveto="d_q2" nodeset="/data/q2" type="string"/>
392+
</model>
393+
</h:head>
394+
</h:html>`,
395+
371396
groupRepeat: `<?xml version="1.0"?>
372397
<h:html xmlns="http://www.w3.org/2002/xforms" xmlns:ev="http://www.w3.org/2001/xml-events" xmlns:h="http://www.w3.org/1999/xhtml" xmlns:jr="http://openrosa.org/javarosa" xmlns:odk="http://www.opendatakit.org/xforms" xmlns:orx="http://openrosa.org/xforms" xmlns:xsd="http://www.w3.org/2001/XMLSchema">
373398
<h:head>
@@ -579,6 +604,32 @@ module.exports = {
579604
<age>40</age>
580605
</data>`
581606
},
607+
multiPropertyEntity: {
608+
one: `<data xmlns:jr="http://openrosa.org/javarosa" xmlns:entities="http://www.opendatakit.org/xforms" id="multiPropertyEntity" version="1.0">
609+
<meta>
610+
<instanceID>one</instanceID>
611+
<entities:entity dataset="foo" id="uuid:12345678-1234-4123-8234-123456789aaa" create="1">
612+
<entities:label>one</entities:label>
613+
</entities:entity>
614+
</meta>
615+
<q1>w</q1>
616+
<q2>x</q2>
617+
<q3>y</q3>
618+
<q4>z</q4>
619+
</data>`,
620+
two: `<data xmlns:jr="http://openrosa.org/javarosa" xmlns:entities="http://www.opendatakit.org/xforms" id="multiPropertyEntity" version="1.0">
621+
<meta>
622+
<instanceID>two</instanceID>
623+
<entities:entity dataset="foo" id="uuid:12345678-1234-4123-8234-123456789bbb" create="1">
624+
<entities:label>two</entities:label>
625+
</entities:entity>
626+
</meta>
627+
<q1>a</q1>
628+
<q2>b</q2>
629+
<q3>c</q3>
630+
<q4>d</q4>
631+
</data>`,
632+
},
582633
groupRepeat: {
583634
one: `<data xmlns:jr="http://openrosa.org/javarosa" xmlns:orx="http://openrosa.org/xforms" id="groupRepeat">
584635
<text>xyz</text>

‎test/integration/api/datasets.js‎

Lines changed: 210 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -438,11 +438,11 @@ describe('datasets and entities', () => {
438438
publishedAt.should.not.be.null();
439439
return p;
440440
}).should.be.eql([
441-
{ name: 'age', forms: [ { name: 'simpleEntity', xmlFormId: 'simpleEntity' }, ] },
442441
{ name: 'first_name', forms: [
443442
{ name: 'simpleEntity', xmlFormId: 'simpleEntity' },
444443
{ name: 'simpleEntity2', xmlFormId: 'simpleEntity2' }
445444
] },
445+
{ name: 'age', forms: [ { name: 'simpleEntity', xmlFormId: 'simpleEntity' }, ] },
446446
{ name: 'address', forms: [ { name: 'simpleEntity2', xmlFormId: 'simpleEntity2' }, ] }
447447
]);
448448

@@ -480,6 +480,215 @@ describe('datasets and entities', () => {
480480
});
481481

482482
}));
483+
484+
it('should return properties of a dataset in order', testService(async (service) => {
485+
const asAlice = await service.login('alice', identity);
486+
487+
await asAlice.post('/v1/projects/1/forms?publish=true')
488+
.send(testData.forms.multiPropertyEntity)
489+
.set('Content-Type', 'application/xml')
490+
.expect(200);
491+
492+
await asAlice.get('/v1/projects/1/datasets/foo')
493+
.expect(200)
494+
.then(({ body }) => {
495+
const { properties } = body;
496+
properties.map((p) => p.name)
497+
.should.be.eql([
498+
'b_q1',
499+
'd_q2',
500+
'a_q3',
501+
'c_q4'
502+
]);
503+
});
504+
}));
505+
506+
it('should return dataset properties from multiple forms in order', testService(async (service) => {
507+
const asAlice = await service.login('alice', identity);
508+
509+
await asAlice.post('/v1/projects/1/forms?publish=true')
510+
.send(testData.forms.multiPropertyEntity)
511+
.set('Content-Type', 'application/xml')
512+
.expect(200);
513+
514+
await asAlice.post('/v1/projects/1/forms?publish=true')
515+
.send(testData.forms.multiPropertyEntity
516+
.replace('multiPropertyEntity', 'multiPropertyEntity2')
517+
.replace('b_q1', 'f_q1')
518+
.replace('d_q2', 'e_q2'))
519+
.set('Content-Type', 'application/xml')
520+
.expect(200);
521+
522+
await asAlice.get('/v1/projects/1/datasets/foo')
523+
.expect(200)
524+
.then(({ body }) => {
525+
const { properties } = body;
526+
properties.map((p) => p.name)
527+
.should.be.eql([
528+
'b_q1',
529+
'd_q2',
530+
'a_q3',
531+
'c_q4',
532+
'f_q1',
533+
'e_q2'
534+
]);
535+
});
536+
}));
537+
538+
it('should return dataset properties from multiple forms including updated form with updated schema', testService(async (service) => {
539+
const asAlice = await service.login('alice', identity);
540+
541+
await asAlice.post('/v1/projects/1/forms?publish=true')
542+
.send(testData.forms.multiPropertyEntity)
543+
.set('Content-Type', 'application/xml')
544+
.expect(200);
545+
546+
await asAlice.post('/v1/projects/1/forms?publish=true')
547+
.send(testData.forms.multiPropertyEntity
548+
.replace('multiPropertyEntity', 'multiPropertyEntity2')
549+
.replace('b_q1', 'f_q1')
550+
.replace('d_q2', 'e_q2'))
551+
.set('Content-Type', 'application/xml')
552+
.expect(200);
553+
554+
await asAlice.post('/v1/projects/1/forms/multiPropertyEntity/draft')
555+
.send(testData.forms.multiPropertyEntity
556+
.replace('orx:version="1.0"', 'orx:version="2.0"')
557+
.replace('b_q1', 'g_q1'))
558+
.set('Content-Type', 'application/xml')
559+
.expect(200);
560+
561+
await asAlice.post('/v1/projects/1/forms/multiPropertyEntity/draft/publish').expect(200);
562+
563+
await asAlice.get('/v1/projects/1/datasets/foo')
564+
.expect(200)
565+
.then(({ body }) => {
566+
const { properties } = body;
567+
properties.map((p) => p.name)
568+
.should.be.eql([
569+
'b_q1',
570+
'd_q2',
571+
'a_q3',
572+
'c_q4',
573+
'f_q1',
574+
'e_q2',
575+
'g_q1'
576+
]);
577+
});
578+
}));
579+
580+
it('should return dataset properties when purged draft form shares some properties', testService(async (service, { Forms }) => {
581+
const asAlice = await service.login('alice', identity);
582+
583+
await asAlice.post('/v1/projects/1/forms')
584+
.send(testData.forms.multiPropertyEntity)
585+
.set('Content-Type', 'application/xml')
586+
.expect(200);
587+
588+
await asAlice.post('/v1/projects/1/forms?publish=true')
589+
.send(testData.forms.multiPropertyEntity
590+
.replace('multiPropertyEntity', 'multiPropertyEntity2')
591+
.replace('b_q1', 'f_q1')
592+
.replace('d_q2', 'e_q2'))
593+
.set('Content-Type', 'application/xml')
594+
.expect(200);
595+
596+
await asAlice.delete('/v1/projects/1/forms/multiPropertyEntity')
597+
.expect(200);
598+
599+
await Forms.purge(true);
600+
601+
await asAlice.get('/v1/projects/1/datasets/foo')
602+
.expect(200)
603+
.then(({ body }) => {
604+
const { properties } = body;
605+
properties.map((p) => p.name)
606+
.should.be.eql([
607+
'f_q1',
608+
'e_q2',
609+
'a_q3',
610+
'c_q4'
611+
]);
612+
});
613+
}));
614+
615+
it('should return dataset properties when draft form (purged before second form publish) shares some properties', testService(async (service, { Forms }) => {
616+
const asAlice = await service.login('alice', identity);
617+
618+
await asAlice.post('/v1/projects/1/forms')
619+
.send(testData.forms.multiPropertyEntity)
620+
.set('Content-Type', 'application/xml')
621+
.expect(200);
622+
623+
await asAlice.post('/v1/projects/1/forms')
624+
.send(testData.forms.multiPropertyEntity
625+
.replace('multiPropertyEntity', 'multiPropertyEntity2')
626+
.replace('b_q1', 'f_q1')
627+
.replace('d_q2', 'e_q2'))
628+
.set('Content-Type', 'application/xml')
629+
.expect(200);
630+
631+
await asAlice.delete('/v1/projects/1/forms/multiPropertyEntity')
632+
.expect(200);
633+
634+
await Forms.purge(true);
635+
636+
await asAlice.post('/v1/projects/1/forms/multiPropertyEntity2/draft/publish');
637+
638+
await asAlice.get('/v1/projects/1/datasets/foo')
639+
.expect(200)
640+
.then(({ body }) => {
641+
const { properties } = body;
642+
properties.map((p) => p.name)
643+
.should.be.eql([
644+
'f_q1',
645+
'e_q2',
646+
'a_q3',
647+
'c_q4'
648+
]);
649+
});
650+
}));
651+
652+
it.skip('should return ordered dataset properties including from deleted published form', testService(async (service, { Forms }) => {
653+
const asAlice = await service.login('alice', identity);
654+
655+
await asAlice.post('/v1/projects/1/forms?publish=true')
656+
.send(testData.forms.multiPropertyEntity)
657+
.set('Content-Type', 'application/xml')
658+
.expect(200);
659+
660+
await asAlice.delete('/v1/projects/1/forms/multiPropertyEntity')
661+
.expect(200);
662+
663+
await Forms.purge(true);
664+
665+
await asAlice.post('/v1/projects/1/forms?publish=true')
666+
.send(testData.forms.multiPropertyEntity
667+
.replace('multiPropertyEntity', 'multiPropertyEntity2')
668+
.replace('b_q1', 'f_q1')
669+
.replace('d_q2', 'e_q2'))
670+
.set('Content-Type', 'application/xml')
671+
.expect(200);
672+
673+
await asAlice.get('/v1/projects/1/datasets/foo')
674+
.expect(200)
675+
.then(({ body }) => {
676+
const { properties } = body;
677+
// Properties are coming out in this other order:
678+
// [ 'a_q3', 'c_q4', 'b_q1', 'd_q2', 'f_q1', 'e_q2' ]
679+
// It's not terrible but would rather all the props of the first form
680+
// show up first.
681+
properties.map((p) => p.name)
682+
.should.be.eql([
683+
'b_q1',
684+
'd_q2',
685+
'a_q3',
686+
'c_q4',
687+
'f_q1',
688+
'e_q2'
689+
]);
690+
});
691+
}));
483692
});
484693
});
485694

0 commit comments

Comments
 (0)