Skip to content

Commit 2a59854

Browse files
KSDaemonKSDaemon
committed
code polish
predecessors comparator OMG! Is it really working?! everything works besides loop detection correct additional hints: everything works besides loop detection
1 parent 106bc7a commit 2a59854

File tree

1 file changed

+182
-71
lines changed

1 file changed

+182
-71
lines changed

‎packages/cubejs-schema-compiler/src/adapter/BaseQuery.js‎

Lines changed: 182 additions & 71 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@ import {
2222
localTimestampToUtc,
2323
timeSeries as timeSeriesBase,
2424
timeSeriesFromCustomInterval,
25-
parseSqlInterval,
2625
findMinGranularityDimension
2726
} from '@cubejs-backend/shared';
2827

@@ -436,81 +435,148 @@ export class BaseQuery {
436435
*/
437436
get allJoinHints() {
438437
if (!this.collectedJoinHints) {
439-
const [rootOfJoin, ...allMembersJoinHints] = this.collectJoinHintsFromMembers(this.allMembersConcat(false));
438+
const allMembersJoinHints = this.collectJoinHintsFromMembers(this.allMembersConcat(false));
440439
const customSubQueryJoinHints = this.collectJoinHintsFromMembers(this.joinMembersFromCustomSubQuery());
441-
let joinMembersJoinHints = this.collectJoinHintsFromMembers(this.joinMembersFromJoin(this.join));
442-
443-
// One cube may join the other cube via transitive joined cubes,
444-
// members from which are referenced in the join `on` clauses.
445-
// We need to collect such join hints and push them upfront of the joining one
446-
// but only if they don't exist yet. Cause in other case we might affect what
447-
// join path will be constructed in join graph.
448-
// It is important to use queryLevelJoinHints during the calculation if it is set.
449-
450-
const constructJH = () => {
451-
const filteredJoinMembersJoinHints = joinMembersJoinHints.filter(m => !allMembersJoinHints.includes(m));
452-
return [
453-
...this.queryLevelJoinHints,
454-
...(rootOfJoin ? [rootOfJoin] : []),
455-
...filteredJoinMembersJoinHints,
456-
...allMembersJoinHints,
457-
...customSubQueryJoinHints,
458-
];
459-
};
460-
461-
let prevJoins = this.join;
462-
let prevJoinMembersJoinHints = joinMembersJoinHints;
463-
let newJoin = this.joinGraph.buildJoin(constructJH());
464-
465-
const isOrderPreserved = (base, updated) => {
466-
const common = base.filter(value => updated.includes(value));
467-
const bFiltered = updated.filter(value => common.includes(value));
468-
469-
return common.every((x, i) => x === bFiltered[i]);
470-
};
471-
472-
const isJoinTreesEqual = (a, b) => {
473-
if (!a || !b || a.root !== b.root || a.joins.length !== b.joins.length) {
474-
return false;
475-
}
476-
477-
// We don't care about the order of joins on the same level, so
478-
// we can compare them as sets.
479-
const aJoinsSet = new Set(a.joins.map(j => `${j.originalFrom}->${j.originalTo}`));
480-
const bJoinsSet = new Set(b.joins.map(j => `${j.originalFrom}->${j.originalTo}`));
481-
482-
if (aJoinsSet.size !== bJoinsSet.size) {
483-
return false;
484-
}
485-
486-
for (const val of aJoinsSet) {
487-
if (!bJoinsSet.has(val)) {
488-
return false;
489-
}
490-
}
491-
492-
return true;
493-
};
494-
495-
// Safeguard against infinite loop in case of cyclic joins somehow managed to slip through
496-
let cnt = 0;
440+
const allJoinHints = [
441+
...this.queryLevelJoinHints,
442+
...allMembersJoinHints,
443+
...customSubQueryJoinHints,
444+
];
497445

498-
while (newJoin?.joins.length > 0 && !isJoinTreesEqual(prevJoins, newJoin) && cnt < 10000) {
499-
prevJoins = newJoin;
500-
joinMembersJoinHints = this.collectJoinHintsFromMembers(this.joinMembersFromJoin(newJoin));
501-
if (!isOrderPreserved(prevJoinMembersJoinHints, joinMembersJoinHints)) {
502-
throw new UserError(`Can not construct joins for the query, potential loop detected: ${prevJoinMembersJoinHints.join('->')} vs ${joinMembersJoinHints.join('->')}`);
503-
}
504-
newJoin = this.joinGraph.buildJoin(constructJH());
505-
prevJoinMembersJoinHints = joinMembersJoinHints;
506-
cnt++;
507-
}
446+
const tempJoin = this.joinGraph.buildJoin(allJoinHints);
508447

509-
if (cnt >= 10000) {
510-
throw new UserError('Can not construct joins for the query, potential loop detected');
448+
if (!tempJoin) {
449+
return allJoinHints;
511450
}
512451

513-
this.collectedJoinHints = R.uniq(constructJH());
452+
const joinMembersJoinHints = this.collectJoinHintsFromMembers(this.joinMembersFromJoin(tempJoin));
453+
const allJoinHintsFlatten = new Set(allJoinHints.flat());
454+
const newCollectedHints = joinMembersJoinHints.filter(j => !allJoinHintsFlatten.has(j));
455+
456+
this.collectedJoinHints = R.uniq([
457+
...this.queryLevelJoinHints,
458+
tempJoin.root,
459+
...newCollectedHints,
460+
...allMembersJoinHints,
461+
]);
462+
463+
//
464+
// let joinMembersJoinHints = [];
465+
// let originalQueryMembersJoinPredecessors = {};
466+
//
467+
// // One cube may join the other cube via transitive joined cubes,
468+
// // members from which are referenced in the join `on` clauses.
469+
// // We need to collect such join hints and push them upfront of the joining one
470+
// // but only if they don't exist yet. Cause in other case we might affect what
471+
// // join path will be constructed in join graph.
472+
// // It is important to use queryLevelJoinHints during the calculation if it is set.
473+
//
474+
// const constructJH = () => {
475+
// const filteredJoinMembersJoinHints = joinMembersJoinHints.filter(m => !allMembersJoinHints.includes(m) && m !== rootOfJoin);
476+
// return [
477+
// ...this.queryLevelJoinHints,
478+
// ...(rootOfJoin ? [rootOfJoin] : []),
479+
// ...filteredJoinMembersJoinHints,
480+
// ...allMembersJoinHints.filter(m => m !== rootOfJoin),
481+
// ...customSubQueryJoinHints,
482+
// ];
483+
// };
484+
//
485+
// if (this.join) {
486+
// joinMembersJoinHints = this.collectJoinHintsFromMembers(this.joinMembersFromJoin(this.join));
487+
// originalQueryMembersJoinPredecessors = this.buildPredecessors(joinMembersJoinHints.flat());
488+
// rootOfJoin = this.join.root;
489+
// }
490+
//
491+
// let prevJoin = this.join;
492+
// let newJoin = this.joinGraph.buildJoin(constructJH());
493+
// if (newJoin) {
494+
// rootOfJoin = newJoin.root;
495+
// }
496+
//
497+
// if (!this.join) {
498+
// joinMembersJoinHints = this.collectJoinHintsFromMembers(this.joinMembersFromJoin(newJoin));
499+
// originalQueryMembersJoinPredecessors = this.buildPredecessors(joinMembersJoinHints.flat());
500+
// }
501+
//
502+
// const isOrderPreserved = (updatedJoinHints) => {
503+
// for (let i = 0, l = updatedJoinHints.length; i < l; i++) {
504+
// const predecessors = originalQueryMembersJoinPredecessors[updatedJoinHints[i]];
505+
//
506+
// if (predecessors?.length > 0) {
507+
// const predLen = predecessors.length;
508+
//
509+
// let predIdx = 0;
510+
// let joinHintIdx = 0;
511+
//
512+
// while (joinHintIdx < i && predIdx < predLen) {
513+
// if (updatedJoinHints[joinHintIdx] === predecessors[predIdx]) {
514+
// joinHintIdx++;
515+
// predIdx++;
516+
// } else {
517+
// joinHintIdx++;
518+
// }
519+
// }
520+
//
521+
// if (predIdx < predLen) {
522+
// // We still have a must be present predecessor for current hint
523+
// return [false, `${updatedJoinHints[i]} <-> ${predecessors[predIdx]}`];
524+
// }
525+
// }
526+
// }
527+
//
528+
// return [true, ''];
529+
// };
530+
//
531+
// const isJoinTreesEqual = (a, b) => {
532+
// if (!a || !b || a.root !== b.root || a.joins.length !== b.joins.length) {
533+
// return false;
534+
// }
535+
//
536+
// // We don't care about the order of joins on the same level, so
537+
// // we can compare them as sets.
538+
// const aJoinsSet = new Set(a.joins.map(j => `${j.originalFrom}->${j.originalTo}`));
539+
// const bJoinsSet = new Set(b.joins.map(j => `${j.originalFrom}->${j.originalTo}`));
540+
//
541+
// if (aJoinsSet.size !== bJoinsSet.size) {
542+
// return false;
543+
// }
544+
//
545+
// for (const val of aJoinsSet) {
546+
// if (!bJoinsSet.has(val)) {
547+
// return false;
548+
// }
549+
// }
550+
//
551+
// return true;
552+
// };
553+
//
554+
// // Safeguard against infinite loop in case of cyclic joins somehow managed to slip through
555+
// let cnt = 0;
556+
//
557+
// while (newJoin?.joins.length > 0 && cnt < 10000) {
558+
// prevJoin = newJoin;
559+
// joinMembersJoinHints = this.collectJoinHintsFromMembers(this.joinMembersFromJoin(newJoin));
560+
// newJoin = this.joinGraph.buildJoin(constructJH());
561+
//
562+
// if (isJoinTreesEqual(prevJoin, newJoin)) {
563+
// break;
564+
// }
565+
//
566+
// const [isOrdered, msg] = isOrderPreserved(joinMembersJoinHints.flat());
567+
//
568+
// if (!isOrdered) {
569+
// throw new UserError(`Can not construct joins for the query, potential loop detected around ${msg}`);
570+
// }
571+
//
572+
// cnt++;
573+
// }
574+
//
575+
// if (cnt >= 10000) {
576+
// throw new UserError('Can not construct joins for the query, potential loop detected');
577+
// }
578+
//
579+
// this.collectedJoinHints = R.uniq(constructJH());
514580
}
515581
return this.collectedJoinHints;
516582
}
@@ -538,6 +604,51 @@ export class BaseQuery {
538604
);
539605
}
540606

607+
/**
608+
* @private
609+
* @param {Array<string>} arr
610+
* @returns {{}|any}
611+
*/
612+
buildPredecessors(arr) {
613+
if (!arr || arr.length === 0) return {};
614+
615+
const root = arr[0];
616+
617+
// the first position of each unique element
618+
const firstPos = new Map();
619+
for (let i = 0; i < arr.length; i++) {
620+
if (!firstPos.has(arr[i])) firstPos.set(arr[i], i);
621+
}
622+
623+
const result = {};
624+
625+
for (const [elem, idx] of firstPos.entries()) {
626+
if (elem === root) {
627+
result[elem] = [];
628+
} else {
629+
// finding the nearest root on the left <<
630+
const seen = new Set();
631+
const path = [];
632+
633+
for (let j = idx - 1; j >= 0; j--) {
634+
const v = arr[j];
635+
if (!seen.has(v)) {
636+
seen.add(v);
637+
path.push(v);
638+
}
639+
if (v === root) {
640+
break;
641+
}
642+
}
643+
644+
path.reverse();
645+
result[elem] = path;
646+
}
647+
}
648+
649+
return result;
650+
}
651+
541652
initUngrouped() {
542653
this.ungrouped = this.options.ungrouped;
543654
if (this.ungrouped) {

0 commit comments

Comments
 (0)