Skip to content

Commit 325b8d8

Browse files
BeyyesBeyyes
authored
Fix error in MergeSortJoin when some columns exist null values & Support cross join with filters
1 parent f62d058 commit 325b8d8

30 files changed

+1212
-463
lines changed

‎integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBMultiIDsWithAttributesTableIT.java‎

Lines changed: 237 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@
3636

3737
import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail;
3838
import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest;
39+
import static org.apache.iotdb.db.queryengine.plan.relational.planner.optimizations.JoinUtils.FULL_JOIN_ONLY_SUPPORT_EQUI_JOIN;
3940
import static org.junit.Assert.fail;
4041

4142
/** In this IT, table has more than one IDs and Attributes. */
@@ -1383,7 +1384,7 @@ public void varianceTest() {
13831384
// ==================================================================
13841385
// no filter
13851386
@Test
1386-
public void innerJoinTest1() {
1387+
public void selfTimeColumnInnerJoinTest1() {
13871388
String[] expectedHeader =
13881389
new String[] {"time", "device", "level", "num", "device", "attr2", "num", "str"};
13891390
String[] retArray =
@@ -1420,7 +1421,7 @@ public void innerJoinTest1() {
14201421

14211422
// has filter
14221423
@Test
1423-
public void innerJoinTest2() {
1424+
public void selfTimeColumnInnerJoinTest2() {
14241425
String[] expectedHeader =
14251426
new String[] {"time", "device", "level", "t1_num_add", "device", "attr2", "num", "str"};
14261427
String[] retArray =
@@ -1510,7 +1511,7 @@ public void innerJoinTest2() {
15101511

15111512
// no filter
15121513
@Test
1513-
public void fullOuterJoinTest1() {
1514+
public void timeColumnFullOuterJoinTest1() {
15141515
expectedHeader =
15151516
new String[] {"time", "device", "level", "num", "device", "attr2", "num", "str"};
15161517
retArray =
@@ -1682,7 +1683,7 @@ public void fullOuterJoinTest1() {
16821683

16831684
// has filter
16841685
@Test
1685-
public void fullOuterJoinTest2() {
1686+
public void timeColumnFullOuterJoinTest2() {
16861687
expectedHeader =
16871688
new String[] {"time", "device", "level", "t1_num_add", "device", "attr2", "num", "str"};
16881689
retArray =
@@ -1785,7 +1786,7 @@ public void fourTableJoinTest() {
17851786
}
17861787

17871788
@Test
1788-
public void innerJoinTest() {
1789+
public void twoTableTimeColumnInnerJoinTest() {
17891790
expectedHeader = new String[] {"time", "device1", "value1", "device2", "value2"};
17901791
sql =
17911792
"SELECT "
@@ -1807,7 +1808,7 @@ public void innerJoinTest() {
18071808
}
18081809

18091810
@Test
1810-
public void innerJoinOnTwoColumns() {
1811+
public void innerJoinOnMultiColumns() {
18111812
expectedHeader = new String[] {"time", "device1", "value1", "device2", "value2"};
18121813
sql =
18131814
"SELECT "
@@ -1826,9 +1827,237 @@ public void innerJoinOnTwoColumns() {
18261827
tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME);
18271828

18281829
sql =
1829-
"select table1.s1 from table1 t1 join table2 t2 on t1.time = t2.time and t1.device = t2.device";
1830+
"SELECT "
1831+
+ " t1.time, "
1832+
+ " t1.device as device1, "
1833+
+ " t1.value as value1, "
1834+
+ " t2.device as device2, "
1835+
+ " t2.value as value2 "
1836+
+ "FROM "
1837+
+ " tableA t1 cross join tableB t2 "
1838+
+ "where t1.time = t2.time and t1.device = t2.device order by t1.time";
1839+
tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME);
1840+
1841+
sql =
1842+
"SELECT "
1843+
+ " t1.time, "
1844+
+ " t1.device as device1, "
1845+
+ " t1.value as value1, "
1846+
+ " t2.device as device2, "
1847+
+ " t2.value as value2 "
1848+
+ "FROM "
1849+
+ " tableA t1 JOIN tableB t2 "
1850+
+ "ON t1.time = t2.time and t1.device = t2.device order by t1.time";
1851+
tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME);
1852+
1853+
expectedHeader = new String[] {"time", "device", "value1", "value2"};
1854+
sql =
1855+
"SELECT "
1856+
+ " time, device, "
1857+
+ " t1.value as value1, "
1858+
+ " t2.value as value2 "
1859+
+ "FROM "
1860+
+ " tableA t1 JOIN tableB t2 "
1861+
+ "USING(time, device) ORDER BY time";
1862+
retArray =
1863+
new String[] {
1864+
"2020-01-01T00:00:03.000Z,d1,3,30,", "2020-01-01T00:00:05.000Z,d2,5,50,",
1865+
};
1866+
tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME);
1867+
1868+
expectedHeader = new String[] {"device", "device", "num", "num", "floatnum", "floatnum"};
1869+
sql =
1870+
"select t0.device, t1.device, t0.num, t1.num, t0.floatnum, t1.floatnum from table0 t0 join table1 t1 on t0.device=t1.device AND t0.attr2=t1.attr2 AND t0.num>t1.num AND t0.floatnum>t1.floatnum ORDER BY t0.device,t1.device,t0.num,t1.num";
1871+
retArray =
1872+
new String[] {
1873+
"d1,d1,4,1,213.1,12.123,", "d1,d1,6,3,1231.21,231.2121,", "d1,d1,14,1,231.34,12.123,",
1874+
};
1875+
tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME);
1876+
1877+
expectedHeader = new String[] {"time1", "time2", "device1", "device2"};
1878+
sql =
1879+
"select t1.time as time1, t2.time as time2, t1.device as device1, t2.device as device2 from tablea t1 join tableb t2 "
1880+
+ "on cast(substring(t1.device,2) as int32) = cast(substring(t2.device,2) as int32)+1 order by time1,time2,device1,device2";
1881+
retArray =
1882+
new String[] {
1883+
"2020-01-01T00:00:05.000Z,2020-01-01T00:00:02.000Z,d2,d1,",
1884+
"2020-01-01T00:00:05.000Z,2020-01-01T00:00:03.000Z,d2,d1,",
1885+
"2020-01-01T00:00:07.000Z,2020-01-01T00:00:02.000Z,d2,d1,",
1886+
"2020-01-01T00:00:07.000Z,2020-01-01T00:00:03.000Z,d2,d1,",
1887+
};
1888+
tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME);
1889+
1890+
expectedHeader = new String[] {"attr1", "attr2"};
1891+
sql =
1892+
"select t0.attr1,t0.attr2 from table0 t0 join table1 t1 on t0.attr1=t1.attr1 AND t0.attr2=t1.attr2";
1893+
retArray =
1894+
new String[] {
1895+
"c,d,", "c,d,", "c,d,", "c,d,", "c,d,", "c,d,", "c,d,", "c,d,", "c,d,", "c,d,", "c,d,",
1896+
"c,d,", "c,d,", "c,d,", "c,d,", "c,d,", "c,d,", "c,d,", "c,d,", "c,d,", "c,d,", "t,a,",
1897+
"t,a,", "t,a,",
1898+
};
1899+
tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME);
1900+
1901+
// expectedHeader = new String[] {"device", "device", "attr1", "attr1", "date", "date"};
1902+
// sql =
1903+
// "select t0.device, t1.device, t0.attr1, t1.attr1, t0.date, t1.date from table0 t0 join
1904+
// table1 t1 on t0.device=t1.device and t0.attr1=t1.attr1 OR t0.date=t1.date";
1905+
// retArray =
1906+
// new String[] {
1907+
// "d1,d1,c,c,null,2023-01-01,",
1908+
// "d1,d1,c,c,null,2023-01-01,",
1909+
// "d1,d1,c,c,null,2023-01-01,",
1910+
// "d1,d1,c,c,null,null,",
1911+
// "d1,d1,c,c,null,null,",
1912+
// "d1,d1,c,c,null,null,",
1913+
// "d1,d1,t,t,null,null,",
1914+
// "d1,d1,t,t,null,null,",
1915+
// "d1,d1,t,t,null,null,",
1916+
// "d2,d1,null,c,2023-01-01,2023-01-01,",
1917+
// };
1918+
// tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME);
1919+
}
1920+
1921+
@Test
1922+
public void fullJoinTest() {
1923+
expectedHeader = new String[] {"date", "date"};
1924+
sql =
1925+
"select t0.date, t1.date from table0 t0 full join table1 t1 on t0.date=t1.date order by t0.date, t1.date";
1926+
retArray =
1927+
new String[] {
1928+
"2022-01-01,null,",
1929+
"2023-01-01,2023-01-01,",
1930+
"null,2023-05-01,",
1931+
"null,2023-10-01,",
1932+
"null,null,",
1933+
"null,null,",
1934+
"null,null,",
1935+
"null,null,",
1936+
"null,null,",
1937+
"null,null,",
1938+
"null,null,",
1939+
"null,null,",
1940+
"null,null,",
1941+
"null,null,",
1942+
"null,null,",
1943+
"null,null,",
1944+
"null,null,",
1945+
"null,null,",
1946+
"null,null,",
1947+
"null,null,",
1948+
"null,null,",
1949+
"null,null,",
1950+
"null,null,",
1951+
"null,null,",
1952+
"null,null,",
1953+
"null,null,",
1954+
"null,null,",
1955+
"null,null,",
1956+
"null,null,",
1957+
"null,null,",
1958+
"null,null,",
1959+
"null,null,",
1960+
"null,null,",
1961+
"null,null,",
1962+
"null,null,",
1963+
"null,null,",
1964+
"null,null,",
1965+
"null,null,",
1966+
"null,null,",
1967+
"null,null,",
1968+
"null,null,",
1969+
"null,null,",
1970+
};
1971+
tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME);
1972+
1973+
expectedHeader = new String[] {"attr1", "attr2", "attr1", "attr2"};
1974+
sql =
1975+
"select t0.attr1,t0.attr2,t1.attr1,t1.attr2 from table0 t0 full join table1 t1 on t0.attr1=t1.attr1 AND t0.attr2=t1.attr2 order by t0.attr1,t0.attr2,t1.attr1,t1.attr2";
1976+
retArray =
1977+
new String[] {
1978+
"c,d,c,d,",
1979+
"c,d,c,d,",
1980+
"c,d,c,d,",
1981+
"c,d,c,d,",
1982+
"c,d,c,d,",
1983+
"c,d,c,d,",
1984+
"c,d,c,d,",
1985+
"c,d,c,d,",
1986+
"c,d,c,d,",
1987+
"c,d,c,d,",
1988+
"c,d,c,d,",
1989+
"c,d,c,d,",
1990+
"c,d,c,d,",
1991+
"c,d,c,d,",
1992+
"c,d,c,d,",
1993+
"c,d,c,d,",
1994+
"c,d,c,d,",
1995+
"c,d,c,d,",
1996+
"c,d,c,d,",
1997+
"c,d,c,d,",
1998+
"c,d,c,d,",
1999+
"d,c,null,null,",
2000+
"d,c,null,null,",
2001+
"d,c,null,null,",
2002+
"t,a,t,a,",
2003+
"t,a,t,a,",
2004+
"t,a,t,a,",
2005+
"vv,null,null,null,",
2006+
"vv,null,null,null,",
2007+
"vv,null,null,null,",
2008+
"yy,zz,null,null,",
2009+
"yy,zz,null,null,",
2010+
"yy,zz,null,null,",
2011+
"null,null,y,z,",
2012+
"null,null,null,null,",
2013+
"null,null,null,null,",
2014+
"null,null,null,null,",
2015+
"null,null,null,null,",
2016+
"null,null,null,null,",
2017+
"null,null,null,null,",
2018+
"null,null,null,null,",
2019+
"null,null,null,null,",
2020+
"null,null,null,null,",
2021+
"null,null,null,null,",
2022+
"null,null,null,null,",
2023+
"null,null,null,null,",
2024+
"null,null,null,null,",
2025+
"null,null,null,null,",
2026+
"null,null,null,null,",
2027+
"null,null,null,null,",
2028+
"null,null,null,null,",
2029+
"null,null,null,null,",
2030+
"null,null,null,null,",
2031+
};
2032+
tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME);
2033+
}
2034+
2035+
@Test
2036+
public void exceptionTest() {
2037+
tableAssertTestFail(
2038+
"select * from table0 t0 full join table1 t1 on t0.num>t1.num",
2039+
FULL_JOIN_ONLY_SUPPORT_EQUI_JOIN,
2040+
DATABASE_NAME);
2041+
2042+
tableAssertTestFail(
2043+
"select * from table0 t0 full join table1 t1 on t0.num!=t1.num",
2044+
FULL_JOIN_ONLY_SUPPORT_EQUI_JOIN,
2045+
DATABASE_NAME);
2046+
2047+
tableAssertTestFail(
2048+
"select * from table0 t0 full join table1 t1 on t0.device=t1.device AND t0.num>t1.num",
2049+
FULL_JOIN_ONLY_SUPPORT_EQUI_JOIN,
2050+
DATABASE_NAME);
2051+
18302052
tableAssertTestFail(
1831-
sql, "701: Only support time column equi-join in current version", DATABASE_NAME);
2053+
"select * from table0 t0 full join table1 t1 on t0.device=t1.device OR t0.num>t1.num",
2054+
FULL_JOIN_ONLY_SUPPORT_EQUI_JOIN,
2055+
DATABASE_NAME);
2056+
2057+
tableAssertTestFail(
2058+
"select * from table0 t0 full join table1 t1 on t0.device=t1.device OR t0.time=t1.time",
2059+
FULL_JOIN_ONLY_SUPPORT_EQUI_JOIN,
2060+
DATABASE_NAME);
18322061
}
18332062

18342063
public static String[] buildHeaders(int length) {

‎integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTableAggregationIT.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3718,7 +3718,6 @@ public void exceptionTest() {
37183718
"select s1 from table1 where s2 in (select s2 from table1)",
37193719
"Not a valid IR expression",
37203720
DATABASE_NAME);
3721-
37223721
tableAssertTestFail(
37233722
"select avg() from table1",
37243723
"701: Aggregate functions [avg] should only have one argument",

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/comparator/AscBinaryTypeJoinKeyComparator.java‎

Lines changed: 34 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,8 @@
2121

2222
import org.apache.tsfile.read.common.block.TsBlock;
2323

24+
import java.util.Optional;
25+
2426
public class AscBinaryTypeJoinKeyComparator implements JoinKeyComparator {
2527

2628
private static final AscBinaryTypeJoinKeyComparator INSTANCE =
@@ -35,43 +37,61 @@ public static AscBinaryTypeJoinKeyComparator getInstance() {
3537
}
3638

3739
@Override
38-
public boolean lessThan(
40+
public Optional<Boolean> lessThan(
3941
TsBlock left,
4042
int leftColumnIndex,
4143
int leftRowIndex,
4244
TsBlock right,
4345
int rightColumnIndex,
4446
int rightRowIndex) {
45-
return left.getColumn(leftColumnIndex)
46-
.getBinary(leftRowIndex)
47-
.compareTo(right.getColumn(rightColumnIndex).getBinary(rightRowIndex))
48-
< 0;
47+
if (left.getColumn(leftColumnIndex).isNull(leftRowIndex)
48+
|| right.getColumn(rightColumnIndex).isNull(rightRowIndex)) {
49+
return Optional.empty();
50+
}
51+
52+
return Optional.of(
53+
left.getColumn(leftColumnIndex)
54+
.getBinary(leftRowIndex)
55+
.compareTo(right.getColumn(rightColumnIndex).getBinary(rightRowIndex))
56+
< 0);
4957
}
5058

5159
@Override
52-
public boolean equalsTo(
60+
public Optional<Boolean> equalsTo(
5361
TsBlock left,
5462
int leftColumnIndex,
5563
int leftRowIndex,
5664
TsBlock right,
5765
int rightColumnIndex,
5866
int rightRowIndex) {
59-
return left.getColumn(leftColumnIndex)
60-
.getBinary(leftRowIndex)
61-
.equals(right.getColumn(rightColumnIndex).getBinary(rightRowIndex));
67+
if (left.getColumn(leftColumnIndex).isNull(leftRowIndex)
68+
|| right.getColumn(rightColumnIndex).isNull(rightRowIndex)) {
69+
return Optional.empty();
70+
}
71+
72+
return Optional.of(
73+
left.getColumn(leftColumnIndex)
74+
.getBinary(leftRowIndex)
75+
.equals(right.getColumn(rightColumnIndex).getBinary(rightRowIndex)));
6276
}
6377

6478
@Override
65-
public boolean lessThanOrEqual(
79+
public Optional<Boolean> lessThanOrEqual(
6680
TsBlock left,
6781
int leftColumnIndex,
6882
int leftRowIndex,
6983
TsBlock right,
7084
int rightColumnIndex,
7185
int rightRowIndex) {
72-
return left.getColumn(leftColumnIndex)
73-
.getBinary(leftRowIndex)
74-
.compareTo(right.getColumn(rightColumnIndex).getBinary(rightRowIndex))
75-
<= 0;
86+
if (left.getColumn(leftColumnIndex).isNull(leftRowIndex)
87+
|| right.getColumn(rightColumnIndex).isNull(rightRowIndex)) {
88+
return Optional.empty();
89+
}
90+
91+
return Optional.of(
92+
left.getColumn(leftColumnIndex)
93+
.getBinary(leftRowIndex)
94+
.compareTo(right.getColumn(rightColumnIndex).getBinary(rightRowIndex))
95+
<= 0);
7696
}
7797
}

0 commit comments

Comments
 (0)