From f79440d73067f85f51d19be44d63443efba48f22 Mon Sep 17 00:00:00 2001 From: 0xB <76414902+YC27@users.noreply.github.com> Date: Mon, 13 Jan 2025 18:23:51 +0800 Subject: [PATCH 01/10] Collector Node: Build basic architecture, configuration file reading and REST service (#14689) --- iotdb-collector/collector-core/pom.xml | 63 ++++ .../org/apache/iotdb/collector/Collector.java | 107 +++++++ .../collector/config/CollectorConfig.java | 39 +++ .../collector/config/CollectorConstant.java | 24 ++ .../collector/config/CollectorDescriptor.java | 126 ++++++++ .../CollectorSystemPropertiesHandler.java | 52 ++++ .../collector/config/ConfigFileUtils.java | 101 +++++++ .../protocol/rest/filter/ApiOriginFilter.java | 52 ++++ .../rest/filter/AuthorizationFilter.java | 46 +++ .../rest/impl/PingApiServiceImpl.java | 39 +++ .../rest/v1/impl/AdminApiServiceImpl.java | 68 +++++ .../collector/service/CollectorMBean.java | 21 ++ .../service/CollectorRestService.java | 113 +++++++ .../iotdb/collector/service/IService.java | 39 +++ .../collector/service/RegisterManager.java | 82 +++++ .../iotdb-collector-system.properties | 20 ++ iotdb-collector/collector-openapi/pom.xml | 185 ++++++++++++ .../openapi3/iotdb_collector_rest_common.yaml | 63 ++++ .../openapi3/iotdb_collector_rest_v1.yaml | 282 ++++++++++++++++++ iotdb-collector/pom.xml | 42 +++ pom.xml | 1 + 21 files changed, 1565 insertions(+) create mode 100644 iotdb-collector/collector-core/pom.xml create mode 100644 iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/Collector.java create mode 100644 iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/CollectorConfig.java create mode 100644 iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/CollectorConstant.java create mode 100644 iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/CollectorDescriptor.java create mode 100644 iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/CollectorSystemPropertiesHandler.java create mode 100644 iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/ConfigFileUtils.java create mode 100644 iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/protocol/rest/filter/ApiOriginFilter.java create mode 100644 iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/protocol/rest/filter/AuthorizationFilter.java create mode 100644 iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/protocol/rest/impl/PingApiServiceImpl.java create mode 100644 iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/protocol/rest/v1/impl/AdminApiServiceImpl.java create mode 100644 iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/service/CollectorMBean.java create mode 100644 iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/service/CollectorRestService.java create mode 100644 iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/service/IService.java create mode 100644 iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/service/RegisterManager.java create mode 100644 iotdb-collector/collector-core/src/main/resources/iotdb-collector-system.properties create mode 100644 iotdb-collector/collector-openapi/pom.xml create mode 100644 iotdb-collector/collector-openapi/src/main/openapi3/iotdb_collector_rest_common.yaml create mode 100644 iotdb-collector/collector-openapi/src/main/openapi3/iotdb_collector_rest_v1.yaml create mode 100644 iotdb-collector/pom.xml diff --git a/iotdb-collector/collector-core/pom.xml b/iotdb-collector/collector-core/pom.xml new file mode 100644 index 000000000000..820c73a54526 --- /dev/null +++ b/iotdb-collector/collector-core/pom.xml @@ -0,0 +1,63 @@ + + + + 4.0.0 + + org.apache.iotdb + iotdb-collector + 2.0.0-SNAPSHOT + + collector-core + IoTDB: Collector: Core + + + org.apache.iotdb + collector-openapi + 2.0.0-SNAPSHOT + + + org.eclipse.jetty + jetty-http + + + org.eclipse.jetty + jetty-util + + + org.eclipse.jetty + jetty-server + + + org.eclipse.jetty + jetty-servlet + + + org.glassfish.jersey.containers + jersey-container-servlet-core + + + org.glassfish.jersey.inject + jersey-hk2 + runtime + + + diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/Collector.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/Collector.java new file mode 100644 index 000000000000..af107db6f550 --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/Collector.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector; + +import org.apache.iotdb.collector.config.CollectorConfig; +import org.apache.iotdb.collector.config.CollectorSystemPropertiesHandler; +import org.apache.iotdb.collector.service.CollectorMBean; +import org.apache.iotdb.collector.service.CollectorRestService; +import org.apache.iotdb.collector.service.RegisterManager; +import org.apache.iotdb.commons.ServerCommandLine; +import org.apache.iotdb.commons.exception.IoTDBException; +import org.apache.iotdb.commons.exception.StartupException; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.charset.Charset; +import java.util.Set; + +public class Collector extends ServerCommandLine implements CollectorMBean { + + private static final Logger LOGGER = LoggerFactory.getLogger(Collector.class); + + private static final RegisterManager REGISTER_MANAGER = RegisterManager.getInstance(); + + public Collector() { + super("Collector"); + CollectorHolder.INSTANCE = this; + } + + public static void main(String[] args) { + LOGGER.info( + "IoTDB-Collector environment variables: {}", CollectorConfig.getEnvironmentVariables()); + LOGGER.info("IoTDB-Collector default charset is: {}", Charset.defaultCharset().displayName()); + + final Collector collector = new Collector(); + final int returnCode = collector.run(args); + if (returnCode != 0) { + System.exit(returnCode); + } + } + + @Override + protected void start() { + boolean isFirstStart; + try { + isFirstStart = prepareCollector(); + if (isFirstStart) { + LOGGER.info("Collector is starting for the first time..."); + } else { + LOGGER.info("Collector is restarting..."); + } + + pullAndCheckSystemConfigurations(); + + initProtocols(); + } catch (final StartupException e) { + LOGGER.error("Collector start failed", e); + stop(); + System.exit(-1); + } + } + + private boolean prepareCollector() { + return CollectorSystemPropertiesHandler.getInstance().fileExist(); + } + + private void pullAndCheckSystemConfigurations() { + LOGGER.info("Pulling system configurations from the ConfigNode-leader..."); + } + + private void initProtocols() throws StartupException { + REGISTER_MANAGER.register(CollectorRestService.getInstance()); + } + + private void stop() {} + + @Override + protected void remove(final Set nodeIds) throws IoTDBException { + // empty method + } + + private static class CollectorHolder { + private static Collector INSTANCE; + + private CollectorHolder() { + // empty constructor + } + } +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/CollectorConfig.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/CollectorConfig.java new file mode 100644 index 000000000000..99a40488e552 --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/CollectorConfig.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.config; + +public class CollectorConfig { + + public static final String CONFIG_NAME = "iotdb-collector-system.properties"; + + private int restServicePort = 17070; + + public static String getEnvironmentVariables() { + return ""; + } + + public int getRestServicePort() { + return restServicePort; + } + + public void setRestServicePort(int restServicePort) { + this.restServicePort = restServicePort; + } +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/CollectorConstant.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/CollectorConstant.java new file mode 100644 index 000000000000..ed9bf141a2b1 --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/CollectorConstant.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.config; + +public class CollectorConstant { + public static final String PROPERTIES_FILE_NAME = "system.properties"; +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/CollectorDescriptor.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/CollectorDescriptor.java new file mode 100644 index 000000000000..6046254e64dc --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/CollectorDescriptor.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.config; + +import org.apache.iotdb.commons.conf.TrimProperties; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.util.Optional; +import java.util.Properties; + +public class CollectorDescriptor { + + private static final Logger LOGGER = LoggerFactory.getLogger(CollectorDescriptor.class); + + private static final CollectorConfig CONFIG = new CollectorConfig(); + private static final String CONFIG_NAME = CollectorConfig.CONFIG_NAME; + + static { + final Optional systemConfigUrl = getPropsUrl(); + systemConfigUrl.ifPresent( + url -> { + try { + ConfigFileUtils.checkAndMayUpdate(url); + } catch (final Exception e) { + if (e instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + LOGGER.error("Failed to update config file", e); + } + }); + } + + private static Optional getPropsUrl() { + final URL url = CollectorConfig.class.getResource("/" + CONFIG_NAME); + + if (url != null) { + return Optional.of(url); + } else { + LOGGER.warn( + "Cannot find IOTDB_COLLECTOR_HOME or IOTDB_COLLECTOR_CONF environment variable when loading " + + "config file {}, use default configuration", + CONFIG_NAME); + // TODO update path + // IoTDBConfig: updatePath() + return Optional.empty(); + } + } + + protected CollectorDescriptor() { + loadProps(); + } + + private void loadProps() { + final TrimProperties collectorProperties = new TrimProperties(); + final Optional url = getPropsUrl(); + + if (url.isPresent()) { + try (final InputStream inputStream = url.get().openStream()) { + LOGGER.info("Start to read config file {}", url.get()); + final Properties properties = new Properties(); + properties.load(new InputStreamReader(inputStream, StandardCharsets.UTF_8)); + collectorProperties.putAll(properties); + loadProperties(collectorProperties); + } catch (final FileNotFoundException e) { + LOGGER.error("Fail to find config file {}, reject DataNode startup.", url.get(), e); + System.exit(-1); + } catch (final IOException e) { + LOGGER.error("Cannot load config file, reject DataNode startup.", e); + System.exit(-1); + } catch (final Exception e) { + LOGGER.error("Incorrect format in config file, reject DataNode startup.", e); + System.exit(-1); + } + } else { + LOGGER.warn("Couldn't load the configuration {} from any of the known sources.", CONFIG_NAME); + } + } + + // properties config + private void loadProperties(final TrimProperties properties) { + CONFIG.setRestServicePort( + Integer.parseInt( + Optional.ofNullable(properties.getProperty("collector_rest_port")) + .orElse(String.valueOf(CONFIG.getRestServicePort())))); + } + + public static CollectorDescriptor getInstance() { + return CollectorDescriptorHolder.INSTANCE; + } + + public CollectorConfig getConfig() { + return CONFIG; + } + + private static class CollectorDescriptorHolder { + + private static final CollectorDescriptor INSTANCE = new CollectorDescriptor(); + + private CollectorDescriptorHolder() {} + } +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/CollectorSystemPropertiesHandler.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/CollectorSystemPropertiesHandler.java new file mode 100644 index 000000000000..ff4db04b9951 --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/CollectorSystemPropertiesHandler.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.config; + +import org.apache.iotdb.commons.file.SystemPropertiesHandler; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.apache.iotdb.collector.config.CollectorConstant.PROPERTIES_FILE_NAME; + +public class CollectorSystemPropertiesHandler extends SystemPropertiesHandler { + + private static final Logger LOGGER = + LoggerFactory.getLogger(CollectorSystemPropertiesHandler.class); + + private static CollectorSystemPropertiesHandler INSTANCE; + + public CollectorSystemPropertiesHandler(final String filePath) { + super(filePath); + } + + public static SystemPropertiesHandler getInstance() { + if (INSTANCE == null) { + synchronized (CollectorSystemPropertiesHandler.class) { + INSTANCE = + new CollectorSystemPropertiesHandler( + // TODO System File Folder + PROPERTIES_FILE_NAME); + INSTANCE.init(); + } + } + return INSTANCE; + } +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/ConfigFileUtils.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/ConfigFileUtils.java new file mode 100644 index 000000000000..54c687981dc6 --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/ConfigFileUtils.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.config; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.net.URL; +import java.nio.file.Files; +import java.util.StringJoiner; +import java.util.concurrent.TimeUnit; + +public class ConfigFileUtils { + + private static final String lockFileSuffix = ".lock"; + private static final long maxTimeMillsToAcquireLock = TimeUnit.SECONDS.toMillis(20); + private static final long waitTimeMillsPerCheck = TimeUnit.MILLISECONDS.toMillis(100); + private static final Logger LOGGER = LoggerFactory.getLogger(ConfigFileUtils.class); + private static final String lineSeparator = "\n"; + private static final String license = + new StringJoiner(lineSeparator) + .add("# Licensed to the Apache Software Foundation (ASF) under one") + .add("# or more contributor license agreements. See the NOTICE file") + .add("# distributed with this work for additional information") + .add("# regarding copyright ownership. The ASF licenses this file") + .add("# to you under the Apache License, Version 2.0 (the") + .add("# \"License\"); you may not use this file except in compliance") + .add("# with the License. You may obtain a copy of the License at") + .add("#") + .add("# http://www.apache.org/licenses/LICENSE-2.0") + .add("#") + .add("# Unless required by applicable law or agreed to in writing,") + .add("# software distributed under the License is distributed on an") + .add("# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY") + .add("# KIND, either express or implied. See the License for the") + .add("# specific language governing permissions and limitations") + .add("# under the License.") + .toString(); + + public static void checkAndMayUpdate(final URL url) throws IOException, InterruptedException { + final File systemFile = new File(url.getFile()); + if (systemFile.exists()) { + return; + } + final File lockFile = new File(systemFile.getAbsolutePath() + lockFileSuffix); + acquireTargetFileLock(lockFile); + try { + if (systemFile.exists()) { + return; + } + try (final RandomAccessFile raf = new RandomAccessFile(lockFile, "rw")) { + raf.write(license.getBytes()); + } + Files.move(lockFile.toPath(), systemFile.toPath()); + } finally { + releaseFileLock(lockFile); + } + } + + private static void acquireTargetFileLock(final File file) + throws IOException, InterruptedException { + long totalWaitTime = 0; + while (totalWaitTime < maxTimeMillsToAcquireLock) { + if (file.createNewFile()) { + return; + } + totalWaitTime += waitTimeMillsPerCheck; + Thread.sleep(waitTimeMillsPerCheck); + } + LOGGER.warn( + "Waiting for {} seconds to acquire configuration file update lock." + + " There may have been an unexpected interruption in the last" + + " configuration file update. Ignore temporary file {}", + totalWaitTime / 1000, + file.getName()); + } + + private static void releaseFileLock(final File file) throws IOException { + Files.deleteIfExists(file.toPath()); + } +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/protocol/rest/filter/ApiOriginFilter.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/protocol/rest/filter/ApiOriginFilter.java new file mode 100644 index 000000000000..179836fec6eb --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/protocol/rest/filter/ApiOriginFilter.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.protocol.rest.filter; + +import javax.servlet.FilterChain; +import javax.servlet.FilterConfig; +import javax.servlet.ServletException; +import javax.servlet.ServletRequest; +import javax.servlet.ServletResponse; +import javax.servlet.http.HttpServletResponse; + +import java.io.IOException; + +public class ApiOriginFilter implements javax.servlet.Filter { + @Override + public void doFilter( + final ServletRequest request, final ServletResponse response, final FilterChain chain) + throws IOException, ServletException { + final HttpServletResponse res = (HttpServletResponse) response; + res.addHeader("Access-Control-Allow-Origin", "*"); + res.addHeader("Access-Control-Allow-Methods", "GET, POST"); + res.addHeader("Access-Control-Allow-Headers", "*"); + chain.doFilter(request, response); + } + + @Override + public void destroy() { + // do nothing + } + + @Override + public void init(final FilterConfig filterConfig) throws ServletException { + // do nothing + } +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/protocol/rest/filter/AuthorizationFilter.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/protocol/rest/filter/AuthorizationFilter.java new file mode 100644 index 000000000000..93524743bb9d --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/protocol/rest/filter/AuthorizationFilter.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.protocol.rest.filter; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.servlet.annotation.WebFilter; +import javax.ws.rs.container.ContainerRequestContext; +import javax.ws.rs.container.ContainerRequestFilter; +import javax.ws.rs.container.ContainerResponseContext; +import javax.ws.rs.container.ContainerResponseFilter; +import javax.ws.rs.ext.Provider; + +import java.io.IOException; + +@WebFilter("/*") +@Provider +public class AuthorizationFilter implements ContainerRequestFilter, ContainerResponseFilter { + + @Override + public void filter(final ContainerRequestContext containerRequestContext) throws IOException {} + + @Override + public void filter( + final ContainerRequestContext containerRequestContext, + final ContainerResponseContext containerResponseContext) + throws IOException {} +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/protocol/rest/impl/PingApiServiceImpl.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/protocol/rest/impl/PingApiServiceImpl.java new file mode 100644 index 000000000000..8d34d94852f6 --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/protocol/rest/impl/PingApiServiceImpl.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.collector.protocol.rest.impl; + +import org.apache.iotdb.collector.protocol.rest.NotFoundException; +import org.apache.iotdb.collector.protocol.rest.PingApiService; +import org.apache.iotdb.collector.protocol.rest.v1.model.ExecutionStatus; +import org.apache.iotdb.rpc.TSStatusCode; + +import javax.ws.rs.core.Response; +import javax.ws.rs.core.SecurityContext; + +public class PingApiServiceImpl extends PingApiService { + + @Override + public Response tryPing(final SecurityContext securityContext) throws NotFoundException { + return Response.ok() + .entity( + new ExecutionStatus() + .code(TSStatusCode.SUCCESS_STATUS.getStatusCode()) + .message(TSStatusCode.SUCCESS_STATUS.name())) + .build(); + } +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/protocol/rest/v1/impl/AdminApiServiceImpl.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/protocol/rest/v1/impl/AdminApiServiceImpl.java new file mode 100644 index 000000000000..8b9c5ced25b2 --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/protocol/rest/v1/impl/AdminApiServiceImpl.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.protocol.rest.v1.impl; + +import org.apache.iotdb.collector.protocol.rest.v1.AdminApiService; +import org.apache.iotdb.collector.protocol.rest.v1.NotFoundException; +import org.apache.iotdb.collector.protocol.rest.v1.model.AlterPipeRequest; +import org.apache.iotdb.collector.protocol.rest.v1.model.CreatePipeRequest; +import org.apache.iotdb.collector.protocol.rest.v1.model.DropPipeRequest; +import org.apache.iotdb.collector.protocol.rest.v1.model.StartPipeRequest; +import org.apache.iotdb.collector.protocol.rest.v1.model.StopPipeRequest; + +import javax.ws.rs.core.Response; +import javax.ws.rs.core.SecurityContext; + +public class AdminApiServiceImpl extends AdminApiService { + @Override + public Response alterPipe( + final AlterPipeRequest alterPipeRequest, final SecurityContext securityContext) + throws NotFoundException { + return Response.ok("alterPipe").build(); + } + + @Override + public Response createPipe( + final CreatePipeRequest createPipeRequest, final SecurityContext securityContext) + throws NotFoundException { + return Response.ok("createPipe").build(); + } + + @Override + public Response dropPipe( + final DropPipeRequest dropPipeRequest, final SecurityContext securityContext) + throws NotFoundException { + return Response.ok("dropPipe").build(); + } + + @Override + public Response startPipe( + final StartPipeRequest startPipeRequest, final SecurityContext securityContext) + throws NotFoundException { + return Response.ok("startPipe").build(); + } + + @Override + public Response stopPipe( + final StopPipeRequest stopPipeRequest, final SecurityContext securityContext) + throws NotFoundException { + return Response.ok("stopPipe").build(); + } +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/service/CollectorMBean.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/service/CollectorMBean.java new file mode 100644 index 000000000000..60809ff37b11 --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/service/CollectorMBean.java @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.collector.service; + +public interface CollectorMBean {} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/service/CollectorRestService.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/service/CollectorRestService.java new file mode 100644 index 000000000000..b8caa7dad997 --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/service/CollectorRestService.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.service; + +import org.apache.iotdb.collector.config.CollectorConfig; +import org.apache.iotdb.collector.config.CollectorDescriptor; +import org.apache.iotdb.collector.protocol.rest.filter.ApiOriginFilter; +import org.apache.iotdb.commons.service.ServiceType; + +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.servlet.ServletContextHandler; +import org.eclipse.jetty.servlet.ServletHolder; +import org.glassfish.jersey.servlet.ServletContainer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.servlet.DispatcherType; + +import java.util.EnumSet; + +public class CollectorRestService implements IService { + + private static final Logger LOGGER = LoggerFactory.getLogger(CollectorRestService.class); + + private static final CollectorConfig CONFIG = CollectorDescriptor.getInstance().getConfig(); + + private static Server server; + + private CollectorRestService() {} + + @Override + public void start() { + startNonSSL(CONFIG.getRestServicePort()); + } + + private void startNonSSL(final int restServicePort) { + server = new Server(restServicePort); + server.setHandler(constructServletContextHandler()); + serverStart(); + } + + private ServletContextHandler constructServletContextHandler() { + final ServletContextHandler context = + new ServletContextHandler(ServletContextHandler.NO_SESSIONS); + context.addFilter( + ApiOriginFilter.class, "/*", EnumSet.of(DispatcherType.INCLUDE, DispatcherType.REQUEST)); + final ServletHolder holder = context.addServlet(ServletContainer.class, "/*"); + holder.setInitOrder(1); + holder.setInitParameter( + "jersey.config.server.provider.packages", + "io.swagger.jaxrs.listing, io.swagger.sample.resource, org.apache.iotdb.collector.protocol.rest"); + holder.setInitParameter( + "jersey.config.server.provider.classnames", + "org.glassfish.jersey.media.multipart.MultiPartFeature"); + holder.setInitParameter("jersey.config.server.wadl.disableWadl", "true"); + context.setContextPath("/"); + return context; + } + + private void serverStart() { + try { + server.start(); + } catch (final Exception e) { + LOGGER.warn("CollectorRestService failed to start: {}", e.getMessage()); + server.destroy(); + } + LOGGER.info("start CollectorRestService successfully"); + } + + @Override + public void stop() { + try { + server.stop(); + } catch (final Exception e) { + LOGGER.warn("CollectorRestService failed to stop: {}", e.getMessage()); + } finally { + server.destroy(); + } + } + + @Override + public ServiceType getID() { + return ServiceType.REST_SERVICE; + } + + public static CollectorRestService getInstance() { + return CollectorRestServiceHolder.INSTANCE; + } + + private static class CollectorRestServiceHolder { + + private static final CollectorRestService INSTANCE = new CollectorRestService(); + + private CollectorRestServiceHolder() {} + } +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/service/IService.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/service/IService.java new file mode 100644 index 000000000000..95755d260852 --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/service/IService.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.collector.service; + +import org.apache.iotdb.commons.exception.ShutdownException; +import org.apache.iotdb.commons.exception.StartupException; +import org.apache.iotdb.commons.service.ServiceType; + +public interface IService { + void start() throws StartupException; + + default void waitAndStop(long milliseconds) { + stop(); + } + + default void shutdown(long milliseconds) throws ShutdownException { + waitAndStop(milliseconds); + } + + void stop(); + + ServiceType getID(); +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/service/RegisterManager.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/service/RegisterManager.java new file mode 100644 index 000000000000..5498ff403ecb --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/service/RegisterManager.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.service; + +import org.apache.iotdb.commons.exception.StartupException; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +public class RegisterManager { + private static final Logger LOGGER = LoggerFactory.getLogger(RegisterManager.class); + private final List iServices; + private static final long DEREGISTER_TIME_OUT = 10_00L; + + private RegisterManager() { + this.iServices = new ArrayList<>(); + } + + /** register service. */ + public void register(final IService service) throws StartupException { + for (final IService iService : iServices) { + if (iService.getID() == service.getID()) { + LOGGER.debug("{} has already been registered. skip", service.getID().getName()); + return; + } + } + iServices.add(service); + final long startTime = System.currentTimeMillis(); + service.start(); + final long endTime = System.currentTimeMillis(); + LOGGER.info( + "The {} service is started successfully, which takes {} ms.", + service.getID().getName(), + (endTime - startTime)); + } + + /** stop all service and clear iService list. */ + public void deregisterAll() { + Collections.reverse(iServices); + for (IService service : iServices) { + try { + service.waitAndStop(DEREGISTER_TIME_OUT); + LOGGER.debug("{} deregistered", service.getID()); + } catch (final Exception e) { + LOGGER.error("Failed to stop {} because:", service.getID().getName(), e); + } + } + iServices.clear(); + LOGGER.info("deregister all service."); + } + + public static RegisterManager getInstance() { + return RegisterManagerHolder.INSTANCE; + } + + private static class RegisterManagerHolder { + private static final RegisterManager INSTANCE = new RegisterManager(); + + private RegisterManagerHolder() {} + } +} diff --git a/iotdb-collector/collector-core/src/main/resources/iotdb-collector-system.properties b/iotdb-collector/collector-core/src/main/resources/iotdb-collector-system.properties new file mode 100644 index 000000000000..bb4189f4f4c1 --- /dev/null +++ b/iotdb-collector/collector-core/src/main/resources/iotdb-collector-system.properties @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +collector_rest_port=17070 \ No newline at end of file diff --git a/iotdb-collector/collector-openapi/pom.xml b/iotdb-collector/collector-openapi/pom.xml new file mode 100644 index 000000000000..ab3de19dca5f --- /dev/null +++ b/iotdb-collector/collector-openapi/pom.xml @@ -0,0 +1,185 @@ + + + + 4.0.0 + + org.apache.iotdb + iotdb-collector + 2.0.0-SNAPSHOT + + collector-openapi + IoTDB: Collector: OpenAPI + + + io.swagger + swagger-annotations + + + io.swagger + swagger-models + + + com.fasterxml.jackson.core + jackson-annotations + + + com.fasterxml.jackson.jaxrs + jackson-jaxrs-json-provider + + + com.fasterxml.jackson.core + jackson-databind + + + com.fasterxml.jackson.datatype + jackson-datatype-jsr310 + + + jakarta.validation + jakarta.validation-api + + + jakarta.ws.rs + jakarta.ws.rs-api + + + io.swagger + swagger-jaxrs + + + + org.glassfish.jersey.media + jersey-media-multipart + + + jakarta.servlet + jakarta.servlet-api + + + + + + + org.openapitools + openapi-generator-maven-plugin + + + generate-java-rest-codes-common + + generate + + generate-sources + + ${project.basedir}/src/main/openapi3/iotdb_collector_rest_common.yaml + ${project.build.directory}/generated-sources/java + org.apache.iotdb.collector.protocol.rest + org.apache.iotdb.collector.protocol.rest.model + org.apache.iotdb.collector.protocol.rest.invoker + jaxrs-jersey + org.apache.iotdb. + iotdb-collector-rest-service + ${project.version} + true + + Apache License 2.0 + org.apache.iotdb + iotdb-collector-rest-service + ${project.version} + java8 + true + + + + + generate-java-rest-codes-v1 + + generate + + generate-sources + + ${project.basedir}/src/main/openapi3/iotdb_collector_rest_v1.yaml + ${project.build.directory}/generated-sources/java + org.apache.iotdb.collector.protocol.rest.v1 + org.apache.iotdb.collector.protocol.rest.v1.model + org.apache.iotdb.collector.protocol.rest.v1.invoker + jaxrs-jersey + org.apache.iotdb + iotdb-collector-rest-service + ${project.version} + true + + Apache License 2.0 + org.apache.iotdb + iotdb-collector-rest-service + ${project.version} + java8 + true + + + + + + + org.codehaus.mojo + build-helper-maven-plugin + + + add-source + + add-source + + generate-sources + + + ${project.basedir}/target/generated-sources/java/src/gen/java + + + + + + + + org.apache.maven.plugins + maven-jar-plugin + + + **/impl/** + + + + + org.apache.maven.plugins + maven-dependency-plugin + + + + org.glassfish.jersey.media:jersey-media-multipart + + + + + + diff --git a/iotdb-collector/collector-openapi/src/main/openapi3/iotdb_collector_rest_common.yaml b/iotdb-collector/collector-openapi/src/main/openapi3/iotdb_collector_rest_common.yaml new file mode 100644 index 000000000000..a81a242ac878 --- /dev/null +++ b/iotdb-collector/collector-openapi/src/main/openapi3/iotdb_collector_rest_common.yaml @@ -0,0 +1,63 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +openapi: 3.0.0 +info: + title: iotdb_collector_rest_common + description: IoTDB Rest API for Collector + license: + name: Apache 2.0 + url: https://www.apache.org/licenses/LICENSE-2.0.html + version: 1.0.0 +servers: +- url: http://127.0.0.1:17070/ + description: api +security: +- basic: [] +paths: + /ping: + get: + responses: + "200": + description: ExecutionStatus + content: + application/json: + schema: + $ref: '#/components/schemas/ExecutionStatus' + operationId: tryPing + +components: + schemas: + ExecutionStatus: + type: object + properties: + code: + type: integer + format: int32 + message: + type: string + + securitySchemes: + basic: + type: http + scheme: basic + APIKey: + type: apiKey + name: API Key + in: header diff --git a/iotdb-collector/collector-openapi/src/main/openapi3/iotdb_collector_rest_v1.yaml b/iotdb-collector/collector-openapi/src/main/openapi3/iotdb_collector_rest_v1.yaml new file mode 100644 index 000000000000..85802287ffa0 --- /dev/null +++ b/iotdb-collector/collector-openapi/src/main/openapi3/iotdb_collector_rest_v1.yaml @@ -0,0 +1,282 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +openapi: 3.0.0 +info: + title: iotdb-collector-rest + description: IoTDB Rest API for Collector + license: + name: Apache 2.0 + url: https://www.apache.org/licenses/LICENSE-2.0.html + version: 1.0.0 +servers: + - url: http://127.0.0.1:17070/ + description: api +security: + - basic: [] +paths: + /admin/v1/createPipe: + post: + summary: createPipe + description: createPipe + operationId: createPipe + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/CreatePipeRequest' + responses: + "200": + description: ExecutionStatus + content: + application/json: + schema: + $ref: '#/components/schemas/ExecutionStatus' + + /admin/v1/alterPipe: + post: + summary: alterPipe + description: alterPipe + operationId: alterPipe + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/AlterPipeRequest' + responses: + "200": + description: ExecutionStatus + content: + application/json: + schema: + $ref: '#/components/schemas/ExecutionStatus' + + /admin/v1/startPipe: + post: + summary: startPipe + description: startPipe + operationId: startPipe + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/StartPipeRequest' + responses: + "200": + description: ExecutionStatus + content: + application/json: + schema: + $ref: '#/components/schemas/ExecutionStatus' + + /admin/v1/stopPipe: + post: + summary: stopPipe + description: stopPipe + operationId: stopPipe + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/StopPipeRequest' + responses: + "200": + description: ExecutionStatus + content: + application/json: + schema: + $ref: '#/components/schemas/ExecutionStatus' + + /admin/v1/dropPipe: + post: + summary: dropPipe + description: dropPipe + operationId: dropPipe + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/DropPipeRequest' + responses: + "200": + description: ExecutionStatus + content: + application/json: + schema: + $ref: '#/components/schemas/ExecutionStatus' + +components: + schemas: + CreatePipeRequest: + title: CreatePipeRequest + type: object + properties: + timestamps: + type: array + items: + type: integer + format: int64 + measurements: + type: array + items: + type: string + dataTypes: + type: array + items: + type: string + values: + type: array + items: + type: array + items: + type: object + isAligned: + type: boolean + deviceId: + type: string + + AlterPipeRequest: + title: AlterPipeRequest + type: object + properties: + timestamps: + type: array + items: + type: integer + format: int64 + measurements: + type: array + items: + type: string + dataTypes: + type: array + items: + type: string + values: + type: array + items: + type: array + items: + type: object + isAligned: + type: boolean + deviceId: + type: string + + StartPipeRequest: + title: StartPipeRequest + type: object + properties: + timestamps: + type: array + items: + type: integer + format: int64 + measurements: + type: array + items: + type: string + dataTypes: + type: array + items: + type: string + values: + type: array + items: + type: array + items: + type: object + isAligned: + type: boolean + deviceId: + type: string + + StopPipeRequest: + title: StopPipeRequest + type: object + properties: + timestamps: + type: array + items: + type: integer + format: int64 + measurements: + type: array + items: + type: string + dataTypes: + type: array + items: + type: string + values: + type: array + items: + type: array + items: + type: object + isAligned: + type: boolean + deviceId: + type: string + + DropPipeRequest: + title: DropPipeRequest + type: object + properties: + timestamps: + type: array + items: + type: integer + format: int64 + measurements: + type: array + items: + type: string + dataTypes: + type: array + items: + type: string + values: + type: array + items: + type: array + items: + type: object + isAligned: + type: boolean + deviceId: + type: string + + ExecutionStatus: + type: object + properties: + code: + type: integer + format: int32 + message: + type: string + + securitySchemes: + basic: + type: http + scheme: basic + APIKey: + type: apiKey + name: API Key + in: header diff --git a/iotdb-collector/pom.xml b/iotdb-collector/pom.xml new file mode 100644 index 000000000000..261314f254e0 --- /dev/null +++ b/iotdb-collector/pom.xml @@ -0,0 +1,42 @@ + + + + 4.0.0 + + org.apache.iotdb + iotdb-parent + 2.0.0-SNAPSHOT + + iotdb-collector + pom + IoTDB: Collector + + collector-openapi + + + + org.apache.iotdb + node-commons + 2.0.0-SNAPSHOT + + + diff --git a/pom.xml b/pom.xml index e278d579b445..5dc5480caef4 100644 --- a/pom.xml +++ b/pom.xml @@ -35,6 +35,7 @@ iotdb-api iotdb-client + iotdb-collector iotdb-core iotdb-protocol distribution From 4e93faa006262c82f8785af6b086b81cd7385c8f Mon Sep 17 00:00:00 2001 From: 0xB <76414902+YC27@users.noreply.github.com> Date: Mon, 20 Jan 2025 09:31:24 +0800 Subject: [PATCH 02/10] CollectorNode built-in data acquisition plug-in pipeline simple implementation. (#14725) * CollectorNode built-in data acquisition plug-in pipeline implementation * update licensed --- .../org/apache/iotdb/collector/Collector.java | 50 ++------ .../iotdb/collector/agent/CollectorAgent.java | 50 ++++++++ .../collect/CollectorEventCollector.java} | 36 +++--- .../CollectorProcessorTaskExecutor.java | 68 ++++++++++ .../executor/CollectorSinkTaskExecutor.java | 67 ++++++++++ .../executor/CollectorSourceTaskExecutor.java | 67 ++++++++++ .../agent/executor/CollectorTaskExecutor.java | 63 ++++++++++ .../executor/CollectorTaskExecutorAgent.java | 53 ++++++++ .../agent/plugin/CollectorPluginAgent.java | 39 ++++++ .../plugin/CollectorPluginConstructor.java | 77 ++++++++++++ .../agent/task/CollectorProcessorTask.java | 88 +++++++++++++ .../agent/task/CollectorSinkTask.java | 87 +++++++++++++ .../agent/task/CollectorSourceTask.java | 69 +++++++++++ .../task/CollectorTask.java} | 19 ++- .../agent/task/CollectorTaskAgent.java | 117 ++++++++++++++++++ .../collector/config/CollectorConfig.java | 42 ++++++- .../collector/config/CollectorDescriptor.java | 62 ++++------ .../CollectorSystemPropertiesHandler.java | 52 -------- .../collector/config/ConfigFileUtils.java | 101 --------------- .../plugin/BuiltinCollectorPlugin.java | 74 +++++++++++ .../builtin/processor/DoNothingProcessor.java | 64 ++++++++++ .../plugin/builtin/sink/SessionSink.java | 68 ++++++++++ .../plugin/builtin/source/HttpSource.java | 79 ++++++++++++ .../builtin/source/event/SourceEvent.java | 45 +++++++ .../v1/handler/RequestValidationHandler.java | 37 ++++++ .../rest/v1/impl/AdminApiServiceImpl.java | 34 ++++- .../collector/service/RegisterManager.java | 4 +- .../iotdb-collector-system.properties | 2 +- .../openapi3/iotdb_collector_rest_v1.yaml | 55 ++------ 29 files changed, 1372 insertions(+), 297 deletions(-) create mode 100644 iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/agent/CollectorAgent.java rename iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/{protocol/rest/filter/AuthorizationFilter.java => agent/collect/CollectorEventCollector.java} (53%) create mode 100644 iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/agent/executor/CollectorProcessorTaskExecutor.java create mode 100644 iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/agent/executor/CollectorSinkTaskExecutor.java create mode 100644 iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/agent/executor/CollectorSourceTaskExecutor.java create mode 100644 iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/agent/executor/CollectorTaskExecutor.java create mode 100644 iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/agent/executor/CollectorTaskExecutorAgent.java create mode 100644 iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/agent/plugin/CollectorPluginAgent.java create mode 100644 iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/agent/plugin/CollectorPluginConstructor.java create mode 100644 iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/agent/task/CollectorProcessorTask.java create mode 100644 iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/agent/task/CollectorSinkTask.java create mode 100644 iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/agent/task/CollectorSourceTask.java rename iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/{config/CollectorConstant.java => agent/task/CollectorTask.java} (68%) create mode 100644 iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/agent/task/CollectorTaskAgent.java delete mode 100644 iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/CollectorSystemPropertiesHandler.java delete mode 100644 iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/ConfigFileUtils.java create mode 100644 iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/BuiltinCollectorPlugin.java create mode 100644 iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/processor/DoNothingProcessor.java create mode 100644 iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/SessionSink.java create mode 100644 iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/source/HttpSource.java create mode 100644 iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/source/event/SourceEvent.java create mode 100644 iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/protocol/rest/v1/handler/RequestValidationHandler.java diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/Collector.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/Collector.java index af107db6f550..4aaa60f3689c 100644 --- a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/Collector.java +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/Collector.java @@ -20,12 +20,11 @@ package org.apache.iotdb.collector; import org.apache.iotdb.collector.config.CollectorConfig; -import org.apache.iotdb.collector.config.CollectorSystemPropertiesHandler; +import org.apache.iotdb.collector.config.CollectorDescriptor; import org.apache.iotdb.collector.service.CollectorMBean; import org.apache.iotdb.collector.service.CollectorRestService; import org.apache.iotdb.collector.service.RegisterManager; import org.apache.iotdb.commons.ServerCommandLine; -import org.apache.iotdb.commons.exception.IoTDBException; import org.apache.iotdb.commons.exception.StartupException; import org.slf4j.Logger; @@ -39,16 +38,18 @@ public class Collector extends ServerCommandLine implements CollectorMBean { private static final Logger LOGGER = LoggerFactory.getLogger(Collector.class); private static final RegisterManager REGISTER_MANAGER = RegisterManager.getInstance(); + private static final CollectorConfig COLLECTOR_CONFIG = + CollectorDescriptor.getInstance().getConfig(); - public Collector() { + private Collector() { super("Collector"); - CollectorHolder.INSTANCE = this; } public static void main(String[] args) { LOGGER.info( - "IoTDB-Collector environment variables: {}", CollectorConfig.getEnvironmentVariables()); - LOGGER.info("IoTDB-Collector default charset is: {}", Charset.defaultCharset().displayName()); + "IoTDB-CollectorNode environment variables: {}", CollectorConfig.getEnvironmentVariables()); + LOGGER.info( + "IoTDB-CollectorNode default charset is: {}", Charset.defaultCharset().displayName()); final Collector collector = new Collector(); final int returnCode = collector.run(args); @@ -59,49 +60,24 @@ public static void main(String[] args) { @Override protected void start() { - boolean isFirstStart; try { - isFirstStart = prepareCollector(); - if (isFirstStart) { - LOGGER.info("Collector is starting for the first time..."); - } else { - LOGGER.info("Collector is restarting..."); - } - - pullAndCheckSystemConfigurations(); - initProtocols(); + + LOGGER.info("IoTDB-CollectorNode configuration: {}", COLLECTOR_CONFIG.getConfigMessage()); + LOGGER.info( + "Congratulations, IoTDB CollectorNode is set up successfully. Now, enjoy yourself!"); } catch (final StartupException e) { - LOGGER.error("Collector start failed", e); - stop(); + LOGGER.error("CollectorNode start failed", e); System.exit(-1); } } - private boolean prepareCollector() { - return CollectorSystemPropertiesHandler.getInstance().fileExist(); - } - - private void pullAndCheckSystemConfigurations() { - LOGGER.info("Pulling system configurations from the ConfigNode-leader..."); - } - private void initProtocols() throws StartupException { REGISTER_MANAGER.register(CollectorRestService.getInstance()); } - private void stop() {} - @Override - protected void remove(final Set nodeIds) throws IoTDBException { + protected void remove(final Set nodeIds) { // empty method } - - private static class CollectorHolder { - private static Collector INSTANCE; - - private CollectorHolder() { - // empty constructor - } - } } diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/agent/CollectorAgent.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/agent/CollectorAgent.java new file mode 100644 index 000000000000..bca4106987ff --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/agent/CollectorAgent.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.agent; + +import org.apache.iotdb.collector.agent.executor.CollectorTaskExecutorAgent; +import org.apache.iotdb.collector.agent.plugin.CollectorPluginAgent; +import org.apache.iotdb.collector.agent.task.CollectorTaskAgent; + +public class CollectorAgent { + + private final CollectorTaskAgent collectorTaskAgent = CollectorTaskAgent.instance(); + private final CollectorTaskExecutorAgent collectorTaskExecutorAgent = + CollectorTaskExecutorAgent.instance(); + private final CollectorPluginAgent collectorPluginAgent = CollectorPluginAgent.instance(); + + private CollectorAgent() {} + + public static CollectorTaskAgent task() { + return CollectorAgentHolder.INSTANCE.collectorTaskAgent; + } + + public static CollectorTaskExecutorAgent executor() { + return CollectorAgentHolder.INSTANCE.collectorTaskExecutorAgent; + } + + public static CollectorPluginAgent plugin() { + return CollectorAgentHolder.INSTANCE.collectorPluginAgent; + } + + private static class CollectorAgentHolder { + private static final CollectorAgent INSTANCE = new CollectorAgent(); + } +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/protocol/rest/filter/AuthorizationFilter.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/agent/collect/CollectorEventCollector.java similarity index 53% rename from iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/protocol/rest/filter/AuthorizationFilter.java rename to iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/agent/collect/CollectorEventCollector.java index 93524743bb9d..056685e3313a 100644 --- a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/protocol/rest/filter/AuthorizationFilter.java +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/agent/collect/CollectorEventCollector.java @@ -17,30 +17,32 @@ * under the License. */ -package org.apache.iotdb.collector.protocol.rest.filter; +package org.apache.iotdb.collector.agent.collect; + +import org.apache.iotdb.pipe.api.collector.EventCollector; +import org.apache.iotdb.pipe.api.event.Event; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.servlet.annotation.WebFilter; -import javax.ws.rs.container.ContainerRequestContext; -import javax.ws.rs.container.ContainerRequestFilter; -import javax.ws.rs.container.ContainerResponseContext; -import javax.ws.rs.container.ContainerResponseFilter; -import javax.ws.rs.ext.Provider; +import java.util.concurrent.BlockingQueue; -import java.io.IOException; +public class CollectorEventCollector implements EventCollector { -@WebFilter("/*") -@Provider -public class AuthorizationFilter implements ContainerRequestFilter, ContainerResponseFilter { + private static final Logger LOGGER = LoggerFactory.getLogger(CollectorEventCollector.class); - @Override - public void filter(final ContainerRequestContext containerRequestContext) throws IOException {} + private final BlockingQueue pendingQueue; + + public CollectorEventCollector(final BlockingQueue pendingQueue) { + this.pendingQueue = pendingQueue; + } @Override - public void filter( - final ContainerRequestContext containerRequestContext, - final ContainerResponseContext containerResponseContext) - throws IOException {} + public void collect(final Event event) { + try { + pendingQueue.put(event); + } catch (final InterruptedException e) { + LOGGER.warn("collect event failed because {}", e.getMessage(), e); + } + } } diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/agent/executor/CollectorProcessorTaskExecutor.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/agent/executor/CollectorProcessorTaskExecutor.java new file mode 100644 index 000000000000..385fd606628c --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/agent/executor/CollectorProcessorTaskExecutor.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.agent.executor; + +import org.apache.iotdb.collector.agent.task.CollectorTask; +import org.apache.iotdb.commons.concurrent.IoTDBThreadPoolFactory; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; + +public class CollectorProcessorTaskExecutor extends CollectorTaskExecutor { + + private static final Logger LOGGER = + LoggerFactory.getLogger(CollectorProcessorTaskExecutor.class); + + private static final Map PROCESSOR_EXECUTOR = new ConcurrentHashMap<>(); + private static final Map PROCESSOR_TASK_MAP = new ConcurrentHashMap<>(); + + public boolean validateIfAbsent(final String taskId) { + return !PROCESSOR_EXECUTOR.containsKey(taskId) && !PROCESSOR_TASK_MAP.containsKey(taskId); + } + + @Override + public Optional getExecutor(final String taskId) { + return Optional.of( + IoTDBThreadPoolFactory.newSingleThreadExecutor("collector-processor-executor-" + taskId)); + } + + @Override + public void recordExecution( + final CollectorTask collectorTask, final ExecutorService executorService) { + final String taskId = collectorTask.getTaskId(); + PROCESSOR_EXECUTOR.putIfAbsent(taskId, executorService); + PROCESSOR_TASK_MAP.putIfAbsent(taskId, collectorTask); + + LOGGER.info("register collector processor task {}", taskId); + } + + @Override + public void eraseExecution(final String taskId) { + PROCESSOR_TASK_MAP.remove(taskId).stop(); + PROCESSOR_EXECUTOR.remove(taskId).shutdownNow(); + + LOGGER.info("deregister collector processor task {}", taskId); + } +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/agent/executor/CollectorSinkTaskExecutor.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/agent/executor/CollectorSinkTaskExecutor.java new file mode 100644 index 000000000000..a490e614223c --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/agent/executor/CollectorSinkTaskExecutor.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.agent.executor; + +import org.apache.iotdb.collector.agent.task.CollectorTask; +import org.apache.iotdb.commons.concurrent.IoTDBThreadPoolFactory; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; + +public class CollectorSinkTaskExecutor extends CollectorTaskExecutor { + + private static final Logger LOGGER = LoggerFactory.getLogger(CollectorSinkTaskExecutor.class); + + private static final Map SINK_EXECUTOR = new ConcurrentHashMap<>(); + private static final Map SINK_TASK_MAP = new ConcurrentHashMap<>(); + + public boolean validateIfAbsent(final String taskId) { + return !SINK_EXECUTOR.containsKey(taskId) && !SINK_TASK_MAP.containsKey(taskId); + } + + @Override + public Optional getExecutor(final String taskId) { + return Optional.of( + IoTDBThreadPoolFactory.newSingleThreadExecutor("collector-sink-executor-" + taskId)); + } + + @Override + public void recordExecution( + final CollectorTask collectorTask, final ExecutorService executorService) { + final String taskId = collectorTask.getTaskId(); + SINK_EXECUTOR.putIfAbsent(taskId, executorService); + SINK_TASK_MAP.putIfAbsent(taskId, collectorTask); + + LOGGER.info("register collector sink task {}", taskId); + } + + @Override + public void eraseExecution(final String taskId) { + SINK_TASK_MAP.remove(taskId).stop(); + SINK_EXECUTOR.remove(taskId).shutdownNow(); + + LOGGER.info("deregister collector sink task {}", taskId); + } +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/agent/executor/CollectorSourceTaskExecutor.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/agent/executor/CollectorSourceTaskExecutor.java new file mode 100644 index 000000000000..fde823694d4a --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/agent/executor/CollectorSourceTaskExecutor.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.agent.executor; + +import org.apache.iotdb.collector.agent.task.CollectorTask; +import org.apache.iotdb.commons.concurrent.IoTDBThreadPoolFactory; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; + +public class CollectorSourceTaskExecutor extends CollectorTaskExecutor { + + private static final Logger LOGGER = LoggerFactory.getLogger(CollectorSourceTaskExecutor.class); + + private static final Map SOURCE_EXECUTOR = new ConcurrentHashMap<>(); + private static final Map SOURCE_TASK_MAP = new ConcurrentHashMap<>(); + + public boolean validateIfAbsent(final String taskId) { + return !SOURCE_EXECUTOR.containsKey(taskId) && !SOURCE_TASK_MAP.containsKey(taskId); + } + + @Override + public Optional getExecutor(final String taskId) { + return Optional.of( + IoTDBThreadPoolFactory.newSingleThreadExecutor("collector-source-executor-" + taskId)); + } + + @Override + public void recordExecution( + final CollectorTask collectorTask, final ExecutorService executorService) { + final String taskId = collectorTask.getTaskId(); + SOURCE_EXECUTOR.put(taskId, executorService); + SOURCE_TASK_MAP.putIfAbsent(taskId, collectorTask); + + LOGGER.info("register collector source task {}", taskId); + } + + @Override + public void eraseExecution(String taskId) { + SOURCE_TASK_MAP.remove(taskId).stop(); + SOURCE_EXECUTOR.remove(taskId).shutdownNow(); + + LOGGER.info("deregister collector source task {}", taskId); + } +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/agent/executor/CollectorTaskExecutor.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/agent/executor/CollectorTaskExecutor.java new file mode 100644 index 000000000000..51dc5675d373 --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/agent/executor/CollectorTaskExecutor.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.agent.executor; + +import org.apache.iotdb.collector.agent.task.CollectorTask; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Optional; +import java.util.concurrent.ExecutorService; + +public abstract class CollectorTaskExecutor { + + private static final Logger LOGGER = LoggerFactory.getLogger(CollectorTaskExecutor.class); + + public void register(final CollectorTask collectorTask) { + if (validateIfAbsent(collectorTask.getTaskId())) { + getExecutor(collectorTask.getTaskId()) + .ifPresent( + executor -> { + executor.submit(collectorTask); + recordExecution(collectorTask, executor); + }); + } else { + LOGGER.warn("task {} has existed", collectorTask.getTaskId()); + } + } + + public abstract boolean validateIfAbsent(final String taskId); + + public abstract Optional getExecutor(final String taskId); + + public abstract void recordExecution( + final CollectorTask collectorTask, final ExecutorService executorService); + + public void deregister(final String taskId) { + if (!validateIfAbsent(taskId)) { + eraseExecution(taskId); + } else { + LOGGER.warn("task {} has not existed", taskId); + } + } + + public abstract void eraseExecution(final String taskId); +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/agent/executor/CollectorTaskExecutorAgent.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/agent/executor/CollectorTaskExecutorAgent.java new file mode 100644 index 000000000000..5adcacebae78 --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/agent/executor/CollectorTaskExecutorAgent.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.agent.executor; + +public class CollectorTaskExecutorAgent { + + private final CollectorSourceTaskExecutor sourceTaskExecutor; + private final CollectorProcessorTaskExecutor processorTaskExecutor; + private final CollectorSinkTaskExecutor sinkTaskExecutor; + + private CollectorTaskExecutorAgent() { + sourceTaskExecutor = new CollectorSourceTaskExecutor(); + processorTaskExecutor = new CollectorProcessorTaskExecutor(); + sinkTaskExecutor = new CollectorSinkTaskExecutor(); + } + + public CollectorSourceTaskExecutor getSourceTaskExecutor() { + return CollectorTaskExecutorAgentHolder.INSTANCE.sourceTaskExecutor; + } + + public CollectorProcessorTaskExecutor getProcessorTaskExecutor() { + return CollectorTaskExecutorAgentHolder.INSTANCE.processorTaskExecutor; + } + + public CollectorSinkTaskExecutor getSinkTaskExecutor() { + return CollectorTaskExecutorAgentHolder.INSTANCE.sinkTaskExecutor; + } + + public static CollectorTaskExecutorAgent instance() { + return CollectorTaskExecutorAgentHolder.INSTANCE; + } + + private static class CollectorTaskExecutorAgentHolder { + private static final CollectorTaskExecutorAgent INSTANCE = new CollectorTaskExecutorAgent(); + } +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/agent/plugin/CollectorPluginAgent.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/agent/plugin/CollectorPluginAgent.java new file mode 100644 index 000000000000..33ee5cae0320 --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/agent/plugin/CollectorPluginAgent.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.agent.plugin; + +public class CollectorPluginAgent { + private final CollectorPluginConstructor collectorPluginConstructor = + CollectorPluginConstructor.instance(); + + private CollectorPluginAgent() {} + + public CollectorPluginConstructor constructor() { + return CollectorPluginAgentHolder.INSTANCE.collectorPluginConstructor; + } + + public static CollectorPluginAgent instance() { + return new CollectorPluginAgent(); + } + + private static class CollectorPluginAgentHolder { + private static final CollectorPluginAgent INSTANCE = new CollectorPluginAgent(); + } +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/agent/plugin/CollectorPluginConstructor.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/agent/plugin/CollectorPluginConstructor.java new file mode 100644 index 000000000000..cce94f649bcf --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/agent/plugin/CollectorPluginConstructor.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.agent.plugin; + +import org.apache.iotdb.collector.plugin.BuiltinCollectorPlugin; +import org.apache.iotdb.collector.plugin.builtin.processor.DoNothingProcessor; +import org.apache.iotdb.collector.plugin.builtin.sink.SessionSink; +import org.apache.iotdb.collector.plugin.builtin.source.HttpSource; +import org.apache.iotdb.pipe.api.PipePlugin; +import org.apache.iotdb.pipe.api.PipeProcessor; +import org.apache.iotdb.pipe.api.PipeSink; +import org.apache.iotdb.pipe.api.PipeSource; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.HashMap; +import java.util.Map; +import java.util.function.Supplier; + +public class CollectorPluginConstructor { + private static final Logger LOGGER = LoggerFactory.getLogger(CollectorPluginConstructor.class); + + protected final Map> pluginConstructors = new HashMap<>(); + + private CollectorPluginConstructor() { + initConstructors(); + } + + private void initConstructors() { + pluginConstructors.put( + BuiltinCollectorPlugin.HTTP_SOURCE.getCollectorPluginName(), HttpSource::new); + pluginConstructors.put( + BuiltinCollectorPlugin.DO_NOTHING_PROCESSOR.getCollectorPluginName(), + DoNothingProcessor::new); + pluginConstructors.put( + BuiltinCollectorPlugin.IOTDB_SESSION_SINK.getCollectorPluginName(), SessionSink::new); + LOGGER.info("builtin plugin has been initialized"); + } + + public PipeSource getSource(final String pluginName) { + return (PipeSource) pluginConstructors.get(pluginName).get(); + } + + public PipeProcessor getProcessor(final String pluginName) { + return (PipeProcessor) pluginConstructors.get(pluginName).get(); + } + + public PipeSink getSink(final String pluginName) { + return (PipeSink) pluginConstructors.get(pluginName).get(); + } + + public static CollectorPluginConstructor instance() { + return CollectorPluginConstructorHolder.INSTANCE; + } + + private static class CollectorPluginConstructorHolder { + private static final CollectorPluginConstructor INSTANCE = new CollectorPluginConstructor(); + } +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/agent/task/CollectorProcessorTask.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/agent/task/CollectorProcessorTask.java new file mode 100644 index 000000000000..8ee62d5e7313 --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/agent/task/CollectorProcessorTask.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.agent.task; + +import org.apache.iotdb.collector.agent.collect.CollectorEventCollector; +import org.apache.iotdb.commons.pipe.agent.task.connection.EventSupplier; +import org.apache.iotdb.pipe.api.PipeProcessor; +import org.apache.iotdb.pipe.api.event.Event; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Map; +import java.util.concurrent.BlockingQueue; + +public class CollectorProcessorTask extends CollectorTask { + + private static final Logger LOGGER = LoggerFactory.getLogger(CollectorProcessorTask.class); + + private final Map processorAttribute; + private final PipeProcessor pipeProcessor; + private final EventSupplier eventSupplier; + private final BlockingQueue pendingQueue; + private final CollectorEventCollector collectorEventCollector; + private boolean isStarted = true; + + public CollectorProcessorTask( + final String taskId, + final Map processorAttribute, + final PipeProcessor pipeProcessor, + final EventSupplier eventSupplier, + final BlockingQueue pendingQueue) { + super(taskId); + this.processorAttribute = processorAttribute; + this.pipeProcessor = pipeProcessor; + this.eventSupplier = eventSupplier; + this.pendingQueue = pendingQueue; + this.collectorEventCollector = new CollectorEventCollector(pendingQueue); + } + + @Override + public void runMayThrow() { + while (isStarted) { + try { + pipeProcessor.process(eventSupplier.supply(), collectorEventCollector); + } catch (final Exception e) { + LOGGER.warn("error occur while processing event because {}", e.getMessage()); + } + } + } + + public Map getProcessorAttribute() { + return processorAttribute; + } + + public PipeProcessor getPipeProcessor() { + return pipeProcessor; + } + + public EventSupplier getEventSupplier() { + return eventSupplier; + } + + public BlockingQueue getPendingQueue() { + return pendingQueue; + } + + public void stop() { + isStarted = false; + } +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/agent/task/CollectorSinkTask.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/agent/task/CollectorSinkTask.java new file mode 100644 index 000000000000..77e65b4ec469 --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/agent/task/CollectorSinkTask.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.agent.task; + +import org.apache.iotdb.pipe.api.PipeSink; +import org.apache.iotdb.pipe.api.event.Event; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Map; +import java.util.concurrent.BlockingQueue; + +public class CollectorSinkTask extends CollectorTask { + + private static final Logger LOGGER = LoggerFactory.getLogger(CollectorSinkTask.class); + + private final Map sinkAttribute; + private final PipeSink pipeSink; + private final BlockingQueue pendingQueue; + private boolean isStarted = true; + + public CollectorSinkTask( + final String taskId, + final Map sinkAttribute, + final PipeSink pipeSink, + final BlockingQueue pendingQueue) { + super(taskId); + this.sinkAttribute = sinkAttribute; + this.pipeSink = pipeSink; + this.pendingQueue = pendingQueue; + } + + @Override + public void runMayThrow() { + try { + pipeSink.handshake(); + } catch (final Exception e) { + LOGGER.warn("handshake fail because {}", e.getMessage()); + } + isStarted = true; + while (isStarted) { + try { + final Event event = pendingQueue.take(); + pipeSink.transfer(event); + LOGGER.info("transfer event {} success, remain number is {}", event, pendingQueue.size()); + } catch (final InterruptedException e) { + LOGGER.warn("interrupted while waiting for take a event"); + } catch (final Exception e) { + LOGGER.warn("error occur while transfer event to endpoint"); + } + } + } + + public Map getSinkAttribute() { + return sinkAttribute; + } + + public PipeSink getPipeSink() { + return pipeSink; + } + + public void stop() { + isStarted = false; + } + + public BlockingQueue getPendingQueue() { + return pendingQueue; + } +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/agent/task/CollectorSourceTask.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/agent/task/CollectorSourceTask.java new file mode 100644 index 000000000000..26230984713c --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/agent/task/CollectorSourceTask.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.agent.task; + +import org.apache.iotdb.commons.pipe.agent.task.connection.EventSupplier; +import org.apache.iotdb.pipe.api.PipeSource; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Map; + +public class CollectorSourceTask extends CollectorTask { + + private static final Logger LOGGER = LoggerFactory.getLogger(CollectorSourceTask.class); + + private final Map sourceAttribute; + private final PipeSource pipeSource; + + public CollectorSourceTask( + final String taskId, final Map sourceAttribute, final PipeSource pipeSource) { + super(taskId); + this.sourceAttribute = sourceAttribute; + this.pipeSource = pipeSource; + } + + @Override + public void runMayThrow() throws Throwable { + pipeSource.start(); + } + + public Map getSourceAttribute() { + return sourceAttribute; + } + + public PipeSource getPipeSource() { + return pipeSource; + } + + public EventSupplier getEventSupplier() { + return pipeSource::supply; + } + + @Override + public void stop() { + try { + pipeSource.close(); + } catch (final Exception e) { + LOGGER.warn("failed to close pipe source {}", pipeSource, e); + } + } +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/CollectorConstant.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/agent/task/CollectorTask.java similarity index 68% rename from iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/CollectorConstant.java rename to iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/agent/task/CollectorTask.java index ed9bf141a2b1..4f13ff9c4d98 100644 --- a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/CollectorConstant.java +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/agent/task/CollectorTask.java @@ -17,8 +17,21 @@ * under the License. */ -package org.apache.iotdb.collector.config; +package org.apache.iotdb.collector.agent.task; -public class CollectorConstant { - public static final String PROPERTIES_FILE_NAME = "system.properties"; +import org.apache.iotdb.commons.concurrent.WrappedRunnable; + +public abstract class CollectorTask extends WrappedRunnable { + + protected final String taskId; + + protected CollectorTask(final String taskId) { + this.taskId = taskId; + } + + public String getTaskId() { + return taskId; + } + + public abstract void stop(); } diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/agent/task/CollectorTaskAgent.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/agent/task/CollectorTaskAgent.java new file mode 100644 index 000000000000..9cd787df0487 --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/agent/task/CollectorTaskAgent.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.agent.task; + +import org.apache.iotdb.collector.agent.executor.CollectorProcessorTaskExecutor; +import org.apache.iotdb.collector.agent.executor.CollectorSinkTaskExecutor; +import org.apache.iotdb.collector.agent.executor.CollectorSourceTaskExecutor; +import org.apache.iotdb.collector.agent.executor.CollectorTaskExecutorAgent; +import org.apache.iotdb.collector.agent.plugin.CollectorPluginAgent; +import org.apache.iotdb.collector.agent.plugin.CollectorPluginConstructor; +import org.apache.iotdb.collector.plugin.BuiltinCollectorPlugin; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Map; +import java.util.concurrent.LinkedBlockingQueue; + +public class CollectorTaskAgent { + + private static final Logger LOGGER = LoggerFactory.getLogger(CollectorTaskAgent.class); + + private static final CollectorPluginConstructor CONSTRUCTOR = + CollectorPluginAgent.instance().constructor(); + private static final CollectorSourceTaskExecutor SOURCE_TASK_EXECUTOR = + CollectorTaskExecutorAgent.instance().getSourceTaskExecutor(); + private static final CollectorProcessorTaskExecutor PROCESSOR_TASK_EXECUTOR = + CollectorTaskExecutorAgent.instance().getProcessorTaskExecutor(); + private static final CollectorSinkTaskExecutor SINK_TASK_EXECUTOR = + CollectorTaskExecutorAgent.instance().getSinkTaskExecutor(); + + private CollectorTaskAgent() {} + + public boolean createCollectorTask( + final Map sourceAttribute, + final Map processorAttribute, + final Map sinkAttribute, + final String taskId) { + try { + final CollectorSourceTask collectorSourceTask = + new CollectorSourceTask( + taskId, + sourceAttribute, + CONSTRUCTOR.getSource( + sourceAttribute.getOrDefault( + "source-plugin", + BuiltinCollectorPlugin.HTTP_SOURCE.getCollectorPluginName()))); + SOURCE_TASK_EXECUTOR.register(collectorSourceTask); + + final CollectorProcessorTask collectorProcessorTask = + new CollectorProcessorTask( + taskId, + processorAttribute, + CONSTRUCTOR.getProcessor( + processorAttribute.getOrDefault( + "processor-plugin", + BuiltinCollectorPlugin.DO_NOTHING_PROCESSOR.getCollectorPluginName())), + collectorSourceTask.getEventSupplier(), + new LinkedBlockingQueue<>()); + PROCESSOR_TASK_EXECUTOR.register(collectorProcessorTask); + + final CollectorSinkTask collectorSinkTask = + new CollectorSinkTask( + taskId, + sinkAttribute, + CONSTRUCTOR.getSink( + sinkAttribute.getOrDefault( + "sink-plugin", + BuiltinCollectorPlugin.IOTDB_SESSION_SINK.getCollectorPluginName())), + collectorProcessorTask.getPendingQueue()); + SINK_TASK_EXECUTOR.register(collectorSinkTask); + } catch (final Exception e) { + LOGGER.warn("create collector task error", e); + return false; + } + + return true; + } + + public boolean stopCollectorTask(final String taskId) { + try { + SOURCE_TASK_EXECUTOR.deregister(taskId); + PROCESSOR_TASK_EXECUTOR.deregister(taskId); + SINK_TASK_EXECUTOR.deregister(taskId); + } catch (final Exception e) { + LOGGER.warn("stop collector task error", e); + return false; + } + + return true; + } + + public static CollectorTaskAgent instance() { + return CollectorTaskAgentHolder.INSTANCE; + } + + private static class CollectorTaskAgentHolder { + private static final CollectorTaskAgent INSTANCE = new CollectorTaskAgent(); + } +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/CollectorConfig.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/CollectorConfig.java index 99a40488e552..6f0cb1268650 100644 --- a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/CollectorConfig.java +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/CollectorConfig.java @@ -19,9 +19,15 @@ package org.apache.iotdb.collector.config; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.lang.reflect.Field; +import java.util.Arrays; + public class CollectorConfig { - public static final String CONFIG_NAME = "iotdb-collector-system.properties"; + private static final Logger LOGGER = LoggerFactory.getLogger(CollectorConfig.class); private int restServicePort = 17070; @@ -36,4 +42,38 @@ public int getRestServicePort() { public void setRestServicePort(int restServicePort) { this.restServicePort = restServicePort; } + + public String getConfigMessage() { + final StringBuilder configMessage = new StringBuilder(); + String configContent; + + for (final Field configField : CollectorConfig.class.getDeclaredFields()) { + try { + final String configType = configField.getGenericType().getTypeName(); + if (configType.contains("java.lang.String[][]")) { + final String[][] configList = (String[][]) configField.get(this); + final StringBuilder builder = new StringBuilder(); + for (final String[] strings : configList) { + builder.append(Arrays.asList(strings)).append(";"); + } + configContent = builder.toString(); + } else if (configType.contains("java.lang.String[]")) { + final String[] configList = (String[]) configField.get(this); + configContent = Arrays.asList(configList).toString(); + } else { + configContent = configField.get(this).toString(); + } + configMessage + .append("\n\t") + .append(configField.getName()) + .append("=") + .append(configContent) + .append(";"); + } catch (final IllegalAccessException e) { + LOGGER.warn("failed to show config message", e); + } + } + + return configMessage.toString(); + } } diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/CollectorDescriptor.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/CollectorDescriptor.java index 6046254e64dc..ae8bf553188b 100644 --- a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/CollectorDescriptor.java +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/CollectorDescriptor.java @@ -37,41 +37,10 @@ public class CollectorDescriptor { private static final Logger LOGGER = LoggerFactory.getLogger(CollectorDescriptor.class); + private static final String CONFIG_FILE_NAME = "iotdb-collector-system.properties"; private static final CollectorConfig CONFIG = new CollectorConfig(); - private static final String CONFIG_NAME = CollectorConfig.CONFIG_NAME; - - static { - final Optional systemConfigUrl = getPropsUrl(); - systemConfigUrl.ifPresent( - url -> { - try { - ConfigFileUtils.checkAndMayUpdate(url); - } catch (final Exception e) { - if (e instanceof InterruptedException) { - Thread.currentThread().interrupt(); - } - LOGGER.error("Failed to update config file", e); - } - }); - } - - private static Optional getPropsUrl() { - final URL url = CollectorConfig.class.getResource("/" + CONFIG_NAME); - - if (url != null) { - return Optional.of(url); - } else { - LOGGER.warn( - "Cannot find IOTDB_COLLECTOR_HOME or IOTDB_COLLECTOR_CONF environment variable when loading " - + "config file {}, use default configuration", - CONFIG_NAME); - // TODO update path - // IoTDBConfig: updatePath() - return Optional.empty(); - } - } - protected CollectorDescriptor() { + private CollectorDescriptor() { loadProps(); } @@ -87,17 +56,34 @@ private void loadProps() { collectorProperties.putAll(properties); loadProperties(collectorProperties); } catch (final FileNotFoundException e) { - LOGGER.error("Fail to find config file {}, reject DataNode startup.", url.get(), e); + LOGGER.error("Fail to find config file {}, reject CollectorNode startup.", url.get(), e); System.exit(-1); } catch (final IOException e) { - LOGGER.error("Cannot load config file, reject DataNode startup.", e); + LOGGER.error("Cannot load config file, reject CollectorNode startup.", e); System.exit(-1); } catch (final Exception e) { - LOGGER.error("Incorrect format in config file, reject DataNode startup.", e); + LOGGER.error("Incorrect format in config file, reject CollectorNode startup.", e); System.exit(-1); } } else { - LOGGER.warn("Couldn't load the configuration {} from any of the known sources.", CONFIG_NAME); + LOGGER.warn( + "Couldn't load the configuration {} from any of the known sources.", CONFIG_FILE_NAME); + System.exit(-1); + } + } + + private static Optional getPropsUrl() { + final URL url = CollectorConfig.class.getResource("/" + CONFIG_FILE_NAME); + + if (url != null) { + return Optional.of(url); + } else { + LOGGER.warn( + "Cannot find IOTDB_COLLECTOR_HOME or IOTDB_COLLECTOR_CONF environment variable when loading " + + "config file {}, use default configuration", + CONFIG_FILE_NAME); + + return Optional.empty(); } } @@ -105,7 +91,7 @@ private void loadProps() { private void loadProperties(final TrimProperties properties) { CONFIG.setRestServicePort( Integer.parseInt( - Optional.ofNullable(properties.getProperty("collector_rest_port")) + Optional.ofNullable(properties.getProperty("collector_rest_service_port")) .orElse(String.valueOf(CONFIG.getRestServicePort())))); } diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/CollectorSystemPropertiesHandler.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/CollectorSystemPropertiesHandler.java deleted file mode 100644 index ff4db04b9951..000000000000 --- a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/CollectorSystemPropertiesHandler.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.collector.config; - -import org.apache.iotdb.commons.file.SystemPropertiesHandler; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import static org.apache.iotdb.collector.config.CollectorConstant.PROPERTIES_FILE_NAME; - -public class CollectorSystemPropertiesHandler extends SystemPropertiesHandler { - - private static final Logger LOGGER = - LoggerFactory.getLogger(CollectorSystemPropertiesHandler.class); - - private static CollectorSystemPropertiesHandler INSTANCE; - - public CollectorSystemPropertiesHandler(final String filePath) { - super(filePath); - } - - public static SystemPropertiesHandler getInstance() { - if (INSTANCE == null) { - synchronized (CollectorSystemPropertiesHandler.class) { - INSTANCE = - new CollectorSystemPropertiesHandler( - // TODO System File Folder - PROPERTIES_FILE_NAME); - INSTANCE.init(); - } - } - return INSTANCE; - } -} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/ConfigFileUtils.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/ConfigFileUtils.java deleted file mode 100644 index 54c687981dc6..000000000000 --- a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/ConfigFileUtils.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.collector.config; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.IOException; -import java.io.RandomAccessFile; -import java.net.URL; -import java.nio.file.Files; -import java.util.StringJoiner; -import java.util.concurrent.TimeUnit; - -public class ConfigFileUtils { - - private static final String lockFileSuffix = ".lock"; - private static final long maxTimeMillsToAcquireLock = TimeUnit.SECONDS.toMillis(20); - private static final long waitTimeMillsPerCheck = TimeUnit.MILLISECONDS.toMillis(100); - private static final Logger LOGGER = LoggerFactory.getLogger(ConfigFileUtils.class); - private static final String lineSeparator = "\n"; - private static final String license = - new StringJoiner(lineSeparator) - .add("# Licensed to the Apache Software Foundation (ASF) under one") - .add("# or more contributor license agreements. See the NOTICE file") - .add("# distributed with this work for additional information") - .add("# regarding copyright ownership. The ASF licenses this file") - .add("# to you under the Apache License, Version 2.0 (the") - .add("# \"License\"); you may not use this file except in compliance") - .add("# with the License. You may obtain a copy of the License at") - .add("#") - .add("# http://www.apache.org/licenses/LICENSE-2.0") - .add("#") - .add("# Unless required by applicable law or agreed to in writing,") - .add("# software distributed under the License is distributed on an") - .add("# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY") - .add("# KIND, either express or implied. See the License for the") - .add("# specific language governing permissions and limitations") - .add("# under the License.") - .toString(); - - public static void checkAndMayUpdate(final URL url) throws IOException, InterruptedException { - final File systemFile = new File(url.getFile()); - if (systemFile.exists()) { - return; - } - final File lockFile = new File(systemFile.getAbsolutePath() + lockFileSuffix); - acquireTargetFileLock(lockFile); - try { - if (systemFile.exists()) { - return; - } - try (final RandomAccessFile raf = new RandomAccessFile(lockFile, "rw")) { - raf.write(license.getBytes()); - } - Files.move(lockFile.toPath(), systemFile.toPath()); - } finally { - releaseFileLock(lockFile); - } - } - - private static void acquireTargetFileLock(final File file) - throws IOException, InterruptedException { - long totalWaitTime = 0; - while (totalWaitTime < maxTimeMillsToAcquireLock) { - if (file.createNewFile()) { - return; - } - totalWaitTime += waitTimeMillsPerCheck; - Thread.sleep(waitTimeMillsPerCheck); - } - LOGGER.warn( - "Waiting for {} seconds to acquire configuration file update lock." - + " There may have been an unexpected interruption in the last" - + " configuration file update. Ignore temporary file {}", - totalWaitTime / 1000, - file.getName()); - } - - private static void releaseFileLock(final File file) throws IOException { - Files.deleteIfExists(file.toPath()); - } -} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/BuiltinCollectorPlugin.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/BuiltinCollectorPlugin.java new file mode 100644 index 000000000000..2d676e5137e3 --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/BuiltinCollectorPlugin.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.plugin; + +import org.apache.iotdb.collector.plugin.builtin.processor.DoNothingProcessor; +import org.apache.iotdb.collector.plugin.builtin.sink.SessionSink; +import org.apache.iotdb.collector.plugin.builtin.source.HttpSource; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; + +public enum BuiltinCollectorPlugin { + + // Sources + HTTP_SOURCE("http-source", HttpSource.class), + + // Processors + DO_NOTHING_PROCESSOR("do-nothing-processor", DoNothingProcessor.class), + + // Sinks + IOTDB_SESSION_SINK("iotdb-session-sink", SessionSink.class); + + private final String collectorPluginName; + private final Class collectorPluginClass; + private final String className; + + BuiltinCollectorPlugin(String collectorPluginName, Class collectorPluginClass) { + this.collectorPluginName = collectorPluginName; + this.collectorPluginClass = collectorPluginClass; + this.className = collectorPluginClass.getName(); + } + + public String getCollectorPluginName() { + return collectorPluginName; + } + + public Class getCollectorPluginClass() { + return collectorPluginClass; + } + + public String getClassName() { + return className; + } + + public static final Set SHOW_COLLECTOR_PLUGINS_BLACKLIST = + Collections.unmodifiableSet( + new HashSet<>( + Arrays.asList( + // Sources + HTTP_SOURCE.getCollectorPluginName().toUpperCase(), + // Processors + DO_NOTHING_PROCESSOR.getCollectorPluginName().toUpperCase(), + // Sinks + IOTDB_SESSION_SINK.getCollectorPluginName().toUpperCase()))); +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/processor/DoNothingProcessor.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/processor/DoNothingProcessor.java new file mode 100644 index 000000000000..436fc2086477 --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/processor/DoNothingProcessor.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.plugin.builtin.processor; + +import org.apache.iotdb.pipe.api.PipeProcessor; +import org.apache.iotdb.pipe.api.collector.EventCollector; +import org.apache.iotdb.pipe.api.customizer.configuration.PipeProcessorRuntimeConfiguration; +import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameterValidator; +import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters; +import org.apache.iotdb.pipe.api.event.Event; +import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class DoNothingProcessor implements PipeProcessor { + + private static final Logger LOGGER = LoggerFactory.getLogger(DoNothingProcessor.class); + + @Override + public void validate(PipeParameterValidator validator) throws Exception { + // Do Nothing + } + + @Override + public void customize(PipeParameters parameters, PipeProcessorRuntimeConfiguration configuration) + throws Exception { + // Do Nothing + } + + @Override + public void process(TabletInsertionEvent tabletInsertionEvent, EventCollector eventCollector) + throws Exception { + // Do Nothing + } + + @Override + public void process(Event event, EventCollector eventCollector) throws Exception { + LOGGER.info("DoNothingProcessor process event: {}", event); + eventCollector.collect(event); + } + + @Override + public void close() throws Exception { + // Do Nothing + } +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/SessionSink.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/SessionSink.java new file mode 100644 index 000000000000..e2a5260520ff --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/sink/SessionSink.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.plugin.builtin.sink; + +import org.apache.iotdb.collector.plugin.builtin.source.event.SourceEvent; +import org.apache.iotdb.pipe.api.PipeSink; +import org.apache.iotdb.pipe.api.customizer.configuration.PipeConnectorRuntimeConfiguration; +import org.apache.iotdb.pipe.api.customizer.configuration.PipeSinkRuntimeConfiguration; +import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameterValidator; +import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters; +import org.apache.iotdb.pipe.api.event.Event; +import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class SessionSink implements PipeSink { + + private static final Logger LOGGER = LoggerFactory.getLogger(SessionSink.class); + + @Override + public void validate(PipeParameterValidator validator) throws Exception {} + + @Override + public void customize(PipeParameters parameters, PipeConnectorRuntimeConfiguration configuration) + throws Exception {} + + @Override + public void customize(PipeParameters parameters, PipeSinkRuntimeConfiguration configuration) + throws Exception {} + + @Override + public void handshake() throws Exception { + LOGGER.info("SessionSink handshake successfully"); + } + + @Override + public void heartbeat() throws Exception {} + + @Override + public void transfer(TabletInsertionEvent tabletInsertionEvent) throws Exception {} + + @Override + public void transfer(Event event) throws Exception { + final SourceEvent sourceEvent = (SourceEvent) event; + LOGGER.info("SessionSink transfer successfully {}", sourceEvent); + } + + @Override + public void close() throws Exception {} +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/source/HttpSource.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/source/HttpSource.java new file mode 100644 index 000000000000..65ba79b288aa --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/source/HttpSource.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.plugin.builtin.source; + +import org.apache.iotdb.collector.plugin.builtin.source.event.SourceEvent; +import org.apache.iotdb.pipe.api.PipeSource; +import org.apache.iotdb.pipe.api.customizer.configuration.PipeExtractorRuntimeConfiguration; +import org.apache.iotdb.pipe.api.customizer.configuration.PipeSourceRuntimeConfiguration; +import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameterValidator; +import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters; +import org.apache.iotdb.pipe.api.event.Event; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Random; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; + +public class HttpSource implements PipeSource { + + private static final Logger LOGGER = LoggerFactory.getLogger(HttpSource.class); + + private static final BlockingQueue queue = new LinkedBlockingQueue<>(); + private boolean isStarted = true; + + @Override + public void validate(PipeParameterValidator validator) throws Exception {} + + @Override + public void customize(PipeParameters parameters, PipeExtractorRuntimeConfiguration configuration) + throws Exception {} + + @Override + public void customize(PipeParameters parameters, PipeSourceRuntimeConfiguration configuration) + throws Exception {} + + @Override + public void start() { + isStarted = true; + while (isStarted) { + Event event = new SourceEvent(String.valueOf(new Random().nextInt(1000))); + try { + queue.put(event); + Thread.sleep(1000); + LOGGER.info("event: {} created success", event); + } catch (final InterruptedException e) { + LOGGER.warn("failed to create event because {}", e.getMessage()); + } + } + } + + @Override + public Event supply() throws Exception { + return queue.take(); + } + + @Override + public void close() throws Exception { + isStarted = false; + } +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/source/event/SourceEvent.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/source/event/SourceEvent.java new file mode 100644 index 000000000000..813e25ee16cf --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/plugin/builtin/source/event/SourceEvent.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.plugin.builtin.source.event; + +import org.apache.iotdb.pipe.api.event.Event; + +public class SourceEvent implements Event { + private String name; + + public SourceEvent() {} + + public SourceEvent(final String name) { + this.name = name; + } + + public String getName() { + return name; + } + + public void setName(final String name) { + this.name = name; + } + + @Override + public String toString() { + return "SourceEvent [name=" + name + "]"; + } +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/protocol/rest/v1/handler/RequestValidationHandler.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/protocol/rest/v1/handler/RequestValidationHandler.java new file mode 100644 index 000000000000..e072feff8851 --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/protocol/rest/v1/handler/RequestValidationHandler.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.protocol.rest.v1.handler; + +import org.apache.iotdb.collector.protocol.rest.v1.model.CreatePipeRequest; +import org.apache.iotdb.collector.protocol.rest.v1.model.StopPipeRequest; + +import java.util.Objects; + +public class RequestValidationHandler { + private RequestValidationHandler() {} + + public static void validateCreateRequest(final CreatePipeRequest createPipeRequest) { + Objects.requireNonNull(createPipeRequest.getTaskId(), "taskId cannot be null"); + } + + public static void validateStopRequest(final StopPipeRequest stopPipeRequest) { + Objects.requireNonNull(stopPipeRequest.getTaskId(), "taskId cannot be null"); + } +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/protocol/rest/v1/impl/AdminApiServiceImpl.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/protocol/rest/v1/impl/AdminApiServiceImpl.java index 8b9c5ced25b2..6f74f75e2bb5 100644 --- a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/protocol/rest/v1/impl/AdminApiServiceImpl.java +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/protocol/rest/v1/impl/AdminApiServiceImpl.java @@ -19,18 +19,26 @@ package org.apache.iotdb.collector.protocol.rest.v1.impl; +import org.apache.iotdb.collector.agent.CollectorAgent; import org.apache.iotdb.collector.protocol.rest.v1.AdminApiService; import org.apache.iotdb.collector.protocol.rest.v1.NotFoundException; +import org.apache.iotdb.collector.protocol.rest.v1.handler.RequestValidationHandler; import org.apache.iotdb.collector.protocol.rest.v1.model.AlterPipeRequest; import org.apache.iotdb.collector.protocol.rest.v1.model.CreatePipeRequest; import org.apache.iotdb.collector.protocol.rest.v1.model.DropPipeRequest; import org.apache.iotdb.collector.protocol.rest.v1.model.StartPipeRequest; import org.apache.iotdb.collector.protocol.rest.v1.model.StopPipeRequest; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import javax.ws.rs.core.Response; import javax.ws.rs.core.SecurityContext; public class AdminApiServiceImpl extends AdminApiService { + + private static final Logger LOGGER = LoggerFactory.getLogger(AdminApiServiceImpl.class); + @Override public Response alterPipe( final AlterPipeRequest alterPipeRequest, final SecurityContext securityContext) @@ -42,7 +50,21 @@ public Response alterPipe( public Response createPipe( final CreatePipeRequest createPipeRequest, final SecurityContext securityContext) throws NotFoundException { - return Response.ok("createPipe").build(); + RequestValidationHandler.validateCreateRequest(createPipeRequest); + + final boolean createdResult = + CollectorAgent.task() + .createCollectorTask( + createPipeRequest.getSourceAttribute(), + createPipeRequest.getProcessorAttribute(), + createPipeRequest.getSinkAttribute(), + createPipeRequest.getTaskId()); + if (createdResult) { + LOGGER.info("Create task successful"); + return Response.status(Response.Status.OK).entity("create task success").build(); + } + LOGGER.warn("Create task failed"); + return Response.status(Response.Status.BAD_REQUEST).entity("create task fail").build(); } @Override @@ -63,6 +85,14 @@ public Response startPipe( public Response stopPipe( final StopPipeRequest stopPipeRequest, final SecurityContext securityContext) throws NotFoundException { - return Response.ok("stopPipe").build(); + RequestValidationHandler.validateStopRequest(stopPipeRequest); + + final boolean stopResult = CollectorAgent.task().stopCollectorTask(stopPipeRequest.getTaskId()); + if (stopResult) { + LOGGER.info("Stop task: {} successful", stopPipeRequest.getTaskId()); + return Response.ok().entity("stop task: " + stopPipeRequest.getTaskId() + " success").build(); + } + LOGGER.warn("Stop task: {} failed", stopPipeRequest.getTaskId()); + return Response.status(Response.Status.BAD_REQUEST).entity("stop task fail").build(); } } diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/service/RegisterManager.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/service/RegisterManager.java index 5498ff403ecb..a91a8e31c15d 100644 --- a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/service/RegisterManager.java +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/service/RegisterManager.java @@ -25,7 +25,6 @@ import org.slf4j.LoggerFactory; import java.util.ArrayList; -import java.util.Collections; import java.util.List; public class RegisterManager { @@ -57,8 +56,7 @@ public void register(final IService service) throws StartupException { /** stop all service and clear iService list. */ public void deregisterAll() { - Collections.reverse(iServices); - for (IService service : iServices) { + for (final IService service : iServices) { try { service.waitAndStop(DEREGISTER_TIME_OUT); LOGGER.debug("{} deregistered", service.getID()); diff --git a/iotdb-collector/collector-core/src/main/resources/iotdb-collector-system.properties b/iotdb-collector/collector-core/src/main/resources/iotdb-collector-system.properties index bb4189f4f4c1..4e5db9edc942 100644 --- a/iotdb-collector/collector-core/src/main/resources/iotdb-collector-system.properties +++ b/iotdb-collector/collector-core/src/main/resources/iotdb-collector-system.properties @@ -17,4 +17,4 @@ # under the License. # -collector_rest_port=17070 \ No newline at end of file +collector_rest_service_port=17070 \ No newline at end of file diff --git a/iotdb-collector/collector-openapi/src/main/openapi3/iotdb_collector_rest_v1.yaml b/iotdb-collector/collector-openapi/src/main/openapi3/iotdb_collector_rest_v1.yaml index 85802287ffa0..acb16b2c5d44 100644 --- a/iotdb-collector/collector-openapi/src/main/openapi3/iotdb_collector_rest_v1.yaml +++ b/iotdb-collector/collector-openapi/src/main/openapi3/iotdb_collector_rest_v1.yaml @@ -127,30 +127,22 @@ components: title: CreatePipeRequest type: object properties: - timestamps: - type: array - items: - type: integer - format: int64 - measurements: - type: array - items: + sourceAttribute: + type: object + additionalProperties: type: string - dataTypes: - type: array - items: + processorAttribute: + type: object + additionalProperties: type: string - values: - type: array - items: - type: array - items: - type: object - isAligned: - type: boolean - deviceId: + sinkAttribute: + type: object + additionalProperties: + type: string + taskId: type: string + AlterPipeRequest: title: AlterPipeRequest type: object @@ -211,28 +203,7 @@ components: title: StopPipeRequest type: object properties: - timestamps: - type: array - items: - type: integer - format: int64 - measurements: - type: array - items: - type: string - dataTypes: - type: array - items: - type: string - values: - type: array - items: - type: array - items: - type: object - isAligned: - type: boolean - deviceId: + taskId: type: string DropPipeRequest: From 73d5f82d939ce862193fa10ce2c0327f3e394bc5 Mon Sep 17 00:00:00 2001 From: Steve Yurong Su Date: Tue, 21 Jan 2025 20:03:41 +0800 Subject: [PATCH 03/10] refactor pom.xml --- iotdb-collector/collector-core/pom.xml | 1 - iotdb-collector/pom.xml | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/iotdb-collector/collector-core/pom.xml b/iotdb-collector/collector-core/pom.xml index 820c73a54526..e317f3b35c47 100644 --- a/iotdb-collector/collector-core/pom.xml +++ b/iotdb-collector/collector-core/pom.xml @@ -57,7 +57,6 @@ org.glassfish.jersey.inject jersey-hk2 - runtime diff --git a/iotdb-collector/pom.xml b/iotdb-collector/pom.xml index 261314f254e0..d23d331dc4d4 100644 --- a/iotdb-collector/pom.xml +++ b/iotdb-collector/pom.xml @@ -30,6 +30,7 @@ pom IoTDB: Collector + collector-core collector-openapi From 015007d4058237b7d5701633534e4abcf16b2756 Mon Sep 17 00:00:00 2001 From: Steve Yurong Su Date: Thu, 23 Jan 2025 17:58:31 +0800 Subject: [PATCH 04/10] refactor --- iotdb-collector/collector-core/pom.xml | 2 +- .../apache/iotdb/collector/Application.java | 47 +++++++++++ .../org/apache/iotdb/collector/Collector.java | 83 ------------------- .../rest => api}/filter/ApiOriginFilter.java | 2 +- .../rest => api}/impl/PingApiServiceImpl.java | 8 +- .../v1/handler/RequestValidationHandler.java | 6 +- .../v1/impl/AdminApiServiceImpl.java | 18 ++-- .../collector/config/CollectorDescriptor.java | 2 +- ...lectorRestService.java => ApiService.java} | 35 ++------ .../collector/service/CollectorMBean.java | 21 ----- .../iotdb/collector/service/IService.java | 39 --------- .../collector/service/RegisterManager.java | 80 ------------------ ...stem.properties => application.properties} | 0 iotdb-collector/collector-openapi/pom.xml | 24 +++--- ...b_collector_rest_common.yaml => ping.yaml} | 2 +- iotdb-collector/pom.xml | 2 +- 16 files changed, 88 insertions(+), 283 deletions(-) create mode 100644 iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/Application.java delete mode 100644 iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/Collector.java rename iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/{protocol/rest => api}/filter/ApiOriginFilter.java (96%) rename iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/{protocol/rest => api}/impl/PingApiServiceImpl.java (83%) rename iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/{protocol/rest => api}/v1/handler/RequestValidationHandler.java (85%) rename iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/{protocol/rest => api}/v1/impl/AdminApiServiceImpl.java (83%) rename iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/service/{CollectorRestService.java => ApiService.java} (74%) delete mode 100644 iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/service/CollectorMBean.java delete mode 100644 iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/service/IService.java delete mode 100644 iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/service/RegisterManager.java rename iotdb-collector/collector-core/src/main/resources/{iotdb-collector-system.properties => application.properties} (100%) rename iotdb-collector/collector-openapi/src/main/openapi3/{iotdb_collector_rest_common.yaml => ping.yaml} (97%) diff --git a/iotdb-collector/collector-core/pom.xml b/iotdb-collector/collector-core/pom.xml index e317f3b35c47..1f7cf610d290 100644 --- a/iotdb-collector/collector-core/pom.xml +++ b/iotdb-collector/collector-core/pom.xml @@ -23,7 +23,7 @@ 4.0.0 org.apache.iotdb - iotdb-collector + iotdb-application 2.0.0-SNAPSHOT collector-core diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/Application.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/Application.java new file mode 100644 index 000000000000..959472b1cb7f --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/Application.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector; + +import org.apache.iotdb.collector.config.CollectorConfig; +import org.apache.iotdb.collector.config.CollectorDescriptor; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.charset.Charset; + +public class Application { + + private static final Logger LOGGER = LoggerFactory.getLogger(Application.class); + + private static final CollectorConfig COLLECTOR_CONFIG = + CollectorDescriptor.getInstance().getConfig(); + + public static void main(String[] args) { + LOGGER.info("Environment variables: {}", CollectorConfig.getEnvironmentVariables()); + LOGGER.info("Default charset is: {}", Charset.defaultCharset().displayName()); + + final Application application = new Application(); + + LOGGER.info("IoTDB-CollectorNode configuration: {}", COLLECTOR_CONFIG.getConfigMessage()); + LOGGER.info( + "Congratulations, IoTDB CollectorNode is set up successfully. Now, enjoy yourself!"); + } +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/Collector.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/Collector.java deleted file mode 100644 index 4aaa60f3689c..000000000000 --- a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/Collector.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.collector; - -import org.apache.iotdb.collector.config.CollectorConfig; -import org.apache.iotdb.collector.config.CollectorDescriptor; -import org.apache.iotdb.collector.service.CollectorMBean; -import org.apache.iotdb.collector.service.CollectorRestService; -import org.apache.iotdb.collector.service.RegisterManager; -import org.apache.iotdb.commons.ServerCommandLine; -import org.apache.iotdb.commons.exception.StartupException; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.nio.charset.Charset; -import java.util.Set; - -public class Collector extends ServerCommandLine implements CollectorMBean { - - private static final Logger LOGGER = LoggerFactory.getLogger(Collector.class); - - private static final RegisterManager REGISTER_MANAGER = RegisterManager.getInstance(); - private static final CollectorConfig COLLECTOR_CONFIG = - CollectorDescriptor.getInstance().getConfig(); - - private Collector() { - super("Collector"); - } - - public static void main(String[] args) { - LOGGER.info( - "IoTDB-CollectorNode environment variables: {}", CollectorConfig.getEnvironmentVariables()); - LOGGER.info( - "IoTDB-CollectorNode default charset is: {}", Charset.defaultCharset().displayName()); - - final Collector collector = new Collector(); - final int returnCode = collector.run(args); - if (returnCode != 0) { - System.exit(returnCode); - } - } - - @Override - protected void start() { - try { - initProtocols(); - - LOGGER.info("IoTDB-CollectorNode configuration: {}", COLLECTOR_CONFIG.getConfigMessage()); - LOGGER.info( - "Congratulations, IoTDB CollectorNode is set up successfully. Now, enjoy yourself!"); - } catch (final StartupException e) { - LOGGER.error("CollectorNode start failed", e); - System.exit(-1); - } - } - - private void initProtocols() throws StartupException { - REGISTER_MANAGER.register(CollectorRestService.getInstance()); - } - - @Override - protected void remove(final Set nodeIds) { - // empty method - } -} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/protocol/rest/filter/ApiOriginFilter.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/api/filter/ApiOriginFilter.java similarity index 96% rename from iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/protocol/rest/filter/ApiOriginFilter.java rename to iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/api/filter/ApiOriginFilter.java index 179836fec6eb..008e30904194 100644 --- a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/protocol/rest/filter/ApiOriginFilter.java +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/api/filter/ApiOriginFilter.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.collector.protocol.rest.filter; +package org.apache.iotdb.collector.api.filter; import javax.servlet.FilterChain; import javax.servlet.FilterConfig; diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/protocol/rest/impl/PingApiServiceImpl.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/api/impl/PingApiServiceImpl.java similarity index 83% rename from iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/protocol/rest/impl/PingApiServiceImpl.java rename to iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/api/impl/PingApiServiceImpl.java index 8d34d94852f6..0fc582467df8 100644 --- a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/protocol/rest/impl/PingApiServiceImpl.java +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/api/impl/PingApiServiceImpl.java @@ -15,11 +15,11 @@ * limitations under the License. */ -package org.apache.iotdb.collector.protocol.rest.impl; +package org.apache.iotdb.collector.api.impl; -import org.apache.iotdb.collector.protocol.rest.NotFoundException; -import org.apache.iotdb.collector.protocol.rest.PingApiService; -import org.apache.iotdb.collector.protocol.rest.v1.model.ExecutionStatus; +import org.apache.iotdb.collector.api.NotFoundException; +import org.apache.iotdb.collector.api.PingApiService; +import org.apache.iotdb.collector.api.v1.model.ExecutionStatus; import org.apache.iotdb.rpc.TSStatusCode; import javax.ws.rs.core.Response; diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/protocol/rest/v1/handler/RequestValidationHandler.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/api/v1/handler/RequestValidationHandler.java similarity index 85% rename from iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/protocol/rest/v1/handler/RequestValidationHandler.java rename to iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/api/v1/handler/RequestValidationHandler.java index e072feff8851..9456c2f33954 100644 --- a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/protocol/rest/v1/handler/RequestValidationHandler.java +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/api/v1/handler/RequestValidationHandler.java @@ -17,10 +17,10 @@ * under the License. */ -package org.apache.iotdb.collector.protocol.rest.v1.handler; +package org.apache.iotdb.collector.api.v1.handler; -import org.apache.iotdb.collector.protocol.rest.v1.model.CreatePipeRequest; -import org.apache.iotdb.collector.protocol.rest.v1.model.StopPipeRequest; +import org.apache.iotdb.collector.api.v1.model.CreatePipeRequest; +import org.apache.iotdb.collector.api.v1.model.StopPipeRequest; import java.util.Objects; diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/protocol/rest/v1/impl/AdminApiServiceImpl.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/api/v1/impl/AdminApiServiceImpl.java similarity index 83% rename from iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/protocol/rest/v1/impl/AdminApiServiceImpl.java rename to iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/api/v1/impl/AdminApiServiceImpl.java index 6f74f75e2bb5..07d1f3e7878b 100644 --- a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/protocol/rest/v1/impl/AdminApiServiceImpl.java +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/api/v1/impl/AdminApiServiceImpl.java @@ -17,17 +17,17 @@ * under the License. */ -package org.apache.iotdb.collector.protocol.rest.v1.impl; +package org.apache.iotdb.collector.api.v1.impl; import org.apache.iotdb.collector.agent.CollectorAgent; -import org.apache.iotdb.collector.protocol.rest.v1.AdminApiService; -import org.apache.iotdb.collector.protocol.rest.v1.NotFoundException; -import org.apache.iotdb.collector.protocol.rest.v1.handler.RequestValidationHandler; -import org.apache.iotdb.collector.protocol.rest.v1.model.AlterPipeRequest; -import org.apache.iotdb.collector.protocol.rest.v1.model.CreatePipeRequest; -import org.apache.iotdb.collector.protocol.rest.v1.model.DropPipeRequest; -import org.apache.iotdb.collector.protocol.rest.v1.model.StartPipeRequest; -import org.apache.iotdb.collector.protocol.rest.v1.model.StopPipeRequest; +import org.apache.iotdb.collector.api.v1.AdminApiService; +import org.apache.iotdb.collector.api.v1.NotFoundException; +import org.apache.iotdb.collector.api.v1.handler.RequestValidationHandler; +import org.apache.iotdb.collector.api.v1.model.AlterPipeRequest; +import org.apache.iotdb.collector.api.v1.model.CreatePipeRequest; +import org.apache.iotdb.collector.api.v1.model.DropPipeRequest; +import org.apache.iotdb.collector.api.v1.model.StartPipeRequest; +import org.apache.iotdb.collector.api.v1.model.StopPipeRequest; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/CollectorDescriptor.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/CollectorDescriptor.java index ae8bf553188b..ee0d5bcbcf19 100644 --- a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/CollectorDescriptor.java +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/CollectorDescriptor.java @@ -37,7 +37,7 @@ public class CollectorDescriptor { private static final Logger LOGGER = LoggerFactory.getLogger(CollectorDescriptor.class); - private static final String CONFIG_FILE_NAME = "iotdb-collector-system.properties"; + private static final String CONFIG_FILE_NAME = "application.properties"; private static final CollectorConfig CONFIG = new CollectorConfig(); private CollectorDescriptor() { diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/service/CollectorRestService.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/service/ApiService.java similarity index 74% rename from iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/service/CollectorRestService.java rename to iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/service/ApiService.java index b8caa7dad997..fdedb6814ae8 100644 --- a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/service/CollectorRestService.java +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/service/ApiService.java @@ -19,10 +19,9 @@ package org.apache.iotdb.collector.service; +import org.apache.iotdb.collector.api.filter.ApiOriginFilter; import org.apache.iotdb.collector.config.CollectorConfig; import org.apache.iotdb.collector.config.CollectorDescriptor; -import org.apache.iotdb.collector.protocol.rest.filter.ApiOriginFilter; -import org.apache.iotdb.commons.service.ServiceType; import org.eclipse.jetty.server.Server; import org.eclipse.jetty.servlet.ServletContextHandler; @@ -35,17 +34,16 @@ import java.util.EnumSet; -public class CollectorRestService implements IService { +public class ApiService { - private static final Logger LOGGER = LoggerFactory.getLogger(CollectorRestService.class); + private static final Logger LOGGER = LoggerFactory.getLogger(ApiService.class); private static final CollectorConfig CONFIG = CollectorDescriptor.getInstance().getConfig(); private static Server server; - private CollectorRestService() {} + private ApiService() {} - @Override public void start() { startNonSSL(CONFIG.getRestServicePort()); } @@ -65,7 +63,7 @@ private ServletContextHandler constructServletContextHandler() { holder.setInitOrder(1); holder.setInitParameter( "jersey.config.server.provider.packages", - "io.swagger.jaxrs.listing, io.swagger.sample.resource, org.apache.iotdb.collector.protocol.rest"); + "io.swagger.jaxrs.listing, io.swagger.sample.resource, org.apache.iotdb.collector.api"); holder.setInitParameter( "jersey.config.server.provider.classnames", "org.glassfish.jersey.media.multipart.MultiPartFeature"); @@ -78,36 +76,19 @@ private void serverStart() { try { server.start(); } catch (final Exception e) { - LOGGER.warn("CollectorRestService failed to start: {}", e.getMessage()); + LOGGER.warn("ApiService failed to start: {}", e.getMessage()); server.destroy(); } - LOGGER.info("start CollectorRestService successfully"); + LOGGER.info("start ApiService successfully"); } - @Override public void stop() { try { server.stop(); } catch (final Exception e) { - LOGGER.warn("CollectorRestService failed to stop: {}", e.getMessage()); + LOGGER.warn("ApiService failed to stop: {}", e.getMessage()); } finally { server.destroy(); } } - - @Override - public ServiceType getID() { - return ServiceType.REST_SERVICE; - } - - public static CollectorRestService getInstance() { - return CollectorRestServiceHolder.INSTANCE; - } - - private static class CollectorRestServiceHolder { - - private static final CollectorRestService INSTANCE = new CollectorRestService(); - - private CollectorRestServiceHolder() {} - } } diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/service/CollectorMBean.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/service/CollectorMBean.java deleted file mode 100644 index 60809ff37b11..000000000000 --- a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/service/CollectorMBean.java +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iotdb.collector.service; - -public interface CollectorMBean {} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/service/IService.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/service/IService.java deleted file mode 100644 index 95755d260852..000000000000 --- a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/service/IService.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iotdb.collector.service; - -import org.apache.iotdb.commons.exception.ShutdownException; -import org.apache.iotdb.commons.exception.StartupException; -import org.apache.iotdb.commons.service.ServiceType; - -public interface IService { - void start() throws StartupException; - - default void waitAndStop(long milliseconds) { - stop(); - } - - default void shutdown(long milliseconds) throws ShutdownException { - waitAndStop(milliseconds); - } - - void stop(); - - ServiceType getID(); -} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/service/RegisterManager.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/service/RegisterManager.java deleted file mode 100644 index a91a8e31c15d..000000000000 --- a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/service/RegisterManager.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.collector.service; - -import org.apache.iotdb.commons.exception.StartupException; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.List; - -public class RegisterManager { - private static final Logger LOGGER = LoggerFactory.getLogger(RegisterManager.class); - private final List iServices; - private static final long DEREGISTER_TIME_OUT = 10_00L; - - private RegisterManager() { - this.iServices = new ArrayList<>(); - } - - /** register service. */ - public void register(final IService service) throws StartupException { - for (final IService iService : iServices) { - if (iService.getID() == service.getID()) { - LOGGER.debug("{} has already been registered. skip", service.getID().getName()); - return; - } - } - iServices.add(service); - final long startTime = System.currentTimeMillis(); - service.start(); - final long endTime = System.currentTimeMillis(); - LOGGER.info( - "The {} service is started successfully, which takes {} ms.", - service.getID().getName(), - (endTime - startTime)); - } - - /** stop all service and clear iService list. */ - public void deregisterAll() { - for (final IService service : iServices) { - try { - service.waitAndStop(DEREGISTER_TIME_OUT); - LOGGER.debug("{} deregistered", service.getID()); - } catch (final Exception e) { - LOGGER.error("Failed to stop {} because:", service.getID().getName(), e); - } - } - iServices.clear(); - LOGGER.info("deregister all service."); - } - - public static RegisterManager getInstance() { - return RegisterManagerHolder.INSTANCE; - } - - private static class RegisterManagerHolder { - private static final RegisterManager INSTANCE = new RegisterManager(); - - private RegisterManagerHolder() {} - } -} diff --git a/iotdb-collector/collector-core/src/main/resources/iotdb-collector-system.properties b/iotdb-collector/collector-core/src/main/resources/application.properties similarity index 100% rename from iotdb-collector/collector-core/src/main/resources/iotdb-collector-system.properties rename to iotdb-collector/collector-core/src/main/resources/application.properties diff --git a/iotdb-collector/collector-openapi/pom.xml b/iotdb-collector/collector-openapi/pom.xml index ab3de19dca5f..cff506a84e49 100644 --- a/iotdb-collector/collector-openapi/pom.xml +++ b/iotdb-collector/collector-openapi/pom.xml @@ -23,7 +23,7 @@ 4.0.0 org.apache.iotdb - iotdb-collector + iotdb-application 2.0.0-SNAPSHOT collector-openapi @@ -89,20 +89,20 @@ generate-sources - ${project.basedir}/src/main/openapi3/iotdb_collector_rest_common.yaml + ${project.basedir}/src/main/openapi3/ping.yaml ${project.build.directory}/generated-sources/java - org.apache.iotdb.collector.protocol.rest - org.apache.iotdb.collector.protocol.rest.model - org.apache.iotdb.collector.protocol.rest.invoker + org.apache.iotdb.application.protocol.rest + org.apache.iotdb.application.protocol.rest.model + org.apache.iotdb.application.protocol.rest.invoker jaxrs-jersey org.apache.iotdb. - iotdb-collector-rest-service + iotdb-application-rest-service ${project.version} true Apache License 2.0 org.apache.iotdb - iotdb-collector-rest-service + iotdb-application-rest-service ${project.version} java8 true @@ -118,18 +118,18 @@ ${project.basedir}/src/main/openapi3/iotdb_collector_rest_v1.yaml ${project.build.directory}/generated-sources/java - org.apache.iotdb.collector.protocol.rest.v1 - org.apache.iotdb.collector.protocol.rest.v1.model - org.apache.iotdb.collector.protocol.rest.v1.invoker + org.apache.iotdb.application.protocol.rest.v1 + org.apache.iotdb.application.protocol.rest.v1.model + org.apache.iotdb.application.protocol.rest.v1.invoker jaxrs-jersey org.apache.iotdb - iotdb-collector-rest-service + iotdb-application-rest-service ${project.version} true Apache License 2.0 org.apache.iotdb - iotdb-collector-rest-service + iotdb-application-rest-service ${project.version} java8 true diff --git a/iotdb-collector/collector-openapi/src/main/openapi3/iotdb_collector_rest_common.yaml b/iotdb-collector/collector-openapi/src/main/openapi3/ping.yaml similarity index 97% rename from iotdb-collector/collector-openapi/src/main/openapi3/iotdb_collector_rest_common.yaml rename to iotdb-collector/collector-openapi/src/main/openapi3/ping.yaml index a81a242ac878..36adab419d96 100644 --- a/iotdb-collector/collector-openapi/src/main/openapi3/iotdb_collector_rest_common.yaml +++ b/iotdb-collector/collector-openapi/src/main/openapi3/ping.yaml @@ -19,7 +19,7 @@ openapi: 3.0.0 info: - title: iotdb_collector_rest_common + title: ping description: IoTDB Rest API for Collector license: name: Apache 2.0 diff --git a/iotdb-collector/pom.xml b/iotdb-collector/pom.xml index d23d331dc4d4..4f12d7679c60 100644 --- a/iotdb-collector/pom.xml +++ b/iotdb-collector/pom.xml @@ -26,7 +26,7 @@ iotdb-parent 2.0.0-SNAPSHOT - iotdb-collector + iotdb-application pom IoTDB: Collector From 8b7a762b88a32f67cf4d3a62208c887720f6c502 Mon Sep 17 00:00:00 2001 From: Steve Yurong Su Date: Fri, 24 Jan 2025 14:20:42 +0800 Subject: [PATCH 05/10] fix --- iotdb-collector/collector-core/pom.xml | 2 +- iotdb-collector/collector-openapi/pom.xml | 10 +++++----- iotdb-collector/pom.xml | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/iotdb-collector/collector-core/pom.xml b/iotdb-collector/collector-core/pom.xml index 1f7cf610d290..e317f3b35c47 100644 --- a/iotdb-collector/collector-core/pom.xml +++ b/iotdb-collector/collector-core/pom.xml @@ -23,7 +23,7 @@ 4.0.0 org.apache.iotdb - iotdb-application + iotdb-collector 2.0.0-SNAPSHOT collector-core diff --git a/iotdb-collector/collector-openapi/pom.xml b/iotdb-collector/collector-openapi/pom.xml index cff506a84e49..e96b50ed9f33 100644 --- a/iotdb-collector/collector-openapi/pom.xml +++ b/iotdb-collector/collector-openapi/pom.xml @@ -23,7 +23,7 @@ 4.0.0 org.apache.iotdb - iotdb-application + iotdb-collector 2.0.0-SNAPSHOT collector-openapi @@ -96,13 +96,13 @@ org.apache.iotdb.application.protocol.rest.invoker jaxrs-jersey org.apache.iotdb. - iotdb-application-rest-service + iotdb-collector-rest-service ${project.version} true Apache License 2.0 org.apache.iotdb - iotdb-application-rest-service + iotdb-collector-rest-service ${project.version} java8 true @@ -123,13 +123,13 @@ org.apache.iotdb.application.protocol.rest.v1.invoker jaxrs-jersey org.apache.iotdb - iotdb-application-rest-service + iotdb-collector-rest-service ${project.version} true Apache License 2.0 org.apache.iotdb - iotdb-application-rest-service + iotdb-collector-rest-service ${project.version} java8 true diff --git a/iotdb-collector/pom.xml b/iotdb-collector/pom.xml index 4f12d7679c60..d23d331dc4d4 100644 --- a/iotdb-collector/pom.xml +++ b/iotdb-collector/pom.xml @@ -26,7 +26,7 @@ iotdb-parent 2.0.0-SNAPSHOT - iotdb-application + iotdb-collector pom IoTDB: Collector From b9bca568cfedb65d8c83673c9ac8073a02812f47 Mon Sep 17 00:00:00 2001 From: Steve Yurong Su Date: Wed, 5 Feb 2025 18:45:19 +0800 Subject: [PATCH 06/10] refactor IService --- .../apache/iotdb/collector/Application.java | 62 ++++++++++++++++--- .../collector/config/CollectorConfig.java | 4 -- .../iotdb/collector/service/ApiService.java | 48 +++++++------- .../iotdb/collector/service/IService.java | 36 +++++++++++ iotdb-collector/collector-openapi/pom.xml | 8 +-- 5 files changed, 120 insertions(+), 38 deletions(-) create mode 100644 iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/service/IService.java diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/Application.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/Application.java index 959472b1cb7f..b0b9430e83b1 100644 --- a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/Application.java +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/Application.java @@ -19,29 +19,77 @@ package org.apache.iotdb.collector; -import org.apache.iotdb.collector.config.CollectorConfig; import org.apache.iotdb.collector.config.CollectorDescriptor; +import org.apache.iotdb.collector.service.ApiService; +import org.apache.iotdb.collector.service.IService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.nio.charset.Charset; +import java.util.LinkedList; public class Application { private static final Logger LOGGER = LoggerFactory.getLogger(Application.class); - private static final CollectorConfig COLLECTOR_CONFIG = - CollectorDescriptor.getInstance().getConfig(); + private final LinkedList services = new LinkedList<>(); - public static void main(String[] args) { - LOGGER.info("Environment variables: {}", CollectorConfig.getEnvironmentVariables()); + private Application() { + services.add(new ApiService()); + + LOGGER.info( + "IoTDB-CollectorNode configuration: {}", + CollectorDescriptor.getInstance().getConfig().getConfigMessage()); + LOGGER.info( + "Congratulations, IoTDB CollectorNode is set up successfully. Now, enjoy yourself!"); LOGGER.info("Default charset is: {}", Charset.defaultCharset().displayName()); + } + + public static void main(String[] args) { + LOGGER.info("[Application] Starting ..."); + final long startTime = System.currentTimeMillis(); final Application application = new Application(); - LOGGER.info("IoTDB-CollectorNode configuration: {}", COLLECTOR_CONFIG.getConfigMessage()); + application.registerShutdownHook(); + application.startServices(); + LOGGER.info( - "Congratulations, IoTDB CollectorNode is set up successfully. Now, enjoy yourself!"); + "[Application] Successfully started in {}ms", System.currentTimeMillis() - startTime); + } + + public void registerShutdownHook() { + Runtime.getRuntime() + .addShutdownHook( + new Thread( + () -> { + LOGGER.warn("[Application] Exiting ..."); + + for (final IService service : services) { + try { + service.stop(); + } catch (final Exception e) { + LOGGER.warn( + "[{}] Unexpected exception occurred when stopping: {}", + service.name(), + e.getMessage(), + e); + } + } + + LOGGER.warn( + "[Application] JVM report: total memory {}, free memory {}, used memory {}", + Runtime.getRuntime().totalMemory(), + Runtime.getRuntime().freeMemory(), + Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()); + LOGGER.warn("[Application] Exited."); + })); + } + + public void startServices() { + for (final IService service : services) { + service.start(); + } } } diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/CollectorConfig.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/CollectorConfig.java index 6f0cb1268650..ea8266084ec3 100644 --- a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/CollectorConfig.java +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/CollectorConfig.java @@ -31,10 +31,6 @@ public class CollectorConfig { private int restServicePort = 17070; - public static String getEnvironmentVariables() { - return ""; - } - public int getRestServicePort() { return restServicePort; } diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/service/ApiService.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/service/ApiService.java index fdedb6814ae8..da7c23ad4797 100644 --- a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/service/ApiService.java +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/service/ApiService.java @@ -20,7 +20,6 @@ package org.apache.iotdb.collector.service; import org.apache.iotdb.collector.api.filter.ApiOriginFilter; -import org.apache.iotdb.collector.config.CollectorConfig; import org.apache.iotdb.collector.config.CollectorDescriptor; import org.eclipse.jetty.server.Server; @@ -34,24 +33,25 @@ import java.util.EnumSet; -public class ApiService { +public class ApiService implements IService { private static final Logger LOGGER = LoggerFactory.getLogger(ApiService.class); - private static final CollectorConfig CONFIG = CollectorDescriptor.getInstance().getConfig(); - - private static Server server; - - private ApiService() {} + private Server server; + @Override public void start() { - startNonSSL(CONFIG.getRestServicePort()); - } - - private void startNonSSL(final int restServicePort) { - server = new Server(restServicePort); + server = new Server(CollectorDescriptor.getInstance().getConfig().getRestServicePort()); server.setHandler(constructServletContextHandler()); - serverStart(); + try { + server.start(); + LOGGER.info( + "[ApiService] Started successfully. Listening on port {}", + CollectorDescriptor.getInstance().getConfig().getRestServicePort()); + } catch (final Exception e) { + LOGGER.warn("[ApiService] Failed to start: {}", e.getMessage(), e); + server.destroy(); + } } private ServletContextHandler constructServletContextHandler() { @@ -72,23 +72,25 @@ private ServletContextHandler constructServletContextHandler() { return context; } - private void serverStart() { - try { - server.start(); - } catch (final Exception e) { - LOGGER.warn("ApiService failed to start: {}", e.getMessage()); - server.destroy(); + @Override + public void stop() { + if (server == null) { + LOGGER.info("[ApiService] Not started yet. Nothing to stop."); + return; } - LOGGER.info("start ApiService successfully"); - } - public void stop() { try { server.stop(); + LOGGER.info("[ApiService] Stopped successfully."); } catch (final Exception e) { - LOGGER.warn("ApiService failed to stop: {}", e.getMessage()); + LOGGER.warn("[ApiService] Failed to stop: {}", e.getMessage(), e); } finally { server.destroy(); } } + + @Override + public String name() { + return "ApiService"; + } } diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/service/IService.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/service/IService.java new file mode 100644 index 000000000000..16358bb51cb4 --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/service/IService.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.service; + +public interface IService { + + /** Start the service. */ + void start(); + + /** Stop the service. */ + void stop(); + + /** + * Get the name of the service. + * + * @return the name of the service + */ + String name(); +} diff --git a/iotdb-collector/collector-openapi/pom.xml b/iotdb-collector/collector-openapi/pom.xml index e96b50ed9f33..3ba2259f6636 100644 --- a/iotdb-collector/collector-openapi/pom.xml +++ b/iotdb-collector/collector-openapi/pom.xml @@ -157,10 +157,10 @@ + TODO: For some reason I am unable to prevent the generation of the default implementations in the "impl" + package. Also I was unable to prevent the maven-compiler-plugin from compiling them. So we're simply + excluding them from the jar as a measure of last resort. + --> org.apache.maven.plugins maven-jar-plugin From 4f4bc968aee7dde3909c4551916a80c5399f6bcf Mon Sep 17 00:00:00 2001 From: Steve Yurong Su Date: Thu, 6 Feb 2025 10:17:50 +0800 Subject: [PATCH 07/10] refactor CollectorConfig.java --- .../java/org/apache/iotdb/collector/Application.java | 2 +- .../apache/iotdb/collector/config/CollectorConfig.java | 9 ++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/Application.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/Application.java index b0b9430e83b1..37595768df9d 100644 --- a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/Application.java +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/Application.java @@ -40,7 +40,7 @@ private Application() { LOGGER.info( "IoTDB-CollectorNode configuration: {}", - CollectorDescriptor.getInstance().getConfig().getConfigMessage()); + CollectorDescriptor.getInstance().getConfig().getAllFormattedConfigFields()); LOGGER.info( "Congratulations, IoTDB CollectorNode is set up successfully. Now, enjoy yourself!"); LOGGER.info("Default charset is: {}", Charset.defaultCharset().displayName()); diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/CollectorConfig.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/CollectorConfig.java index ea8266084ec3..3b843ea67053 100644 --- a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/CollectorConfig.java +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/CollectorConfig.java @@ -39,13 +39,12 @@ public void setRestServicePort(int restServicePort) { this.restServicePort = restServicePort; } - public String getConfigMessage() { + public String getAllFormattedConfigFields() { final StringBuilder configMessage = new StringBuilder(); - String configContent; - for (final Field configField : CollectorConfig.class.getDeclaredFields()) { try { final String configType = configField.getGenericType().getTypeName(); + final String configContent; if (configType.contains("java.lang.String[][]")) { final String[][] configList = (String[][]) configField.get(this); final StringBuilder builder = new StringBuilder(); @@ -66,10 +65,10 @@ public String getConfigMessage() { .append(configContent) .append(";"); } catch (final IllegalAccessException e) { - LOGGER.warn("failed to show config message", e); + LOGGER.warn("Failed to get config message for field {}: {}", + configField.getName(), e.getMessage(), e); } } - return configMessage.toString(); } } From d27c8d021281a2b8125dfdbb36ffd4ef52ac7466 Mon Sep 17 00:00:00 2001 From: Steve Yurong Su Date: Thu, 6 Feb 2025 17:47:11 +0800 Subject: [PATCH 08/10] refactor --- .../apache/iotdb/collector/Application.java | 20 ++-- .../collector/config/ApiServiceOptions.java | 31 ++++++ .../collector/config/CollectorConfig.java | 74 ------------ ...ctorDescriptor.java => Configuration.java} | 54 +++------ .../iotdb/collector/config/Options.java | 105 ++++++++++++++++++ .../iotdb/collector/service/ApiService.java | 6 +- .../src/main/resources/application.properties | 2 +- 7 files changed, 165 insertions(+), 127 deletions(-) create mode 100644 iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/ApiServiceOptions.java delete mode 100644 iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/CollectorConfig.java rename iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/{CollectorDescriptor.java => Configuration.java} (60%) create mode 100644 iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/Options.java diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/Application.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/Application.java index 37595768df9d..6cf6456d98cc 100644 --- a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/Application.java +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/Application.java @@ -19,31 +19,24 @@ package org.apache.iotdb.collector; -import org.apache.iotdb.collector.config.CollectorDescriptor; +import org.apache.iotdb.collector.config.Configuration; import org.apache.iotdb.collector.service.ApiService; import org.apache.iotdb.collector.service.IService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.nio.charset.Charset; import java.util.LinkedList; public class Application { private static final Logger LOGGER = LoggerFactory.getLogger(Application.class); + private final Configuration configuration = new Configuration(); private final LinkedList services = new LinkedList<>(); private Application() { services.add(new ApiService()); - - LOGGER.info( - "IoTDB-CollectorNode configuration: {}", - CollectorDescriptor.getInstance().getConfig().getAllFormattedConfigFields()); - LOGGER.info( - "Congratulations, IoTDB CollectorNode is set up successfully. Now, enjoy yourself!"); - LOGGER.info("Default charset is: {}", Charset.defaultCharset().displayName()); } public static void main(String[] args) { @@ -52,6 +45,7 @@ public static void main(String[] args) { final Application application = new Application(); + application.logAllOptions(); application.registerShutdownHook(); application.startServices(); @@ -59,7 +53,11 @@ public static void main(String[] args) { "[Application] Successfully started in {}ms", System.currentTimeMillis() - startTime); } - public void registerShutdownHook() { + private void logAllOptions() { + configuration.logAllOptions(); + } + + private void registerShutdownHook() { Runtime.getRuntime() .addShutdownHook( new Thread( @@ -87,7 +85,7 @@ public void registerShutdownHook() { })); } - public void startServices() { + private void startServices() { for (final IService service : services) { service.start(); } diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/ApiServiceOptions.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/ApiServiceOptions.java new file mode 100644 index 000000000000..2e3ee556a16a --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/ApiServiceOptions.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.config; + +public class ApiServiceOptions extends Options { + + public static final Option PORT = + new Option("api_service_port", 17070) { + @Override + public void setValue(String valueString) { + value = Integer.parseInt(valueString); + } + }; +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/CollectorConfig.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/CollectorConfig.java deleted file mode 100644 index 3b843ea67053..000000000000 --- a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/CollectorConfig.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.collector.config; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.lang.reflect.Field; -import java.util.Arrays; - -public class CollectorConfig { - - private static final Logger LOGGER = LoggerFactory.getLogger(CollectorConfig.class); - - private int restServicePort = 17070; - - public int getRestServicePort() { - return restServicePort; - } - - public void setRestServicePort(int restServicePort) { - this.restServicePort = restServicePort; - } - - public String getAllFormattedConfigFields() { - final StringBuilder configMessage = new StringBuilder(); - for (final Field configField : CollectorConfig.class.getDeclaredFields()) { - try { - final String configType = configField.getGenericType().getTypeName(); - final String configContent; - if (configType.contains("java.lang.String[][]")) { - final String[][] configList = (String[][]) configField.get(this); - final StringBuilder builder = new StringBuilder(); - for (final String[] strings : configList) { - builder.append(Arrays.asList(strings)).append(";"); - } - configContent = builder.toString(); - } else if (configType.contains("java.lang.String[]")) { - final String[] configList = (String[]) configField.get(this); - configContent = Arrays.asList(configList).toString(); - } else { - configContent = configField.get(this).toString(); - } - configMessage - .append("\n\t") - .append(configField.getName()) - .append("=") - .append(configContent) - .append(";"); - } catch (final IllegalAccessException e) { - LOGGER.warn("Failed to get config message for field {}: {}", - configField.getName(), e.getMessage(), e); - } - } - return configMessage.toString(); - } -} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/CollectorDescriptor.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/Configuration.java similarity index 60% rename from iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/CollectorDescriptor.java rename to iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/Configuration.java index ee0d5bcbcf19..e41144ce69d0 100644 --- a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/CollectorDescriptor.java +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/Configuration.java @@ -33,47 +33,45 @@ import java.util.Optional; import java.util.Properties; -public class CollectorDescriptor { +public class Configuration { - private static final Logger LOGGER = LoggerFactory.getLogger(CollectorDescriptor.class); + private static final Logger LOGGER = LoggerFactory.getLogger(Configuration.class); private static final String CONFIG_FILE_NAME = "application.properties"; - private static final CollectorConfig CONFIG = new CollectorConfig(); - private CollectorDescriptor() { + private final Options options = new Options(); + + public Configuration() { loadProps(); } private void loadProps() { - final TrimProperties collectorProperties = new TrimProperties(); final Optional url = getPropsUrl(); - if (url.isPresent()) { try (final InputStream inputStream = url.get().openStream()) { LOGGER.info("Start to read config file {}", url.get()); final Properties properties = new Properties(); properties.load(new InputStreamReader(inputStream, StandardCharsets.UTF_8)); - collectorProperties.putAll(properties); - loadProperties(collectorProperties); + final TrimProperties trimProperties = new TrimProperties(); + trimProperties.putAll(properties); + options.loadProperties(trimProperties); } catch (final FileNotFoundException e) { - LOGGER.error("Fail to find config file {}, reject CollectorNode startup.", url.get(), e); + LOGGER.error("Fail to find config file, reject startup.", e); System.exit(-1); } catch (final IOException e) { - LOGGER.error("Cannot load config file, reject CollectorNode startup.", e); + LOGGER.error("IO exception when reading config file, reject startup.", e); System.exit(-1); } catch (final Exception e) { - LOGGER.error("Incorrect format in config file, reject CollectorNode startup.", e); + LOGGER.error("Unexpected exception when reading config file, reject startup.", e); System.exit(-1); } } else { - LOGGER.warn( - "Couldn't load the configuration {} from any of the known sources.", CONFIG_FILE_NAME); - System.exit(-1); + LOGGER.warn("{} is not found, use default configuration", CONFIG_FILE_NAME); } } - private static Optional getPropsUrl() { - final URL url = CollectorConfig.class.getResource("/" + CONFIG_FILE_NAME); + private Optional getPropsUrl() { + final URL url = Options.class.getResource("/" + CONFIG_FILE_NAME); if (url != null) { return Optional.of(url); @@ -82,31 +80,11 @@ private static Optional getPropsUrl() { "Cannot find IOTDB_COLLECTOR_HOME or IOTDB_COLLECTOR_CONF environment variable when loading " + "config file {}, use default configuration", CONFIG_FILE_NAME); - return Optional.empty(); } } - // properties config - private void loadProperties(final TrimProperties properties) { - CONFIG.setRestServicePort( - Integer.parseInt( - Optional.ofNullable(properties.getProperty("collector_rest_service_port")) - .orElse(String.valueOf(CONFIG.getRestServicePort())))); - } - - public static CollectorDescriptor getInstance() { - return CollectorDescriptorHolder.INSTANCE; - } - - public CollectorConfig getConfig() { - return CONFIG; - } - - private static class CollectorDescriptorHolder { - - private static final CollectorDescriptor INSTANCE = new CollectorDescriptor(); - - private CollectorDescriptorHolder() {} + public void logAllOptions() { + options.logAllOptions(); } } diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/Options.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/Options.java new file mode 100644 index 000000000000..bad372bf8f1d --- /dev/null +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/config/Options.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.collector.config; + +import org.apache.iotdb.commons.conf.TrimProperties; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; + +public class Options { + + private static final Logger LOGGER = LoggerFactory.getLogger(Options.class); + + private static final Map> OPTIONS = new ConcurrentHashMap<>(); + + public abstract static class Option { + + private final String key; + private final T defaultValue; + protected T value; + + Option(final String key, final T defaultValue) { + this.key = key; + this.defaultValue = defaultValue; + + OPTIONS.put(key, this); + } + + public String key() { + return key; + } + + public boolean hasDefaultValue() { + return defaultValue != null; + } + + public T defaultValue() { + return defaultValue; + } + + public T value() { + return value == null ? defaultValue : value; + } + + public abstract void setValue(final String valueString); + + @Override + public String toString() { + return key + " = " + value(); + } + } + + public void loadProperties(final TrimProperties properties) { + properties + .stringPropertyNames() + .forEach( + key -> { + final Option option = OPTIONS.get(key); + if (option != null) { + try { + option.setValue(properties.getProperty(key)); + } catch (final Exception e) { + LOGGER.warn( + "Unexpected exception when setting value for option: {}, given value: {}", + key, + properties.getProperty(key), + e); + } + } + }); + } + + public Optional> getOption(final String key) { + return Optional.ofNullable(OPTIONS.get(key)); + } + + public void logAllOptions() { + LOGGER.info("========================== OPTIONS =========================="); + for (final Option option : OPTIONS.values()) { + LOGGER.info(option.toString()); + } + LOGGER.info("============================================================="); + } +} diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/service/ApiService.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/service/ApiService.java index da7c23ad4797..b079014794ad 100644 --- a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/service/ApiService.java +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/service/ApiService.java @@ -20,7 +20,7 @@ package org.apache.iotdb.collector.service; import org.apache.iotdb.collector.api.filter.ApiOriginFilter; -import org.apache.iotdb.collector.config.CollectorDescriptor; +import org.apache.iotdb.collector.config.ApiServiceOptions; import org.eclipse.jetty.server.Server; import org.eclipse.jetty.servlet.ServletContextHandler; @@ -41,13 +41,13 @@ public class ApiService implements IService { @Override public void start() { - server = new Server(CollectorDescriptor.getInstance().getConfig().getRestServicePort()); + server = new Server(ApiServiceOptions.PORT.value()); server.setHandler(constructServletContextHandler()); try { server.start(); LOGGER.info( "[ApiService] Started successfully. Listening on port {}", - CollectorDescriptor.getInstance().getConfig().getRestServicePort()); + ApiServiceOptions.PORT.value()); } catch (final Exception e) { LOGGER.warn("[ApiService] Failed to start: {}", e.getMessage(), e); server.destroy(); diff --git a/iotdb-collector/collector-core/src/main/resources/application.properties b/iotdb-collector/collector-core/src/main/resources/application.properties index 4e5db9edc942..7bcf3f0356b3 100644 --- a/iotdb-collector/collector-core/src/main/resources/application.properties +++ b/iotdb-collector/collector-core/src/main/resources/application.properties @@ -17,4 +17,4 @@ # under the License. # -collector_rest_service_port=17070 \ No newline at end of file +api_service_port=17070 \ No newline at end of file From 959b57527ac61a0399e2f115b6d2cf355ea153df Mon Sep 17 00:00:00 2001 From: 0xB <76414902+YC27@users.noreply.github.com> Date: Thu, 20 Feb 2025 14:25:08 +0800 Subject: [PATCH 09/10] Collector Node: Fix compile error & dependency issue (#14906) --- .github/workflows/cluster-it-1c1d.yml | 9 +- .github/workflows/cluster-it-1c1d1a.yml | 9 +- .github/workflows/cluster-it-1c3d.yml | 9 +- .github/workflows/compile-check.yml | 11 +- .github/workflows/dependency-check.yml | 1 - .github/workflows/multi-language-client.yml | 1 - .../{pipe-it-2cluster.yml => pipe-it.yml} | 314 +++++- .github/workflows/sonar-codecov.yml | 1 - .github/workflows/table-cluster-it-1c1d.yml | 9 +- .github/workflows/table-cluster-it-1c3d.yml | 9 +- .github/workflows/unit-test.yml | 1 - code-coverage/pom.xml | 2 +- distribution/pom.xml | 12 +- docker/src/main/DockerCompose/entrypoint.sh | 5 +- .../DockerCompose/replace-conf-from-env.sh | 33 +- example/client-cpp-example/pom.xml | 2 +- example/jdbc/pom.xml | 2 +- example/mqtt-customize/pom.xml | 2 +- example/mqtt/pom.xml | 2 +- example/pipe-count-point-processor/pom.xml | 2 +- example/pipe-opc-ua-sink/pom.xml | 2 +- example/pom.xml | 2 +- example/rest-java-example/pom.xml | 2 +- example/schema/pom.xml | 2 +- example/session/pom.xml | 2 +- example/trigger/pom.xml | 2 +- example/udf/pom.xml | 2 +- integration-test/pom.xml | 74 +- .../iotdb/it/env/cluster/env/AbstractEnv.java | 12 + .../it/env/remote/env/RemoteServerEnv.java | 10 + ... MultiClusterIT2DualTableManualBasic.java} | 2 +- ...ultiClusterIT2DualTableManualEnhanced.java | 22 + ... => MultiClusterIT2DualTreeAutoBasic.java} | 2 +- .../MultiClusterIT2DualTreeAutoEnhanced.java | 22 + ...ava => MultiClusterIT2DualTreeManual.java} | 2 +- .../org/apache/iotdb/itbase/env/BaseEnv.java | 3 + .../itbase/runtime/ClusterTestStatement.java | 5 +- .../iotdb/db/it/IoTDBInsertWithQueryIT.java | 1 + .../apache/iotdb/db/it/IoTDBLoadTsFileIT.java | 73 ++ .../apache/iotdb/db/it/auth/IoTDBAuthIT.java | 38 + .../db/it/auth/IoTDBRelationalAuthIT.java | 183 +++ .../db/it/auth/IoTDBSystemPermissionIT.java | 2 +- .../iotdb/db/it/query/IoTDBQueryDemoIT.java | 25 + .../apache/iotdb/db/it/utils/TestUtils.java | 227 +++- .../tablemodel/TableModelUtils.java | 2 +- .../AbstractPipeTableModelDualManualIT.java} | 4 +- .../manual/basic}/IoTDBPipeAlterIT.java | 10 +- .../manual/basic}/IoTDBPipeDataSinkIT.java | 14 +- .../manual/basic}/IoTDBPipeExtractorIT.java | 10 +- .../manual/basic}/IoTDBPipeIsolationIT.java | 10 +- .../manual/basic}/IoTDBPipeLifeCycleIT.java | 157 ++- .../manual/basic}/IoTDBPipeProtocolIT.java | 10 +- .../basic}/IoTDBPipeSwitchStatusIT.java | 10 +- .../manual/basic}/IoTDBPipeSyntaxIT.java | 9 +- .../manual/basic}/IoTDBPipeWithLoadIT.java | 10 +- .../basic}/IoTDBTablePatternFormatIT.java | 10 +- .../enhanced}/IoTDBPipeAutoConflictIT.java | 10 +- .../manual/enhanced}/IoTDBPipeAutoDropIT.java | 10 +- .../manual/enhanced}/IoTDBPipeClusterIT.java | 10 +- .../IoTDBPipeConnectorCompressionIT.java | 10 +- .../IoTDBPipeConnectorParallelIT.java | 10 +- .../enhanced}/IoTDBPipeDoubleLivingIT.java | 13 +- .../enhanced/IoTDBPipeIdempotentIT.java | 197 ++++ .../manual/enhanced/IoTDBPipeMetaIT.java} | 87 +- .../enhanced}/IoTDBPipeNullValueIT.java | 10 +- .../IoTDBPipeTypeConversionISessionIT.java | 18 +- .../enhanced}/IoTDBPipeTypeConversionIT.java | 18 +- .../AbstractPipeDualTreeModelAutoIT.java} | 4 +- .../auto/basic}/IoTDBPipeAlterIT.java | 9 +- .../basic}/IoTDBPipeConnectorParallelIT.java | 9 +- .../auto/basic}/IoTDBPipeDataSinkIT.java | 9 +- .../auto/basic}/IoTDBPipeExtractorIT.java | 9 +- .../auto/basic}/IoTDBPipeLifeCycleIT.java | 22 +- .../auto/basic}/IoTDBPipeProcessorIT.java | 9 +- .../auto/basic}/IoTDBPipeProtocolIT.java | 9 +- .../auto/basic}/IoTDBPipeSwitchStatusIT.java | 9 +- .../auto/basic}/IoTDBPipeSyntaxIT.java | 9 +- .../auto/basic}/IoTDBTreePatternFormatIT.java | 9 +- .../enhanced}/IoTDBPipeAutoConflictIT.java | 9 +- .../auto/enhanced}/IoTDBPipeAutoDropIT.java | 9 +- .../auto/enhanced}/IoTDBPipeClusterIT.java | 11 +- .../IoTDBPipeConditionalOperationsIT.java | 9 +- .../IoTDBPipeConnectorCompressionIT.java | 9 +- .../auto/enhanced}/IoTDBPipeIdempotentIT.java | 100 +- .../auto/enhanced}/IoTDBPipeNullValueIT.java | 9 +- .../auto/enhanced}/IoTDBPipeWithLoadIT.java | 9 +- .../auto/enhanced}/PipeNowFunctionIT.java | 9 +- .../AbstractPipeDualTreeModelManualIT.java} | 4 +- .../manual/IoTDBPipeInclusionIT.java | 8 +- .../manual/IoTDBPipeManualConflictIT.java | 8 +- .../manual/IoTDBPipeMetaHistoricalIT.java | 8 +- .../manual/IoTDBPipeMetaLeaderChangeIT.java | 8 +- .../manual/IoTDBPipeMetaRestartIT.java | 8 +- .../manual/IoTDBPipeMultiSchemaRegionIT.java | 8 +- .../manual/IoTDBPipePermissionIT.java | 8 +- .../manual/IoTDBPipeReqAutoSliceIT.java | 8 +- .../IoTDBPipeTypeConversionISessionIT.java | 8 +- .../manual/IoTDBPipeTypeConversionIT.java | 8 +- .../pipe/it/single/IoTDBPipeOPCUAIT.java | 2 +- .../it/db/it/IoTDBAuthenticationTableIT.java | 1001 +++++++++++++++++ .../it/db/it/IoTDBDeletionTableIT.java | 44 +- .../it/db/it/IoTDBInsertTableIT.java | 11 + .../db/it/IoTDBLoadConfigurationTableIT.java | 85 ++ .../it/db/it/IoTDBSetSystemStatusTableIT.java | 104 ++ .../old/query/IoTDBGreatestLeastTableIT.java | 305 +++++ .../it/query/recent/IoTDBMaintainAuthIT.java | 289 +++++ .../it/query/recent/IoTDBNullIdQueryIT.java | 33 + .../it/query/recent/IoTDBNullValueIT.java | 75 ++ .../it/query/recent/IoTDBQueryAuthIT.java | 320 ++++++ .../relational/it/schema/IoTDBDatabaseIT.java | 145 ++- .../relational/it/schema/IoTDBTableIT.java | 57 + .../session/it/IoTDBSessionInsertNullIT.java | 58 + .../it/local/IoTDBSubscriptionBasicIT.java | 67 ++ iotdb-api/external-api/pom.xml | 2 +- iotdb-api/pipe-api/pom.xml | 2 +- iotdb-api/pom.xml | 2 +- iotdb-api/trigger-api/pom.xml | 2 +- iotdb-api/udf-api/pom.xml | 2 +- iotdb-client/cli/pom.xml | 22 +- .../tool/common/ImportTsFileOperation.java | 59 + .../iotdb/tool/data/AbstractDataTool.java | 6 +- .../apache/iotdb/tool/data/ImportData.java | 22 +- .../iotdb/tool/tsfile/ImportTsFile.java | 57 +- .../iotdb/tool/tsfile/ImportTsFileBase.java | 9 +- .../iotdb/tool/ImportTsFileOperationTest.java | 49 + iotdb-client/client-cpp/pom.xml | 4 +- iotdb-client/client-py/pom.xml | 6 +- iotdb-client/isession/pom.xml | 8 +- iotdb-client/jdbc/pom.xml | 8 +- .../java/org/apache/iotdb/jdbc/Constant.java | 3 + .../apache/iotdb/jdbc/IoTDBConnection.java | 4 + .../iotdb/jdbc/IoTDBConnectionParams.java | 4 +- .../org/apache/iotdb/jdbc/IoTDBStatement.java | 7 + iotdb-client/pom.xml | 2 +- iotdb-client/service-rpc/pom.xml | 6 +- .../java/org/apache/iotdb/rpc/RpcUtils.java | 14 + .../org/apache/iotdb/rpc/RpcUtilsTest.java | 10 + iotdb-client/session/pom.xml | 10 +- .../org/apache/iotdb/session/Session.java | 25 +- .../iotdb/session/SessionConnection.java | 12 +- .../iotdb/session/TableSessionBuilder.java | 4 +- .../iotdb/session/pool/SessionPool.java | 5 +- .../session/pool/TableSessionWrapper.java | 15 + .../base/AbstractSubscriptionConsumer.java | 4 +- .../base/AbstractSubscriptionProviders.java | 48 +- iotdb-collector/collector-core/pom.xml | 49 +- .../api/impl/PingApiServiceImpl.java | 7 +- .../v1/handler/RequestValidationHandler.java | 4 +- .../api/v1/impl/AdminApiServiceImpl.java | 14 +- iotdb-collector/collector-openapi/pom.xml | 2 +- iotdb-collector/pom.xml | 9 +- iotdb-core/ainode/pom.xml | 10 +- iotdb-core/ainode/pyproject.toml | 14 +- .../ainode/resources/conf/ainode-env.sh | 2 +- iotdb-core/antlr/pom.xml | 2 +- .../iotdb/db/qp/sql/IdentifierParser.g4 | 5 + .../apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 14 +- .../org/apache/iotdb/db/qp/sql/SqlLexer.g4 | 20 + iotdb-core/confignode/pom.xml | 26 +- .../request/ConfigPhysicalPlanVisitor.java | 221 +++- .../confignode/manager/PermissionManager.java | 7 +- .../confignode/manager/ProcedureManager.java | 36 +- .../event/PipeConfigRegionSnapshotEvent.java | 14 +- .../ConfigRegionListeningFilter.java | 84 +- .../extractor/IoTDBConfigRegionExtractor.java | 23 +- ...gPhysicalPlanTablePatternParseVisitor.java | 62 + ...figPhysicalPlanTableScopeParseVisitor.java | 86 ++ ...igPhysicalPlanTreePatternParseVisitor.java | 30 +- ...nfigPhysicalPlanTreeScopeParseVisitor.java | 80 ++ .../protocol/IoTDBConfigNodeReceiver.java | 162 ++- ...PipeConfigPhysicalPlanTSStatusVisitor.java | 207 +++- .../confignode/persistence/AuthorInfo.java | 70 +- ...nfigPhysicalPlanScopeParseVisitorTest.java | 116 ++ ...sicalPlanTablePatternParseVisitorTest.java | 25 + iotdb-core/consensus/pom.xml | 14 +- iotdb-core/datanode/pom.xml | 40 +- .../iotdb/db/auth/AuthorityChecker.java | 3 + .../PipeRealtimePriorityBlockingQueue.java | 2 + .../processor/PipeProcessorSubtask.java | 5 +- .../processor/PipeProcessorSubtaskWorker.java | 13 +- .../tsfile/PipeTsFileInsertionEvent.java | 49 +- .../SchemaRegionListeningFilter.java | 8 +- .../thrift/impl/ClientRPCServiceImpl.java | 27 +- .../exchange/SharedTsBlockQueue.java | 8 +- .../execution/exchange/sink/SinkChannel.java | 11 +- .../exchange/source/LocalSourceHandle.java | 4 +- .../executor/RegionWriteExecutor.java | 33 +- .../fragment/FragmentInstanceContext.java | 49 +- .../execution/operator/AbstractOperator.java | 2 +- .../process/AbstractSortOperator.java | 2 +- .../operator/process/SortOperator.java | 2 +- .../process/TableStreamSortOperator.java | 2 +- .../operator/source/FileLoaderUtils.java | 4 +- ...formationSchemaContentSupplierFactory.java | 341 +++++- .../grouped/HashAggregationOperator.java | 4 +- .../grouped/StreamingAggregationOperator.java | 9 +- .../StreamingHashAggregationOperator.java | 4 +- .../relational/ColumnTransformerBuilder.java | 15 + .../execution/schedule/DriverScheduler.java | 8 - .../db/queryengine/plan/Coordinator.java | 16 +- .../plan/analyze/AnalyzeUtils.java | 9 +- .../plan/analyze/AnalyzeVisitor.java | 4 +- .../dualkeycache/IDualKeyCacheStats.java | 6 + .../schema/dualkeycache/impl/CacheStats.java | 24 +- .../dualkeycache/impl/DualKeyCacheImpl.java | 122 +- .../plan/analyze/load/LoadTsFileAnalyzer.java | 7 + .../plan/analyze/schema/SchemaValidator.java | 13 +- .../plan/execution/IQueryExecution.java | 2 + .../plan/execution/QueryExecution.java | 5 + .../execution/config/ConfigExecution.java | 9 +- .../config/TableConfigTaskVisitor.java | 76 ++ .../config/TreeConfigTaskVisitor.java | 23 + .../executor/ClusterConfigTaskExecutor.java | 69 +- .../config/executor/IConfigTaskExecutor.java | 19 +- .../config/metadata/ShowPipePluginsTask.java | 4 +- .../metadata/region/ExtendRegionTask.java | 20 +- .../metadata/region/MigrateRegionTask.java | 21 +- .../region/ReconstructRegionTask.java | 25 +- .../metadata/region/RemoveRegionTask.java | 20 +- .../config/session/SetSqlDialectTask.java | 41 + .../queryengine/plan/parser/ASTVisitor.java | 31 +- .../plan/planner/TableOperatorGenerator.java | 9 +- .../plan/node/load/LoadTsFilePieceNode.java | 6 +- .../analyzer/StatementAnalyzer.java | 11 +- .../metadata/TableMetadataImpl.java | 20 + .../fetcher/cache/TableDeviceSchemaCache.java | 19 +- .../cache/TableDeviceSchemaCacheMetrics.java | 34 + .../DataNodeLocationSupplierFactory.java | 22 +- .../relational/security/AccessControl.java | 8 + .../security/AccessControlImpl.java | 62 +- .../security/AllowAllAccessControl.java | 5 + .../plan/relational/sql/ast/AstVisitor.java | 28 + .../plan/relational/sql/ast/ExtendRegion.java | 77 ++ .../plan/relational/sql/ast/InsertRows.java | 2 +- .../relational/sql/ast/LoadConfiguration.java | 34 + .../relational/sql/ast/MigrateRegion.java | 90 ++ .../relational/sql/ast/ReconstructRegion.java | 76 ++ .../plan/relational/sql/ast/RemoveRegion.java | 77 ++ .../relational/sql/ast/SetSqlDialect.java | 79 ++ .../relational/sql/ast/SetSystemStatus.java | 34 + .../relational/sql/parser/AstBuilder.java | 162 ++- .../scheduler/load/LoadTsFileScheduler.java | 47 +- .../plan/statement/StatementVisitor.java | 16 + .../statement/sys/SetSqlDialectStatement.java | 76 ++ .../sys/ShowCurrentSqlDialectStatement.java | 45 + .../sys/ShowCurrentUserStatement.java | 45 + ...bstractGreatestLeastColumnTransformer.java | 109 ++ .../BinaryGreatestColumnTransformer.java | 53 + .../multi/BinaryLeastColumnTransformer.java | 53 + .../BooleanGreatestColumnTransformer.java | 52 + .../multi/BooleanLeastColumnTransformer.java | 49 + .../DoubleGreatestColumnTransformer.java | 52 + .../multi/DoubleLeastColumnTransformer.java | 52 + .../multi/FloatGreatestColumnTransformer.java | 52 + .../multi/FloatLeastColumnTransformer.java | 52 + .../multi/Int32GreatestColumnTransformer.java | 52 + .../multi/Int32LeastColumnTransformer.java | 52 + .../multi/Int64GreatestColumnTransformer.java | 52 + .../multi/Int64LeastColumnTransformer.java | 52 + .../utils/ResourceByPathUtils.java | 31 +- .../iotdb/db/storageengine/StorageEngine.java | 23 +- .../impl/ReadChunkCompactionPerformer.java | 11 +- .../utils/MultiTsFileDeviceIterator.java | 14 +- ...edFastAlignedSeriesCompactionExecutor.java | 26 +- ...dChunkAlignedSeriesCompactionExecutor.java | 40 +- .../AlignedSeriesBatchCompactionUtils.java | 28 +- .../FastAlignedSeriesCompactionExecutor.java | 23 +- .../fast/SeriesCompactionExecutor.java | 10 +- ...dChunkAlignedSeriesCompactionExecutor.java | 31 +- .../writer/FastCrossCompactionWriter.java | 10 +- .../writer/FastInnerCompactionWriter.java | 10 +- .../io/CompactionDiskTSMIterator.java | 43 + .../compaction/io/CompactionTsFileInput.java | 122 +- .../compaction/io/CompactionTsFileReader.java | 60 +- .../compaction/io/CompactionTsFileWriter.java | 9 + .../memtable/AlignedReadOnlyMemChunk.java | 10 +- .../dataregion/memtable/TsFileProcessor.java | 7 +- .../reader/chunk/DiskAlignedChunkLoader.java | 5 +- .../reader/chunk/MemAlignedChunkReader.java | 4 +- .../reader/chunk/MemAlignedPageReader.java | 6 +- .../DiskAlignedChunkMetadataLoader.java | 4 +- .../storageengine/load/LoadTsFileManager.java | 5 +- ...tementDataTypeConvertExecutionVisitor.java | 4 + ...tementDataTypeConvertExecutionVisitor.java | 4 + .../broker/SubscriptionPrefetchingQueue.java | 11 + .../SubscriptionEventTabletResponse.java | 5 - .../TsFileRewriteOverPrecisedI64Scan.java | 223 ++++ .../utils/TsFileRewriteSmallRangeI64Scan.java | 230 ++++ .../db/tools/utils/TsFileSequenceScan.java | 34 +- .../db/tools/utils/TsFileStatisticScan.java | 266 +++++ .../iotdb/db/utils/ErrorHandlingUtils.java | 6 +- .../iotdb/db/utils/ModificationUtils.java | 9 +- .../db/utils/TimestampPrecisionUtils.java | 4 +- .../iotdb/db/utils/constant/SqlConstant.java | 2 +- .../queryengine/execution/exchange/Utils.java | 3 + .../operator/MergeTreeSortOperatorTest.java | 6 + .../operator/OperatorMemoryTest.java | 2 + .../plan/parser/StatementGeneratorTest.java | 3 + .../relational/analyzer/ShowQueriesTest.java | 7 +- ...sistentCompressionTypeAndEncodingTest.java | 6 +- .../FastInnerCompactionPerformerTest.java | 6 +- .../RepairUnsortedFileCompactionTest.java | 8 +- .../CompactionTableModelTestFileWriter.java | 2 +- .../utils/BatchCompactionUtilsTest.java | 39 +- .../db/utils/TimestampPrecisionUtilsTest.java | 10 + iotdb-core/metrics/core/pom.xml | 4 +- iotdb-core/metrics/interface/pom.xml | 8 +- iotdb-core/metrics/pom.xml | 2 +- iotdb-core/node-commons/pom.xml | 24 +- .../conf/iotdb-system.properties.template | 10 +- .../auth/authorizer/BasicAuthorizer.java | 21 + .../commons/auth/authorizer/IAuthorizer.java | 4 + .../iotdb/commons/auth/entity/Role.java | 8 + .../iotdb/commons/conf/CommonConfig.java | 15 +- .../iotdb/commons/conf/CommonDescriptor.java | 24 +- .../task/connection/BlockingPendingQueue.java | 14 + .../options/PipeInclusionOptions.java | 15 +- .../schema/column/ColumnHeaderConstant.java | 30 + .../schema/table/InformationSchema.java | 109 ++ .../commons/service/metric/enums/Metric.java | 1 + .../TableBuiltinScalarFunction.java | 2 + iotdb-core/pom.xml | 2 +- iotdb-core/relational-grammar/pom.xml | 2 +- .../relational/grammar/sql/RelationalSql.g4 | 30 +- iotdb-protocol/openapi/pom.xml | 2 +- iotdb-protocol/pom.xml | 2 +- iotdb-protocol/thrift-ainode/pom.xml | 4 +- iotdb-protocol/thrift-commons/pom.xml | 2 +- iotdb-protocol/thrift-confignode/pom.xml | 4 +- .../src/main/thrift/confignode.thrift | 4 + iotdb-protocol/thrift-consensus/pom.xml | 4 +- iotdb-protocol/thrift-datanode/pom.xml | 4 +- library-udf/pom.xml | 4 +- pom.xml | 19 +- 334 files changed, 9970 insertions(+), 1464 deletions(-) rename .github/workflows/{pipe-it-2cluster.yml => pipe-it.yml} (62%) rename integration-test/src/main/java/org/apache/iotdb/itbase/category/{MultiClusterIT2AutoCreateSchema.java => MultiClusterIT2DualTableManualBasic.java} (94%) create mode 100644 integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2DualTableManualEnhanced.java rename integration-test/src/main/java/org/apache/iotdb/itbase/category/{MultiClusterIT2ManualCreateSchema.java => MultiClusterIT2DualTreeAutoBasic.java} (94%) create mode 100644 integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2DualTreeAutoEnhanced.java rename integration-test/src/main/java/org/apache/iotdb/itbase/category/{MultiClusterIT2TableModel.java => MultiClusterIT2DualTreeManual.java} (94%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{ => dual}/tablemodel/TableModelUtils.java (99%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{tablemodel/AbstractPipeTableModelTestIT.java => dual/tablemodel/manual/AbstractPipeTableModelDualManualIT.java} (95%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{tablemodel => dual/tablemodel/manual/basic}/IoTDBPipeAlterIT.java (98%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{tablemodel => dual/tablemodel/manual/basic}/IoTDBPipeDataSinkIT.java (98%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{tablemodel => dual/tablemodel/manual/basic}/IoTDBPipeExtractorIT.java (99%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{tablemodel => dual/tablemodel/manual/basic}/IoTDBPipeIsolationIT.java (97%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{tablemodel => dual/tablemodel/manual/basic}/IoTDBPipeLifeCycleIT.java (83%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{tablemodel => dual/tablemodel/manual/basic}/IoTDBPipeProtocolIT.java (97%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{tablemodel => dual/tablemodel/manual/basic}/IoTDBPipeSwitchStatusIT.java (97%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{tablemodel => dual/tablemodel/manual/basic}/IoTDBPipeSyntaxIT.java (98%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{tablemodel => dual/tablemodel/manual/basic}/IoTDBPipeWithLoadIT.java (97%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{tablemodel => dual/tablemodel/manual/basic}/IoTDBTablePatternFormatIT.java (98%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{tablemodel => dual/tablemodel/manual/enhanced}/IoTDBPipeAutoConflictIT.java (96%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{tablemodel => dual/tablemodel/manual/enhanced}/IoTDBPipeAutoDropIT.java (94%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{tablemodel => dual/tablemodel/manual/enhanced}/IoTDBPipeClusterIT.java (98%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{tablemodel => dual/tablemodel/manual/enhanced}/IoTDBPipeConnectorCompressionIT.java (97%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{tablemodel => dual/tablemodel/manual/enhanced}/IoTDBPipeConnectorParallelIT.java (91%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{tablemodel => dual/tablemodel/manual/enhanced}/IoTDBPipeDoubleLivingIT.java (96%) create mode 100644 integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeIdempotentIT.java rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{manual/IoTDBPipeTableManualIT.java => dual/tablemodel/manual/enhanced/IoTDBPipeMetaIT.java} (77%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{tablemodel => dual/tablemodel/manual/enhanced}/IoTDBPipeNullValueIT.java (94%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{tablemodel => dual/tablemodel/manual/enhanced}/IoTDBPipeTypeConversionISessionIT.java (96%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{tablemodel => dual/tablemodel/manual/enhanced}/IoTDBPipeTypeConversionIT.java (97%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{autocreate/AbstractPipeDualAutoIT.java => dual/treemodel/auto/AbstractPipeDualTreeModelAutoIT.java} (97%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{autocreate => dual/treemodel/auto/basic}/IoTDBPipeAlterIT.java (98%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{autocreate => dual/treemodel/auto/basic}/IoTDBPipeConnectorParallelIT.java (92%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{autocreate => dual/treemodel/auto/basic}/IoTDBPipeDataSinkIT.java (98%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{autocreate => dual/treemodel/auto/basic}/IoTDBPipeExtractorIT.java (99%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{autocreate => dual/treemodel/auto/basic}/IoTDBPipeLifeCycleIT.java (97%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{autocreate => dual/treemodel/auto/basic}/IoTDBPipeProcessorIT.java (94%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{autocreate => dual/treemodel/auto/basic}/IoTDBPipeProtocolIT.java (98%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{autocreate => dual/treemodel/auto/basic}/IoTDBPipeSwitchStatusIT.java (97%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{autocreate => dual/treemodel/auto/basic}/IoTDBPipeSyntaxIT.java (98%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{autocreate => dual/treemodel/auto/basic}/IoTDBTreePatternFormatIT.java (95%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{autocreate => dual/treemodel/auto/enhanced}/IoTDBPipeAutoConflictIT.java (98%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{autocreate => dual/treemodel/auto/enhanced}/IoTDBPipeAutoDropIT.java (95%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{autocreate => dual/treemodel/auto/enhanced}/IoTDBPipeClusterIT.java (98%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{autocreate => dual/treemodel/auto/enhanced}/IoTDBPipeConditionalOperationsIT.java (96%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{autocreate => dual/treemodel/auto/enhanced}/IoTDBPipeConnectorCompressionIT.java (97%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{autocreate => dual/treemodel/auto/enhanced}/IoTDBPipeIdempotentIT.java (80%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{autocreate => dual/treemodel/auto/enhanced}/IoTDBPipeNullValueIT.java (97%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{autocreate => dual/treemodel/auto/enhanced}/IoTDBPipeWithLoadIT.java (93%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{autocreate => dual/treemodel/auto/enhanced}/PipeNowFunctionIT.java (97%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{manual/AbstractPipeDualManualIT.java => dual/treemodel/manual/AbstractPipeDualTreeModelManualIT.java} (97%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{ => dual/treemodel}/manual/IoTDBPipeInclusionIT.java (97%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{ => dual/treemodel}/manual/IoTDBPipeManualConflictIT.java (97%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{ => dual/treemodel}/manual/IoTDBPipeMetaHistoricalIT.java (98%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{ => dual/treemodel}/manual/IoTDBPipeMetaLeaderChangeIT.java (97%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{ => dual/treemodel}/manual/IoTDBPipeMetaRestartIT.java (96%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{ => dual/treemodel}/manual/IoTDBPipeMultiSchemaRegionIT.java (95%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{ => dual/treemodel}/manual/IoTDBPipePermissionIT.java (97%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{ => dual/treemodel}/manual/IoTDBPipeReqAutoSliceIT.java (98%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{ => dual/treemodel}/manual/IoTDBPipeTypeConversionISessionIT.java (99%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{ => dual/treemodel}/manual/IoTDBPipeTypeConversionIT.java (98%) create mode 100644 integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBAuthenticationTableIT.java create mode 100644 integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLoadConfigurationTableIT.java create mode 100644 integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBSetSystemStatusTableIT.java create mode 100644 integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/query/IoTDBGreatestLeastTableIT.java create mode 100644 integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBMaintainAuthIT.java create mode 100644 integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBNullValueIT.java create mode 100644 integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBQueryAuthIT.java create mode 100644 iotdb-client/cli/src/main/java/org/apache/iotdb/tool/common/ImportTsFileOperation.java create mode 100644 iotdb-client/cli/src/test/java/org/apache/iotdb/tool/ImportTsFileOperationTest.java create mode 100644 iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/extractor/PipeConfigPhysicalPlanTableScopeParseVisitor.java create mode 100644 iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/extractor/PipeConfigPhysicalPlanTreeScopeParseVisitor.java create mode 100644 iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/extractor/PipeConfigPhysicalPlanScopeParseVisitorTest.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/session/SetSqlDialectTask.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ExtendRegion.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/LoadConfiguration.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/MigrateRegion.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ReconstructRegion.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/RemoveRegion.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SetSqlDialect.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SetSystemStatus.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/sys/SetSqlDialectStatement.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/sys/ShowCurrentSqlDialectStatement.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/sys/ShowCurrentUserStatement.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/AbstractGreatestLeastColumnTransformer.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/BinaryGreatestColumnTransformer.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/BinaryLeastColumnTransformer.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/BooleanGreatestColumnTransformer.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/BooleanLeastColumnTransformer.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/DoubleGreatestColumnTransformer.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/DoubleLeastColumnTransformer.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/FloatGreatestColumnTransformer.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/FloatLeastColumnTransformer.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/Int32GreatestColumnTransformer.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/Int32LeastColumnTransformer.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/Int64GreatestColumnTransformer.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/Int64LeastColumnTransformer.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/io/CompactionDiskTSMIterator.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/utils/TsFileRewriteOverPrecisedI64Scan.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/utils/TsFileRewriteSmallRangeI64Scan.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/utils/TsFileStatisticScan.java diff --git a/.github/workflows/cluster-it-1c1d.yml b/.github/workflows/cluster-it-1c1d.yml index 40bbec867760..3fc0e6360dc9 100644 --- a/.github/workflows/cluster-it-1c1d.yml +++ b/.github/workflows/cluster-it-1c1d.yml @@ -4,17 +4,16 @@ on: push: branches: - master - - 'rel/1.*' - - 'rc/1.*' - - 'force_ci/**' + - 'rel/*' + - 'rc/*' paths-ignore: - 'docs/**' - 'site/**' pull_request: branches: - master - - 'rel/1.*' - - 'rc/1.*' + - 'rel/*' + - 'rc/*' paths-ignore: - 'docs/**' - 'site/**' diff --git a/.github/workflows/cluster-it-1c1d1a.yml b/.github/workflows/cluster-it-1c1d1a.yml index 42196570a57d..7801c13a26a4 100644 --- a/.github/workflows/cluster-it-1c1d1a.yml +++ b/.github/workflows/cluster-it-1c1d1a.yml @@ -4,17 +4,16 @@ on: push: branches: - master - - 'rel/1.*' - - 'rc/1.*' - - 'force_ci/**' + - 'rel/*' + - 'rc/*' paths-ignore: - 'docs/**' - 'site/**' pull_request: branches: - master - - 'rel/1.*' - - 'rc/1.*' + - 'rel/*' + - 'rc/*' - 'force_ci/**' paths-ignore: - 'docs/**' diff --git a/.github/workflows/cluster-it-1c3d.yml b/.github/workflows/cluster-it-1c3d.yml index fbc15aef5c09..e889d3c98d20 100644 --- a/.github/workflows/cluster-it-1c3d.yml +++ b/.github/workflows/cluster-it-1c3d.yml @@ -4,17 +4,16 @@ on: push: branches: - master - - 'rel/1.*' - - 'rc/1.*' - - 'force_ci/**' + - 'rel/*' + - 'rc/*' paths-ignore: - 'docs/**' - 'site/**' pull_request: branches: - master - - 'rel/1.*' - - 'rc/1.*' + - 'rel/*' + - 'rc/*' - 'force_ci/**' paths-ignore: - 'docs/**' diff --git a/.github/workflows/compile-check.yml b/.github/workflows/compile-check.yml index d70cb70219f0..92522f48970d 100644 --- a/.github/workflows/compile-check.yml +++ b/.github/workflows/compile-check.yml @@ -6,17 +6,16 @@ on: push: branches: - master - - 'rel/1.*' - - 'rc/1.*' - - 'force_ci/**' + - 'rel/*' + - 'rc/*' paths-ignore: - 'docs/**' - 'site/**' pull_request: branches: - master - - 'rel/1.*' - - 'rc/1.*' + - 'rel/*' + - 'rc/*' - 'force_ci/**' paths-ignore: - 'docs/**' @@ -51,4 +50,4 @@ jobs: - name: Compiler Test shell: bash run: | - mvn clean compile -P with-integration-tests -ntp + mvn clean package -P with-integration-tests -DskipTests -ntp diff --git a/.github/workflows/dependency-check.yml b/.github/workflows/dependency-check.yml index 954e9c7e52c4..30f29f16090c 100644 --- a/.github/workflows/dependency-check.yml +++ b/.github/workflows/dependency-check.yml @@ -8,7 +8,6 @@ on: - master - 'rel/*' - "rc/*" - - 'force_ci/**' paths-ignore: - 'docs/**' - 'site/**' diff --git a/.github/workflows/multi-language-client.yml b/.github/workflows/multi-language-client.yml index f3efb776b15c..095f22557d66 100644 --- a/.github/workflows/multi-language-client.yml +++ b/.github/workflows/multi-language-client.yml @@ -4,7 +4,6 @@ on: branches: - master - "rc/*" - - 'force_ci/**' paths: - 'pom.xml' - 'iotdb-client/pom.xml' diff --git a/.github/workflows/pipe-it-2cluster.yml b/.github/workflows/pipe-it.yml similarity index 62% rename from .github/workflows/pipe-it-2cluster.yml rename to .github/workflows/pipe-it.yml index 9be00874afbd..9e7854073af1 100644 --- a/.github/workflows/pipe-it-2cluster.yml +++ b/.github/workflows/pipe-it.yml @@ -4,9 +4,8 @@ on: push: branches: - master - - 'rel/1.*' - - 'rc/1.*' - - 'force_ci/**' + - 'rel/*' + - 'rc/*' paths-ignore: - 'docs/**' - 'site/**' @@ -14,8 +13,8 @@ on: pull_request: branches: - master - - 'rel/1.*' - - 'rc/1.*' + - 'rel/*' + - 'rc/*' - 'force_ci/**' paths-ignore: - 'docs/**' @@ -34,7 +33,110 @@ env: DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} jobs: - auto-create-schema: + single: + strategy: + fail-fast: false + max-parallel: 15 + matrix: + java: [17] + # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet. + cluster1: [LightWeightStandaloneMode, ScalableSingleNodeMode, HighPerformanceMode, PipeConsensusBatchMode, PipeConsensusStreamMode] + cluster2: [LightWeightStandaloneMode, ScalableSingleNodeMode, HighPerformanceMode] + os: [ ubuntu-latest ] + exclude: + - cluster1: LightWeightStandaloneMode + cluster2: LightWeightStandaloneMode + - cluster1: LightWeightStandaloneMode + cluster2: ScalableSingleNodeMode + - cluster1: ScalableSingleNodeMode + cluster2: LightWeightStandaloneMode + - cluster1: ScalableSingleNodeMode + cluster2: HighPerformanceMode + - cluster1: HighPerformanceMode + cluster2: LightWeightStandaloneMode + - cluster1: HighPerformanceMode + cluster2: HighPerformanceMode + - cluster1: PipeConsensusBatchMode + cluster2: LightWeightStandaloneMode + - cluster1: PipeConsensusBatchMode + cluster2: HighPerformanceMode + - cluster1: PipeConsensusStreamMode + cluster2: LightWeightStandaloneMode + - cluster1: PipeConsensusStreamMode + cluster2: HighPerformanceMode + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - name: Set up JDK ${{ matrix.java }} + uses: actions/setup-java@v4 + with: + distribution: liberica + java-version: ${{ matrix.java }} + - name: Cache Maven packages + uses: actions/cache@v4 + with: + path: ~/.m2 + key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} + restore-keys: ${{ runner.os }}-m2- + - name: Sleep for a random duration between 0 and 10000 milliseconds + run: | + sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) + - name: IT Test + shell: bash + # we do not compile client-cpp for saving time, it is tested in client.yml + # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml + run: | + retry() { + local -i max_attempts=3 + local -i attempt=1 + local -i retry_sleep=5 + local test_output + + while [ $attempt -le $max_attempts ]; do + mvn clean verify \ + -P with-integration-tests \ + -DskipUTs \ + -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ + -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \ + -pl integration-test \ + -am -PMultiClusterIT1 \ + -ntp >> ~/run-tests-$attempt.log && return 0 + test_output=$(cat ~/run-tests-$attempt.log) + + mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ + + echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" + echo "$test_output" + echo "==================== END: ~/run-tests-$attempt.log ======================" + + if echo "$test_output" | grep -q "Could not transfer artifact"; then + if [ $attempt -lt $max_attempts ]; then + echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." + sleep $retry_sleep + attempt=$((attempt + 1)) + else + echo "Test failed after $max_attempts attempts due to artifact transfer issue." + echo "Treating this as a success because the issue is likely transient." + return 0 + fi + elif [ $? -ne 0 ]; then + echo "Test failed with a different error." + return 1 + else + echo "Tests passed" + return 0 + fi + done + } + retry + - name: Upload Artifact + if: failure() + uses: actions/upload-artifact@v4 + with: + name: cluster-log-single-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }} + path: integration-test/target/cluster-logs + retention-days: 30 + dual-tree-auto-basic: strategy: fail-fast: false max-parallel: 15 @@ -78,7 +180,7 @@ jobs: -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ -DClusterConfigurations=${{ matrix.cluster }},${{ matrix.cluster }} \ -pl integration-test \ - -am -PMultiClusterIT2AutoCreateSchema \ + -am -PMultiClusterIT2DualTreeAutoBasic \ -ntp >> ~/run-tests-$attempt.log && return 0 test_output=$(cat ~/run-tests-$attempt.log) @@ -112,10 +214,113 @@ jobs: if: failure() uses: actions/upload-artifact@v4 with: - name: cluster-log-auto-create-schema-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster }}-${{ matrix.cluster }} + name: cluster-log-dual-tree-auto-basic-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster }}-${{ matrix.cluster }} + path: integration-test/target/cluster-logs + retention-days: 30 + dual-tree-auto-enhanced: + strategy: + fail-fast: false + max-parallel: 15 + matrix: + java: [17] + # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet. + cluster1: [LightWeightStandaloneMode, ScalableSingleNodeMode, HighPerformanceMode, PipeConsensusBatchMode, PipeConsensusStreamMode] + cluster2: [LightWeightStandaloneMode, ScalableSingleNodeMode, HighPerformanceMode] + os: [ ubuntu-latest ] + exclude: + - cluster1: LightWeightStandaloneMode + cluster2: LightWeightStandaloneMode + - cluster1: LightWeightStandaloneMode + cluster2: ScalableSingleNodeMode + - cluster1: ScalableSingleNodeMode + cluster2: LightWeightStandaloneMode + - cluster1: ScalableSingleNodeMode + cluster2: HighPerformanceMode + - cluster1: HighPerformanceMode + cluster2: LightWeightStandaloneMode + - cluster1: HighPerformanceMode + cluster2: HighPerformanceMode + - cluster1: PipeConsensusBatchMode + cluster2: LightWeightStandaloneMode + - cluster1: PipeConsensusBatchMode + cluster2: HighPerformanceMode + - cluster1: PipeConsensusStreamMode + cluster2: LightWeightStandaloneMode + - cluster1: PipeConsensusStreamMode + cluster2: HighPerformanceMode + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - name: Set up JDK ${{ matrix.java }} + uses: actions/setup-java@v4 + with: + distribution: liberica + java-version: ${{ matrix.java }} + - name: Cache Maven packages + uses: actions/cache@v4 + with: + path: ~/.m2 + key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} + restore-keys: ${{ runner.os }}-m2- + - name: Sleep for a random duration between 0 and 10000 milliseconds + run: | + sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) + - name: IT Test + shell: bash + # we do not compile client-cpp for saving time, it is tested in client.yml + # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml + run: | + retry() { + local -i max_attempts=3 + local -i attempt=1 + local -i retry_sleep=5 + local test_output + + while [ $attempt -le $max_attempts ]; do + mvn clean verify \ + -P with-integration-tests \ + -DskipUTs \ + -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ + -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \ + -pl integration-test \ + -am -PMultiClusterIT2DualTreeAutoEnhanced \ + -ntp >> ~/run-tests-$attempt.log && return 0 + test_output=$(cat ~/run-tests-$attempt.log) + + mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ + + echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" + echo "$test_output" + echo "==================== END: ~/run-tests-$attempt.log ======================" + + if echo "$test_output" | grep -q "Could not transfer artifact"; then + if [ $attempt -lt $max_attempts ]; then + echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." + sleep $retry_sleep + attempt=$((attempt + 1)) + else + echo "Test failed after $max_attempts attempts due to artifact transfer issue." + echo "Treating this as a success because the issue is likely transient." + return 0 + fi + elif [ $? -ne 0 ]; then + echo "Test failed with a different error." + return 1 + else + echo "Tests passed" + return 0 + fi + done + } + retry + - name: Upload Artifact + if: failure() + uses: actions/upload-artifact@v4 + with: + name: cluster-log-dual-tree-auto-enhanced-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }} path: integration-test/target/cluster-logs retention-days: 30 - manual-create-schema: + dual-tree-manual: strategy: fail-fast: false max-parallel: 15 @@ -181,7 +386,7 @@ jobs: -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \ -pl integration-test \ - -am -PMultiClusterIT2ManualCreateSchema \ + -am -PMultiClusterIT2DualTreeManual \ -ntp >> ~/run-tests-$attempt.log && return 0 test_output=$(cat ~/run-tests-$attempt.log) @@ -215,7 +420,7 @@ jobs: if: failure() uses: actions/upload-artifact@v4 with: - name: cluster-log-manual-create-schema-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }} + name: cluster-log-dual-tree-manual-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }} path: integration-test/target/cluster-logs retention-days: 30 subscription-arch-verification: @@ -464,7 +669,7 @@ jobs: name: cluster-log-subscription-regression-misc-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }} path: integration-test/target/cluster-logs retention-days: 30 - table-model: + dual-table-manual-basic: strategy: fail-fast: false max-parallel: 15 @@ -508,7 +713,7 @@ jobs: -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ -DClusterConfigurations=${{ matrix.cluster }},${{ matrix.cluster }} \ -pl integration-test \ - -am -PMultiClusterIT2TableModel \ + -am -PMultiClusterIT2DualTableManualBasic \ -ntp >> ~/run-tests-$attempt.log && return 0 test_output=$(cat ~/run-tests-$attempt.log) @@ -542,6 +747,87 @@ jobs: if: failure() uses: actions/upload-artifact@v4 with: - name: cluster-log-table-model-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster }}-${{ matrix.cluster }} + name: cluster-log-dual-table-manual-basic-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster }}-${{ matrix.cluster }} path: integration-test/target/cluster-logs retention-days: 30 + dual-table-manual-enhanced: + strategy: + fail-fast: false + max-parallel: 15 + matrix: + java: [17] + # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet. + cluster: [LightWeightStandaloneMode, ScalableSingleNodeMode, HighPerformanceMode, PipeConsensusBatchMode, PipeConsensusStreamMode] + os: [ ubuntu-latest ] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - name: Set up JDK ${{ matrix.java }} + uses: actions/setup-java@v4 + with: + distribution: liberica + java-version: ${{ matrix.java }} + - name: Cache Maven packages + uses: actions/cache@v4 + with: + path: ~/.m2 + key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} + restore-keys: ${{ runner.os }}-m2- + - name: Sleep for a random duration between 0 and 10000 milliseconds + run: | + sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) + - name: IT Test + shell: bash + # we do not compile client-cpp for saving time, it is tested in client.yml + # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml + run: | + retry() { + local -i max_attempts=3 + local -i attempt=1 + local -i retry_sleep=5 + local test_output + + while [ $attempt -le $max_attempts ]; do + mvn clean verify \ + -P with-integration-tests \ + -DskipUTs \ + -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ + -DClusterConfigurations=${{ matrix.cluster }},${{ matrix.cluster }} \ + -pl integration-test \ + -am -PMultiClusterIT2DualTableManualEnhanced \ + -ntp >> ~/run-tests-$attempt.log && return 0 + test_output=$(cat ~/run-tests-$attempt.log) + + mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ + + echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" + echo "$test_output" + echo "==================== END: ~/run-tests-$attempt.log ======================" + + if echo "$test_output" | grep -q "Could not transfer artifact"; then + if [ $attempt -lt $max_attempts ]; then + echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." + sleep $retry_sleep + attempt=$((attempt + 1)) + else + echo "Test failed after $max_attempts attempts due to artifact transfer issue." + echo "Treating this as a success because the issue is likely transient." + return 0 + fi + elif [ $? -ne 0 ]; then + echo "Test failed with a different error." + return 1 + else + echo "Tests passed" + return 0 + fi + done + } + retry + - name: Upload Artifact + if: failure() + uses: actions/upload-artifact@v4 + with: + name: cluster-log-dual-table-manual-enhanced-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster }}-${{ matrix.cluster }} + path: integration-test/target/cluster-logs + retention-days: 30 \ No newline at end of file diff --git a/.github/workflows/sonar-codecov.yml b/.github/workflows/sonar-codecov.yml index 5818a0ec7c25..638bf34a3c53 100644 --- a/.github/workflows/sonar-codecov.yml +++ b/.github/workflows/sonar-codecov.yml @@ -9,7 +9,6 @@ on: - master - "rel/*" - "rc/*" - - 'force_ci/**' paths-ignore: - "docs/**" - 'site/**' diff --git a/.github/workflows/table-cluster-it-1c1d.yml b/.github/workflows/table-cluster-it-1c1d.yml index d34b893834da..b04101c6479b 100644 --- a/.github/workflows/table-cluster-it-1c1d.yml +++ b/.github/workflows/table-cluster-it-1c1d.yml @@ -4,17 +4,16 @@ on: push: branches: - master - - 'rel/1.*' - - 'rc/1.*' - - 'force_ci/**' + - 'rel/*' + - 'rc/*' paths-ignore: - 'docs/**' - 'site/**' pull_request: branches: - master - - 'rel/1.*' - - 'rc/1.*' + - 'rel/*' + - 'rc/*' - 'force_ci/**' paths-ignore: - 'docs/**' diff --git a/.github/workflows/table-cluster-it-1c3d.yml b/.github/workflows/table-cluster-it-1c3d.yml index 09e604478964..a2603dd79138 100644 --- a/.github/workflows/table-cluster-it-1c3d.yml +++ b/.github/workflows/table-cluster-it-1c3d.yml @@ -4,17 +4,16 @@ on: push: branches: - master - - 'rel/1.*' - - 'rc/1.*' - - 'force_ci/**' + - 'rel/*' + - 'rc/*' paths-ignore: - 'docs/**' - 'site/**' pull_request: branches: - master - - 'rel/1.*' - - 'rc/1.*' + - 'rel/*' + - 'rc/*' - 'force_ci/**' paths-ignore: - 'docs/**' diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml index 565ce10dd0f8..34fb0262f995 100644 --- a/.github/workflows/unit-test.yml +++ b/.github/workflows/unit-test.yml @@ -9,7 +9,6 @@ on: - master - 'rel/*' - "rc/*" - - 'force_ci/**' paths-ignore: - 'docs/**' - 'site/**' diff --git a/code-coverage/pom.xml b/code-coverage/pom.xml index f7a1f82d5f16..02f7d8f3f09d 100644 --- a/code-coverage/pom.xml +++ b/code-coverage/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-parent - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT iotdb-code-coverage pom diff --git a/distribution/pom.xml b/distribution/pom.xml index 87a457dc2a61..22f3af9c6fe8 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-parent - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT iotdb-distribution pom @@ -33,25 +33,25 @@ org.apache.iotdb iotdb-server - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT zip org.apache.iotdb iotdb-cli - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT zip org.apache.iotdb iotdb-confignode - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT zip org.apache.iotdb library-udf - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT @@ -174,7 +174,7 @@ org.apache.iotdb iotdb-ainode - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT diff --git a/docker/src/main/DockerCompose/entrypoint.sh b/docker/src/main/DockerCompose/entrypoint.sh index 72beea010eeb..68df9db78b77 100755 --- a/docker/src/main/DockerCompose/entrypoint.sh +++ b/docker/src/main/DockerCompose/entrypoint.sh @@ -26,11 +26,8 @@ function on_stop(){ if [[ "$start_what" != "confignode" ]]; then echo "###### manually flush ######"; start-cli.sh -e "flush;" || true - stop-datanode.sh - echo "##### done ######"; - else - stop-confignode.sh; fi + stop-standalone.sh } trap 'on_stop' SIGTERM SIGKILL SIGQUIT diff --git a/docker/src/main/DockerCompose/replace-conf-from-env.sh b/docker/src/main/DockerCompose/replace-conf-from-env.sh index dc3d6cf2cf11..eb183033119c 100755 --- a/docker/src/main/DockerCompose/replace-conf-from-env.sh +++ b/docker/src/main/DockerCompose/replace-conf-from-env.sh @@ -21,36 +21,37 @@ conf_path=${IOTDB_HOME}/conf target_files="iotdb-system.properties" -function process_single() { - local key_value="$1" - local filename=$2 - local key=$(echo $key_value | cut -d = -f1) - local line=$(grep -ni "${key}=" ${filename}) - #echo "line=$line" - if [[ -n "${line}" ]]; then +function process_single(){ + local key_value="$1" + local filename=$2 + local key=$(echo $key_value|cut -d = -f1) + local line=$(grep -ni "${key}=" ${filename}) + #echo "line=$line" + if [[ -n "${line}" ]]; then echo "update $key $filename" - local line_no=$(echo $line | cut -d : -f1) - local content=$(echo $line | cut -d : -f2) + local line_no=$(echo $line|cut -d : -f1) + local content=$(echo $line|cut -d : -f2) if [[ "${content:0:1}" != "#" ]]; then sed -i "${line_no}d" ${filename} fi sed -i "${line_no} i${key_value}" ${filename} else - echo "append $key to $filename" - - echo "${key_value}" >>"${filename}" - fi + echo "append $key $filename" + line_no=$(wc -l $filename) + sed -i "${line_no} a${key_value}" ${filename} + fi } -function replace_configs() { +function replace_configs(){ for v in $(env); do if [[ "${v}" =~ "=" && "${v}" =~ "_" && ! "${v}" =~ "JAVA_" ]]; then - # echo "###### $v ####" +# echo "###### $v ####" for f in ${target_files}; do - process_single $v ${conf_path}/$f + process_single $v ${conf_path}/$f done fi done } replace_configs + diff --git a/example/client-cpp-example/pom.xml b/example/client-cpp-example/pom.xml index 4832533b9aab..b19e84c13dca 100644 --- a/example/client-cpp-example/pom.xml +++ b/example/client-cpp-example/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-examples - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT client-cpp-example IoTDB: Example: CPP Client diff --git a/example/jdbc/pom.xml b/example/jdbc/pom.xml index 73833335f0ee..d8fa0a05e22f 100644 --- a/example/jdbc/pom.xml +++ b/example/jdbc/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-examples - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT jdbc-example IoTDB: Example: JDBC diff --git a/example/mqtt-customize/pom.xml b/example/mqtt-customize/pom.xml index bd0c9e6a7c14..4d05d14dc57a 100644 --- a/example/mqtt-customize/pom.xml +++ b/example/mqtt-customize/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-examples - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT customize-mqtt-example IoTDB: Example: Customized MQTT diff --git a/example/mqtt/pom.xml b/example/mqtt/pom.xml index 892fa11c5456..18d2e83a0729 100644 --- a/example/mqtt/pom.xml +++ b/example/mqtt/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-examples - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT mqtt-example IoTDB: Example: MQTT diff --git a/example/pipe-count-point-processor/pom.xml b/example/pipe-count-point-processor/pom.xml index 4ba2d8c51217..935a5bca588d 100644 --- a/example/pipe-count-point-processor/pom.xml +++ b/example/pipe-count-point-processor/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-examples - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT pipe-count-point-processor-example IoTDB: Example: Pipe: Count Point Processor diff --git a/example/pipe-opc-ua-sink/pom.xml b/example/pipe-opc-ua-sink/pom.xml index 7846ab5e24ab..3b3ef78c7cef 100644 --- a/example/pipe-opc-ua-sink/pom.xml +++ b/example/pipe-opc-ua-sink/pom.xml @@ -23,7 +23,7 @@ org.apache.iotdb iotdb-examples - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT 4.0.0 pipe-opc-ua-sink-example diff --git a/example/pom.xml b/example/pom.xml index c8eaeb7cd90d..9912b7760909 100644 --- a/example/pom.xml +++ b/example/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-parent - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT iotdb-examples pom diff --git a/example/rest-java-example/pom.xml b/example/rest-java-example/pom.xml index 2fc64e72b60b..8af4b94104cf 100644 --- a/example/rest-java-example/pom.xml +++ b/example/rest-java-example/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-examples - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT rest-java-example IoTDB: Example: Java Rest diff --git a/example/schema/pom.xml b/example/schema/pom.xml index b2e927742c26..aef252aede13 100644 --- a/example/schema/pom.xml +++ b/example/schema/pom.xml @@ -24,7 +24,7 @@ iotdb-examples org.apache.iotdb - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT schema-example IoTDB: Example: Schema diff --git a/example/session/pom.xml b/example/session/pom.xml index a7bdd7aa1241..cd5f40dc913e 100644 --- a/example/session/pom.xml +++ b/example/session/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-examples - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT client-example IoTDB: Example: Session Client diff --git a/example/trigger/pom.xml b/example/trigger/pom.xml index efe4342b58a9..c9c57dbf652f 100644 --- a/example/trigger/pom.xml +++ b/example/trigger/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-examples - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT trigger-example IoTDB: Example: Trigger diff --git a/example/udf/pom.xml b/example/udf/pom.xml index 3449e7258b7f..9da91d7b3de9 100644 --- a/example/udf/pom.xml +++ b/example/udf/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-examples - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT udf-example IoTDB: Example: UDF diff --git a/integration-test/pom.xml b/integration-test/pom.xml index 43e30ace4ea9..6438c36e3d5c 100644 --- a/integration-test/pom.xml +++ b/integration-test/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-parent - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT integration-test IoTDB: Integration-Test @@ -87,47 +87,47 @@ org.apache.iotdb iotdb-server - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.apache.iotdb iotdb-session - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.apache.iotdb iotdb-jdbc - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.apache.iotdb trigger-api - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.apache.iotdb isession - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.apache.iotdb service-rpc - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.apache.iotdb iotdb-thrift-confignode - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.apache.iotdb node-commons - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.apache.iotdb iotdb-thrift-commons - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.apache.tsfile @@ -137,7 +137,7 @@ org.apache.iotdb udf-api - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.apache.iotdb @@ -147,7 +147,7 @@ org.apache.iotdb iotdb-consensus - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.slf4j @@ -176,17 +176,17 @@ org.apache.iotdb iotdb-confignode - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.apache.iotdb iotdb-thrift - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.apache.iotdb iotdb-cli - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT commons-codec @@ -216,7 +216,7 @@ org.apache.iotdb iotdb-server - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT test-jar test @@ -445,7 +445,7 @@ org.apache.iotdb.itbase.category.ManualIT - org.apache.iotdb.itbase.category.MultiClusterIT1,org.apache.iotdb.itbase.category.MultiClusterIT2AutoCreateSchema,org.apache.iotdb.itbase.category.MultiClusterIT2ManualCreateSchema,org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionArchVerification,org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegressionConsumer,org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegressionMisc,org.apache.iotdb.itbase.category.MultiClusterIT3,org.apache.iotdb.itbase.category.MultiClusterIT2TableModel + org.apache.iotdb.itbase.category.MultiClusterIT1,org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoBasic,org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoEnhanced,org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeManual,org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionArchVerification,org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegressionConsumer,org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegressionMisc,org.apache.iotdb.itbase.category.MultiClusterIT3,org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualBasic,org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualEnhanced false true true @@ -467,13 +467,13 @@ - MultiClusterIT2AutoCreateSchema + MultiClusterIT2DualTreeAutoBasic false org.apache.iotdb.itbase.category.ManualIT - org.apache.iotdb.itbase.category.MultiClusterIT2AutoCreateSchema + org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoBasic false true true @@ -481,13 +481,13 @@ - MultiClusterIT2ManualCreateSchema + MultiClusterIT2DualTreeAutoEnhanced false org.apache.iotdb.itbase.category.ManualIT - org.apache.iotdb.itbase.category.MultiClusterIT2ManualCreateSchema + org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoEnhanced false true true @@ -495,13 +495,41 @@ - MultiClusterIT2TableModel + MultiClusterIT2DualTableManualBasic false org.apache.iotdb.itbase.category.ManualIT - org.apache.iotdb.itbase.category.MultiClusterIT2TableModel + org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualBasic + false + true + true + MultiCluster + + + + MultiClusterIT2DualTableManualEnhanced + + false + + + org.apache.iotdb.itbase.category.ManualIT + org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualEnhanced + false + true + true + MultiCluster + + + + MultiClusterIT2DualTreeManual + + false + + + org.apache.iotdb.itbase.category.ManualIT + org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeManual false true true diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/env/AbstractEnv.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/env/AbstractEnv.java index acaf76e4514d..a255dde22911 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/env/AbstractEnv.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/env/AbstractEnv.java @@ -565,6 +565,18 @@ public ITableSession getTableSessionConnection() throws IoTDBConnectionException .build(); } + @Override + public ITableSession getTableSessionConnection(String userName, String password) + throws IoTDBConnectionException { + final DataNodeWrapper dataNode = + this.dataNodeWrapperList.get(rand.nextInt(this.dataNodeWrapperList.size())); + return new TableSessionBuilder() + .nodeUrls(Collections.singletonList(dataNode.getIpAndPortString())) + .username(userName) + .password(password) + .build(); + } + @Override public ITableSession getTableSessionConnectionWithDB(final String database) throws IoTDBConnectionException { diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/remote/env/RemoteServerEnv.java b/integration-test/src/main/java/org/apache/iotdb/it/env/remote/env/RemoteServerEnv.java index b29c5b96315b..f0daa951e829 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/remote/env/RemoteServerEnv.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/remote/env/RemoteServerEnv.java @@ -320,6 +320,16 @@ public ITableSession getTableSessionConnection(List nodeUrls) .build(); } + @Override + public ITableSession getTableSessionConnection(String userName, String password) + throws IoTDBConnectionException { + return new TableSessionBuilder() + .nodeUrls(Collections.singletonList(ip_addr + ":" + port)) + .username(userName) + .password(password) + .build(); + } + @Override public ISession getSessionConnection(String userName, String password) throws IoTDBConnectionException { diff --git a/integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2AutoCreateSchema.java b/integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2DualTableManualBasic.java similarity index 94% rename from integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2AutoCreateSchema.java rename to integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2DualTableManualBasic.java index 25000e3012ae..07eb001fcde3 100644 --- a/integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2AutoCreateSchema.java +++ b/integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2DualTableManualBasic.java @@ -19,4 +19,4 @@ package org.apache.iotdb.itbase.category; -public interface MultiClusterIT2AutoCreateSchema {} +public class MultiClusterIT2DualTableManualBasic {} diff --git a/integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2DualTableManualEnhanced.java b/integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2DualTableManualEnhanced.java new file mode 100644 index 000000000000..701401c26854 --- /dev/null +++ b/integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2DualTableManualEnhanced.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.itbase.category; + +public class MultiClusterIT2DualTableManualEnhanced {} diff --git a/integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2ManualCreateSchema.java b/integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2DualTreeAutoBasic.java similarity index 94% rename from integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2ManualCreateSchema.java rename to integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2DualTreeAutoBasic.java index 2fb36dd22920..477a606584b6 100644 --- a/integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2ManualCreateSchema.java +++ b/integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2DualTreeAutoBasic.java @@ -19,4 +19,4 @@ package org.apache.iotdb.itbase.category; -public interface MultiClusterIT2ManualCreateSchema {} +public class MultiClusterIT2DualTreeAutoBasic {} diff --git a/integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2DualTreeAutoEnhanced.java b/integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2DualTreeAutoEnhanced.java new file mode 100644 index 000000000000..7ce45f738104 --- /dev/null +++ b/integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2DualTreeAutoEnhanced.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.itbase.category; + +public class MultiClusterIT2DualTreeAutoEnhanced {} diff --git a/integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2TableModel.java b/integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2DualTreeManual.java similarity index 94% rename from integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2TableModel.java rename to integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2DualTreeManual.java index c9280cd3fc60..b9e3e2f99d12 100644 --- a/integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2TableModel.java +++ b/integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2DualTreeManual.java @@ -19,4 +19,4 @@ package org.apache.iotdb.itbase.category; -public class MultiClusterIT2TableModel {} +public class MultiClusterIT2DualTreeManual {} diff --git a/integration-test/src/main/java/org/apache/iotdb/itbase/env/BaseEnv.java b/integration-test/src/main/java/org/apache/iotdb/itbase/env/BaseEnv.java index ea044ddfd9de..0cae6d1fa254 100644 --- a/integration-test/src/main/java/org/apache/iotdb/itbase/env/BaseEnv.java +++ b/integration-test/src/main/java/org/apache/iotdb/itbase/env/BaseEnv.java @@ -210,6 +210,9 @@ IConfigNodeRPCService.Iface getLeaderConfigNodeConnection() ITableSession getTableSessionConnection(List nodeUrls) throws IoTDBConnectionException; + ITableSession getTableSessionConnection(String userName, String password) + throws IoTDBConnectionException; + /** * Get the index of the first dataNode with a SchemaRegion leader. * diff --git a/integration-test/src/main/java/org/apache/iotdb/itbase/runtime/ClusterTestStatement.java b/integration-test/src/main/java/org/apache/iotdb/itbase/runtime/ClusterTestStatement.java index d99ea24717e3..f5dc5289d126 100644 --- a/integration-test/src/main/java/org/apache/iotdb/itbase/runtime/ClusterTestStatement.java +++ b/integration-test/src/main/java/org/apache/iotdb/itbase/runtime/ClusterTestStatement.java @@ -31,6 +31,7 @@ import java.util.ArrayList; import java.util.List; +import static org.apache.iotdb.rpc.RpcUtils.isSetSqlDialect; import static org.apache.iotdb.rpc.RpcUtils.isUseDatabase; /** The implementation of {@link ClusterTestStatement} in cluster test. */ @@ -191,8 +192,8 @@ public void setCursorName(String name) { public boolean execute(String sql) throws SQLException { sql = sql.trim(); boolean result = writeStatement.execute(sql); - // if use XXXX, sendRequest to all statements - if (isUseDatabase(sql)) { + // if 'use XXXX' or 'set sql_dialect', sendRequest to all statements + if (isUseDatabase(sql) || isSetSqlDialect(sql)) { for (Statement readStatement : readStatements) { boolean tmp = readStatement.execute(sql); result = result && tmp; diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBInsertWithQueryIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBInsertWithQueryIT.java index e9823c505b13..a5032580f225 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBInsertWithQueryIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBInsertWithQueryIT.java @@ -52,6 +52,7 @@ public class IoTDBInsertWithQueryIT { @Before public void setUp() throws Exception { + EnvFactory.getEnv().getConfig().getCommonConfig().setTimestampPrecisionCheckEnabled(false); EnvFactory.getEnv().initClusterEnvironment(); } diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java index b8e431356f55..3e090f99fa56 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java @@ -21,6 +21,8 @@ import org.apache.iotdb.commons.auth.entity.PrivilegeType; import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant; +import org.apache.iotdb.db.storageengine.dataregion.compaction.tablemodel.CompactionTableModelTestFileWriter; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.it.utils.TsFileGenerator; @@ -31,8 +33,10 @@ import org.apache.iotdb.jdbc.IoTDBSQLException; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.read.common.Path; +import org.apache.tsfile.read.common.TimeRange; import org.apache.tsfile.utils.Pair; import org.apache.tsfile.write.record.Tablet; import org.apache.tsfile.write.schema.IMeasurementSchema; @@ -496,6 +500,75 @@ public void testAuth() throws Exception { String.format("load \"%s\" sgLevel=2", tmpDir.getAbsolutePath()), "test", "test123"); } + @Test + public void testTableAuth() throws Exception { + createUser("test", "test123"); + + final TsFileResource resource4 = new TsFileResource(new File(tmpDir, "test1-0-0-0.tsfile")); + try (final CompactionTableModelTestFileWriter writer = + new CompactionTableModelTestFileWriter(resource4)) { + writer.registerTableSchema("t2", Arrays.asList("id1", "id2", "id3")); + writer.startChunkGroup("t2", Arrays.asList("id_field1", "id_field2", "id_field3")); + writer.generateSimpleNonAlignedSeriesToCurrentDevice( + "s1", + new TimeRange[][][] {new TimeRange[][] {new TimeRange[] {new TimeRange(20, 22)}}}, + TSEncoding.PLAIN, + CompressionType.LZ4); + writer.generateSimpleNonAlignedSeriesToCurrentDevice( + "s2", + new TimeRange[][][] {new TimeRange[][] {new TimeRange[] {new TimeRange(20, 22)}}}, + TSEncoding.PLAIN, + CompressionType.LZ4); + writer.endChunkGroup(); + writer.endFile(); + } + + try (final Connection userCon = + EnvFactory.getEnv().getConnection("test", "test123", BaseEnv.TABLE_SQL_DIALECT); + final Statement userStmt = userCon.createStatement()) { + Assert.assertThrows( + SQLException.class, + () -> { + userStmt.execute( + String.format( + "load '%s' with ('database-level'='2', 'database-name'='test')", + tmpDir.getAbsolutePath())); + }); + } + + try (Connection adminCon = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement adminStmt = adminCon.createStatement()) { + adminStmt.execute("grant create on database test to user test"); + } + + try (final Connection userCon = + EnvFactory.getEnv().getConnection("test", "test123", BaseEnv.TABLE_SQL_DIALECT); + final Statement userStmt = userCon.createStatement()) { + Assert.assertThrows( + SQLException.class, + () -> { + userStmt.execute( + String.format( + "load '%s' with ('database-level'='2', 'database-name'='test')", + tmpDir.getAbsolutePath())); + }); + } + + try (final Connection adminCon = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement adminStmt = adminCon.createStatement()) { + adminStmt.execute("grant insert on any to user test"); + } + + try (final Connection userCon = + EnvFactory.getEnv().getConnection("test", "test123", BaseEnv.TABLE_SQL_DIALECT); + final Statement userStmt = userCon.createStatement()) { + userStmt.execute( + String.format( + "load '%s' with ('database-level'='2', 'database-name'='test')", + tmpDir.getAbsolutePath())); + } + } + @Test public void testLoadWithOnSuccess() throws Exception { final File file1 = new File(tmpDir, "1-0-0-0.tsfile"); diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/auth/IoTDBAuthIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/auth/IoTDBAuthIT.java index 277dda1fa10d..301de48bedae 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/auth/IoTDBAuthIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/auth/IoTDBAuthIT.java @@ -21,6 +21,7 @@ import org.apache.iotdb.commons.auth.entity.PrivilegeType; import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant; +import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.ClusterIT; @@ -46,6 +47,8 @@ import java.util.List; import java.util.Set; +import static org.apache.iotdb.db.it.utils.TestUtils.createUser; +import static org.apache.iotdb.db.it.utils.TestUtils.resultSetEqualTest; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -155,6 +158,30 @@ public void allPrivilegesTest() throws SQLException { Assert.assertThrows( SQLException.class, () -> userStmt.execute("GRANT USER tempuser PRIVILEGES WRITE_SCHEMA ON root.a")); + + adminStmt.execute("GRANT ALL ON root.** TO USER tempuser WITH GRANT OPTION"); + userStmt.execute("CREATE USER testuser 'password'"); + userStmt.execute("GRANT ALL ON root.** TO USER testuser WITH GRANT OPTION"); + ResultSet dataSet = userStmt.executeQuery("LIST PRIVILEGES OF USER testuser"); + + Set ansSet = + new HashSet<>( + Arrays.asList( + ",,MANAGE_USER,true,", + ",,MANAGE_ROLE,true,", + ",,USE_TRIGGER,true,", + ",,USE_UDF,true,", + ",,USE_CQ,true,", + ",,USE_PIPE,true,", + ",,USE_MODEL,true,", + ",,EXTEND_TEMPLATE,true,", + ",,MANAGE_DATABASE,true,", + ",,MAINTAIN,true,", + ",root.**,READ_DATA,true,", + ",root.**,WRITE_DATA,true,", + ",root.**,READ_SCHEMA,true,", + ",root.**,WRITE_SCHEMA,true,")); + TestUtils.assertResultSetEqual(dataSet, "Role,Scope,Privileges,GrantOption,", ansSet); } } } @@ -1253,4 +1280,15 @@ public void testCreateRoleIdentifierName() throws SQLException { adminStmt.execute("create role tail"); adminStmt.execute("create user tail 'password'"); } + + @Test + public void noNeedPrivilegeTest() { + createUser("tempuser", "temppw"); + String[] expectedHeader = new String[] {"CurrentUser"}; + String[] retArray = + new String[] { + "tempuser,", + }; + resultSetEqualTest("show current_user", expectedHeader, retArray, "tempuser", "temppw"); + } } diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/auth/IoTDBRelationalAuthIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/auth/IoTDBRelationalAuthIT.java index dfad35cb6963..2454ec95c7b3 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/auth/IoTDBRelationalAuthIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/auth/IoTDBRelationalAuthIT.java @@ -106,6 +106,12 @@ public void listUserPrivileges() throws SQLException { ",testdb.tb,INSERT,true,", ",testdb.tb,DROP,true,")); TestUtils.assertResultSetEqual(rs, "Role,Scope,Privileges,GrantOption,", ans); + adminStmt.execute("create role testrole"); + adminStmt.execute("GRANT ROLE testrole to testuser"); + rs = adminStmt.executeQuery("LIST USER OF ROLE testrole"); + TestUtils.assertResultSetEqual(rs, "User,", Collections.singleton("testuser,")); + rs = adminStmt.executeQuery("LIST ROLE OF USER testuser"); + TestUtils.assertResultSetEqual(rs, "Role,", Collections.singleton("testrole,")); } } @@ -287,6 +293,168 @@ public void checkAuthorStatementPrivilegeCheck() throws SQLException { @Test public void checkGrantRevokeAllPrivileges() throws SQLException { + // In this IT: + // grant + // 1. grant all on table tb1 with grant option + // 2. grant all on database testdb + // 3. grant all on any + // revoke + // 1. revoke grant option for all on table tb1 + // 2. revoke all on table tb1 + // 3. revoke all on database testdb + // 4. revoke all on any + // grant and revoke + // 1. grant all on user/role + // 2. revoke all on any + // 3. revoke all on user/role + + for (boolean isUser : new boolean[] {true, false}) { + try (Connection adminCon = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement adminStmt = adminCon.createStatement()) { + adminStmt.execute("create database testdb"); + adminStmt.execute(isUser ? "create user test 'password'" : "create role test"); + adminStmt.execute("use testdb"); + + // 1. grant all on table tb1 with grant option + adminStmt.execute( + "grant all on table tb1 to " + + (isUser ? "user test" : "role test") + + " with grant option"); + Set listPrivilegeResult = new HashSet<>(); + for (PrivilegeType privilegeType : PrivilegeType.values()) { + if (privilegeType.isRelationalPrivilege()) { + listPrivilegeResult.add( + (isUser ? "," : "test,") + "testdb.tb1," + privilegeType + ",true,"); + } + } + ResultSet resultSet = + adminStmt.executeQuery("List privileges of " + (isUser ? "user test" : "role test")); + TestUtils.assertResultSetEqual( + resultSet, "Role,Scope,Privileges,GrantOption,", listPrivilegeResult); + + // 2. grant all on database testdb + adminStmt.execute( + "grant all on database testdb to " + (isUser ? "user test" : "role test")); + for (PrivilegeType privilegeType : PrivilegeType.values()) { + if (privilegeType.isRelationalPrivilege()) { + listPrivilegeResult.add( + (isUser ? "," : "test,") + "testdb.*," + privilegeType + ",false,"); + } + } + resultSet = + adminStmt.executeQuery("List privileges of " + (isUser ? "user test" : "role test")); + TestUtils.assertResultSetEqual( + resultSet, "Role,Scope,Privileges,GrantOption,", listPrivilegeResult); + + // 3. grant all on any + adminStmt.execute("grant all on any to " + (isUser ? "user test" : "role test")); + for (PrivilegeType privilegeType : PrivilegeType.values()) { + if (privilegeType.isRelationalPrivilege()) { + listPrivilegeResult.add((isUser ? "," : "test,") + "*.*," + privilegeType + ",false,"); + } + } + resultSet = + adminStmt.executeQuery("List privileges of " + (isUser ? "user test" : "role test")); + TestUtils.assertResultSetEqual( + resultSet, "Role,Scope,Privileges,GrantOption,", listPrivilegeResult); + + // 1. revoke grant option for all on table tb1 + adminStmt.execute( + "revoke grant option for all on table tb1 from " + + (isUser ? "user test" : "role test")); + for (PrivilegeType privilegeType : PrivilegeType.values()) { + if (privilegeType.isRelationalPrivilege()) { + listPrivilegeResult.remove( + (isUser ? "," : "test,") + "testdb.tb1," + privilegeType + ",true,"); + listPrivilegeResult.add( + (isUser ? "," : "test,") + "testdb.tb1," + privilegeType + ",false,"); + } + } + resultSet = + adminStmt.executeQuery("List privileges of " + (isUser ? "user test" : "role test")); + TestUtils.assertResultSetEqual( + resultSet, "Role,Scope,Privileges,GrantOption,", listPrivilegeResult); + + // 2. revoke all on table tb1 + adminStmt.execute("revoke all on table tb1 from " + (isUser ? "user test" : "role test")); + for (PrivilegeType privilegeType : PrivilegeType.values()) { + if (privilegeType.isRelationalPrivilege()) { + listPrivilegeResult.remove( + (isUser ? "," : "test,") + "testdb.tb1," + privilegeType + ",false,"); + } + } + resultSet = + adminStmt.executeQuery("List privileges of " + (isUser ? "user test" : "role test")); + TestUtils.assertResultSetEqual( + resultSet, "Role,Scope,Privileges,GrantOption,", listPrivilegeResult); + + // 3. revoke all on database testdb + adminStmt.execute( + "revoke all on database testdb from " + (isUser ? "user test" : "role test")); + for (PrivilegeType privilegeType : PrivilegeType.values()) { + if (privilegeType.isRelationalPrivilege()) { + listPrivilegeResult.remove( + (isUser ? "," : "test,") + "testdb.*," + privilegeType + ",false,"); + } + } + resultSet = + adminStmt.executeQuery("List privileges of " + (isUser ? "user test" : "role test")); + TestUtils.assertResultSetEqual( + resultSet, "Role,Scope,Privileges,GrantOption,", listPrivilegeResult); + + // 4. revoke all on any + adminStmt.execute("revoke all on any from " + (isUser ? "user test" : "role test")); + for (PrivilegeType privilegeType : PrivilegeType.values()) { + if (privilegeType.isRelationalPrivilege()) { + listPrivilegeResult.remove( + (isUser ? "," : "test,") + "*.*," + privilegeType + ",false,"); + } + } + Assert.assertTrue(listPrivilegeResult.isEmpty()); + resultSet = + adminStmt.executeQuery("List privileges of " + (isUser ? "user test" : "role test")); + TestUtils.assertResultSetEqual( + resultSet, "Role,Scope,Privileges,GrantOption,", listPrivilegeResult); + + // 1. grant all on user/role + adminStmt.execute("grant all to " + (isUser ? "user test" : "role test")); + for (PrivilegeType privilegeType : PrivilegeType.values()) { + if (privilegeType.isRelationalPrivilege()) { + listPrivilegeResult.add((isUser ? "," : "test,") + "*.*," + privilegeType + ",false,"); + } else if (privilegeType.forRelationalSys()) { + listPrivilegeResult.add((isUser ? "," : "test,") + "," + privilegeType + ",false,"); + } + } + resultSet = + adminStmt.executeQuery("List privileges of " + (isUser ? "user test" : "role test")); + TestUtils.assertResultSetEqual( + resultSet, "Role,Scope,Privileges,GrantOption,", listPrivilegeResult); + + // 2. revoke all on any + adminStmt.execute("revoke all on any from " + (isUser ? "user test" : "role test")); + for (PrivilegeType privilegeType : PrivilegeType.values()) { + if (privilegeType.isRelationalPrivilege()) { + listPrivilegeResult.remove( + (isUser ? "," : "test,") + "*.*," + privilegeType + ",false,"); + } + } + resultSet = + adminStmt.executeQuery("List privileges of " + (isUser ? "user test" : "role test")); + TestUtils.assertResultSetEqual( + resultSet, "Role,Scope,Privileges,GrantOption,", listPrivilegeResult); + + // 3. revoke all on user/role + adminStmt.execute("revoke all from " + (isUser ? "user test" : "role test")); + listPrivilegeResult.clear(); + resultSet = + adminStmt.executeQuery("List privileges of " + (isUser ? "user test" : "role test")); + TestUtils.assertResultSetEqual( + resultSet, "Role,Scope,Privileges,GrantOption,", listPrivilegeResult); + adminStmt.execute("drop database testdb"); + adminStmt.execute(isUser ? "drop user test" : "drop role test"); + } + } + try (Connection adminCon = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); Statement adminStmt = adminCon.createStatement()) { adminStmt.execute("create user test 'password'"); @@ -345,6 +513,21 @@ public void checkGrantRevokeAllPrivileges() throws SQLException { ResultSet resultSet = adminStmt.executeQuery("List privileges of user test"); TestUtils.assertResultSetEqual( resultSet, "Role,Scope,Privileges,GrantOption,", Collections.emptySet()); + adminStmt.execute("GRANT ALL ON db1.test TO USER test"); + adminStmt.execute("GRANT ALL ON DATABASE db2 TO USER test with grant option"); + resultSet = adminStmt.executeQuery("List privileges of user test"); + Set resultSetALL = new HashSet<>(); + for (PrivilegeType privilegeType : PrivilegeType.values()) { + if (privilegeType.isRelationalPrivilege()) { + resultSetALL.add(",db2.*," + privilegeType + ",true,"); + resultSetALL.add(",db1.test," + privilegeType + ",false,"); + } + } + TestUtils.assertResultSetEqual(resultSet, "Role,Scope,Privileges,GrantOption,", resultSetALL); + adminStmt.execute("REVOKE ALL FROM USER test"); + resultSet = adminStmt.executeQuery("List privileges of user test"); + TestUtils.assertResultSetEqual( + resultSet, "Role,Scope,Privileges,GrantOption,", Collections.emptySet()); } } } diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/auth/IoTDBSystemPermissionIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/auth/IoTDBSystemPermissionIT.java index 4b3cebd68f1b..2b04fcc64b19 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/auth/IoTDBSystemPermissionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/auth/IoTDBSystemPermissionIT.java @@ -219,7 +219,7 @@ public void maintainOperationsTest() { executeNonQuery("show queries", "test", "test123"); assertNonQueryTestFail( "kill query 'test'", - "305: Please ensure your input is correct", + "701: Please ensure your input is correct", "test", "test123"); executeNonQuery("show cluster", "test", "test123"); diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBQueryDemoIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBQueryDemoIT.java index 9fc2cb74b44b..2397e49c1dc0 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBQueryDemoIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBQueryDemoIT.java @@ -18,11 +18,13 @@ */ package org.apache.iotdb.db.it.query; +import org.apache.iotdb.isession.ISession; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.ClusterIT; import org.apache.iotdb.itbase.category.LocalStandaloneIT; +import com.google.common.collect.ImmutableList; import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; @@ -41,6 +43,7 @@ import java.util.List; import java.util.Map; +import static org.apache.iotdb.db.it.utils.TestUtils.assertTestFail; import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) @@ -668,4 +671,26 @@ public void RegexpNonExistTest() { fail(e.getMessage()); } } + + @Test + public void selectWithTimeTest() { + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + session.executeRawDataQuery( + ImmutableList.of("root.ln.wf01.wt01.time", "root.ln.wf01.wt01.temperature"), 0, 100); + + fail(); + } catch (Exception e) { + e.getMessage().contains("509: root.ln.wf01.wt01.time is not a legal path"); + } + + String expectedErrMsg = + "701: Time column is no need to appear in SELECT Clause explicitly, it will always be returned if possible"; + assertTestFail("select time from root.ln.wf01.wt01", expectedErrMsg); + assertTestFail("select time, temperature from root.ln.wf01.wt01", expectedErrMsg); + assertTestFail("select time from root.ln.wf01.wt01 where temperature > 1", expectedErrMsg); + // parse error when process 'wt01.time' + assertTestFail( + "select wt01.time, wt01.temperature from root.ln.wf01", + "700: Error occurred while parsing SQL to physical plan"); + } } diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/utils/TestUtils.java b/integration-test/src/test/java/org/apache/iotdb/db/it/utils/TestUtils.java index 5cd9c63d5272..9e407bba7d54 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/utils/TestUtils.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/utils/TestUtils.java @@ -268,6 +268,75 @@ public static void tableResultSetEqualTest( } } + public static void tableExecuteTest(String sql, String userName, String password) { + try (Connection connection = + EnvFactory.getEnv().getConnection(userName, password, BaseEnv.TABLE_SQL_DIALECT)) { + connection.setClientInfo("time_zone", "+00:00"); + try (Statement statement = connection.createStatement()) { + statement.execute(sql); + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + public static void tableQueryNoVerifyResultTest( + String sql, String[] expectedHeader, String userName, String password) { + try (Connection connection = + EnvFactory.getEnv().getConnection(userName, password, BaseEnv.TABLE_SQL_DIALECT)) { + connection.setClientInfo("time_zone", "+00:00"); + try (Statement statement = connection.createStatement()) { + try (ResultSet resultSet = statement.executeQuery(sql)) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + assertEquals(expectedHeader[i - 1], resultSetMetaData.getColumnName(i)); + } + assertEquals(expectedHeader.length, resultSetMetaData.getColumnCount()); + } + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + public static void tableResultSetEqualTest( + String sql, + String[] expectedHeader, + String[] expectedRetArray, + String userName, + String password) { + try (Connection connection = + EnvFactory.getEnv().getConnection(userName, password, BaseEnv.TABLE_SQL_DIALECT)) { + connection.setClientInfo("time_zone", "+00:00"); + try (Statement statement = connection.createStatement()) { + try (ResultSet resultSet = statement.executeQuery(sql)) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + assertEquals(expectedHeader[i - 1], resultSetMetaData.getColumnName(i)); + } + assertEquals(expectedHeader.length, resultSetMetaData.getColumnCount()); + + int cnt = 0; + while (resultSet.next()) { + StringBuilder builder = new StringBuilder(); + for (int i = 1; i <= expectedHeader.length; i++) { + builder.append(resultSet.getString(i)).append(","); + } + assertEquals(expectedRetArray[cnt], builder.toString()); + // System.out.println(String.format("\"%s\",", builder.toString())); + cnt++; + } + assertEquals(expectedRetArray.length, cnt); + } + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + public static void tableResultSetFuzzyTest( String sql, String[] expectedHeader, int expectedCount, String database) { tableResultSetFuzzyTest( @@ -329,6 +398,18 @@ public static void tableAssertTestFail( } } + public static void tableAssertTestFail( + String sql, String errMsg, String userName, String password) { + try (Connection connection = + EnvFactory.getEnv().getConnection(userName, password, BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.executeQuery(sql); + fail("No exception!"); + } catch (SQLException e) { + Assert.assertTrue(e.getMessage(), e.getMessage().contains(errMsg)); + } + } + public static void resultSetEqualTest( String sql, String[] expectedHeader, @@ -455,6 +536,26 @@ public static void assertTestFail( } } + public static void assertTableTestFail( + final BaseEnv env, + final String sql, + final String errMsg, + final String userName, + final String password, + final String db) { + try (final Connection connection = + env.getConnection(userName, password, BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + if (Objects.nonNull(db)) { + statement.execute("use " + "\"" + db + "\""); + } + statement.executeQuery(sql); + fail("No exception!"); + } catch (final SQLException e) { + Assert.assertTrue(e.getMessage(), e.getMessage().contains(errMsg)); + } + } + public static void assertNonQueryTestFail(String sql, String errMsg) { assertNonQueryTestFail(sql, errMsg, SessionConfig.DEFAULT_USER, SessionConfig.DEFAULT_PASSWORD); } @@ -489,7 +590,9 @@ public static void assertTableNonQueryTestFail( BaseEnv env, String sql, String errMsg, String userName, String password, String db) { try (Connection connection = env.getConnection(userName, password, BaseEnv.TABLE_SQL_DIALECT); Statement statement = connection.createStatement()) { - statement.execute("use " + "\"" + db + "\""); + if (Objects.nonNull(db)) { + statement.execute("use " + "\"" + db + "\""); + } statement.execute(sql); fail("No exception!"); } catch (SQLException e) { @@ -646,13 +749,57 @@ public static void executeNonQuery(String sql, String userName, String password) } } - public static void executeNonQueryWithRetry(BaseEnv env, String sql) { + public static void executeNonQueryWithRetry(final BaseEnv env, final String sql) { + executeNonQueryWithRetry(env, sql, SessionConfig.DEFAULT_USER, SessionConfig.DEFAULT_PASSWORD); + } + + public static void executeNonQueryWithRetry( + final BaseEnv env, final String sql, final String userName, final String password) { + executeNonQueryWithRetry(env, sql, userName, password, null, TREE_SQL_DIALECT); + } + + public static void executeNonQueryWithRetry( + final BaseEnv env, + final String sql, + final String userName, + final String password, + final String database, + final String sqlDialect) { + executeNonQueriesWithRetry( + env, Collections.singletonList(sql), userName, password, database, sqlDialect); + } + + public static void executeNonQueriesWithRetry( + final BaseEnv env, final List sqlList, final String userName, final String password) { + executeNonQueriesWithRetry(env, sqlList, userName, password, "", TREE_SQL_DIALECT); + } + + public static void executeNonQueriesWithRetry( + final BaseEnv env, + final List sqlList, + final String userName, + final String password, + final String database, + final String sqlDialect) { + int lastIndex = 0; for (int retryCountLeft = 10; retryCountLeft >= 0; retryCountLeft--) { - try (Connection connection = env.getConnection(); - Statement statement = connection.createStatement()) { - statement.execute(sql); - break; - } catch (SQLException e) { + try (final Connection connection = + env.getConnection( + userName, + password, + BaseEnv.TABLE_SQL_DIALECT.equals(sqlDialect) + ? BaseEnv.TABLE_SQL_DIALECT + : TREE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + if (BaseEnv.TABLE_SQL_DIALECT.equals(sqlDialect) && database != null) { + statement.execute("use " + database); + } + for (int i = lastIndex; i < sqlList.size(); ++i) { + lastIndex = i; + statement.execute(sqlList.get(i)); + } + return; + } catch (final SQLException e) { if (retryCountLeft > 0) { try { Thread.sleep(10000); @@ -872,27 +1019,57 @@ public static void executeQuery(String sql, String userName, String password) { } public static void executeQueryWithRetry( - BaseEnv env, String sql, String userName, String password) { - try (Connection connection = env.getConnection(userName, password); - Statement statement = connection.createStatement()) { - for (int retryCountLeft = 10; retryCountLeft >= 0; retryCountLeft--) { - try { - statement.executeQuery(sql); - } catch (SQLException e) { - if (retryCountLeft > 0) { - try { - Thread.sleep(10000); - } catch (InterruptedException ignored) { - } - } else { - e.printStackTrace(); - fail(e.getMessage()); + final BaseEnv env, final String sql, final String userName, final String password) { + executeQueryWithRetry(env, sql, userName, password, null, TREE_SQL_DIALECT); + } + + public static void executeQueryWithRetry( + final BaseEnv env, + final String sql, + final String userName, + final String password, + final String database, + final String sqlDialect) { + executeQueriesWithRetry( + env, Collections.singletonList(sql), userName, password, database, sqlDialect); + } + + public static void executeQueriesWithRetry( + final BaseEnv env, + final List sqlList, + final String userName, + final String password, + final String database, + final String sqlDialect) { + int lastIndex = 0; + for (int retryCountLeft = 10; retryCountLeft >= 0; retryCountLeft--) { + try (final Connection connection = + env.getConnection( + userName, + password, + BaseEnv.TABLE_SQL_DIALECT.equals(sqlDialect) + ? BaseEnv.TABLE_SQL_DIALECT + : TREE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + if (BaseEnv.TABLE_SQL_DIALECT.equals(sqlDialect) && database != null) { + statement.execute("use " + database); + } + for (int i = lastIndex; i < sqlList.size(); ++i) { + lastIndex = i; + statement.executeQuery(sqlList.get(i)); + } + return; + } catch (final SQLException e) { + if (retryCountLeft > 0) { + try { + Thread.sleep(10000); + } catch (InterruptedException ignored) { } + } else { + e.printStackTrace(); + fail(e.getMessage()); } } - } catch (SQLException e) { - e.printStackTrace(); - fail(e.getMessage()); } } diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/TableModelUtils.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/TableModelUtils.java similarity index 99% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/TableModelUtils.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/TableModelUtils.java index b116cad502c2..0410a50a9775 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/TableModelUtils.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/TableModelUtils.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.tablemodel; +package org.apache.iotdb.pipe.it.dual.tablemodel; import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.isession.ITableSession; diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/AbstractPipeTableModelTestIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/AbstractPipeTableModelDualManualIT.java similarity index 95% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/AbstractPipeTableModelTestIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/AbstractPipeTableModelDualManualIT.java index 06c7c3867f5b..fff0b8c6bf1a 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/AbstractPipeTableModelTestIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/AbstractPipeTableModelDualManualIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.tablemodel; +package org.apache.iotdb.pipe.it.dual.tablemodel.manual; import org.apache.iotdb.consensus.ConsensusFactory; import org.apache.iotdb.it.env.MultiEnvFactory; @@ -26,7 +26,7 @@ import org.junit.After; import org.junit.Before; -abstract class AbstractPipeTableModelTestIT { +public abstract class AbstractPipeTableModelDualManualIT { protected BaseEnv senderEnv; protected BaseEnv receiverEnv; diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeAlterIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeAlterIT.java similarity index 98% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeAlterIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeAlterIT.java index 1665f9d2ce2b..93bb1424c2b2 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeAlterIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeAlterIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.tablemodel; +package org.apache.iotdb.pipe.it.dual.tablemodel.manual.basic; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; import org.apache.iotdb.confignode.rpc.thrift.TShowPipeInfo; @@ -25,8 +25,10 @@ import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2TableModel; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualBasic; import org.apache.iotdb.itbase.env.BaseEnv; +import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; +import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; import org.junit.Assert; import org.junit.Test; @@ -42,8 +44,8 @@ import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2TableModel.class}) -public class IoTDBPipeAlterIT extends AbstractPipeTableModelTestIT { +@Category({MultiClusterIT2DualTableManualBasic.class}) +public class IoTDBPipeAlterIT extends AbstractPipeTableModelDualManualIT { @Test public void testBasicAlterPipe() throws Exception { diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeDataSinkIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeDataSinkIT.java similarity index 98% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeDataSinkIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeDataSinkIT.java index 2867a3402503..7c98f652f73f 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeDataSinkIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeDataSinkIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.tablemodel; +package org.apache.iotdb.pipe.it.dual.tablemodel.manual.basic; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; @@ -25,7 +25,9 @@ import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2TableModel; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualBasic; +import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; +import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; import org.apache.iotdb.rpc.TSStatusCode; import org.apache.tsfile.write.record.Tablet; @@ -47,8 +49,8 @@ import java.util.function.Consumer; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2TableModel.class}) -public class IoTDBPipeDataSinkIT extends AbstractPipeTableModelTestIT { +@Category({MultiClusterIT2DualTableManualBasic.class}) +public class IoTDBPipeDataSinkIT extends AbstractPipeTableModelDualManualIT { @Test public void testThriftConnectorWithRealtimeFirstDisabled() throws Exception { @@ -190,7 +192,7 @@ private void testSinkFormat(final String format) throws Exception { return; } - TableModelUtils.assertCountData("test", "test", 150, receiverEnv); + TableModelUtils.assertCountData("test", "test", 150, receiverEnv, handleFailure); TestUtils.assertDataEventuallyOnEnv( receiverEnv, @@ -235,7 +237,7 @@ private void testSinkFormat(final String format) throws Exception { new HashSet<>(Arrays.asList("0,1.0,", "1,1.0,", "2,1.0,", "3,1.0,", "4,1.0,"))), handleFailure); - TableModelUtils.assertCountData("test", "test", 350, receiverEnv); + TableModelUtils.assertCountData("test", "test", 350, receiverEnv, handleFailure); } } diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeExtractorIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeExtractorIT.java similarity index 99% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeExtractorIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeExtractorIT.java index 41d6a8434d1e..d432f4c54480 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeExtractorIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeExtractorIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.tablemodel; +package org.apache.iotdb.pipe.it.dual.tablemodel.manual.basic; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; @@ -27,8 +27,10 @@ import org.apache.iotdb.it.env.MultiEnvFactory; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2TableModel; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualBasic; import org.apache.iotdb.itbase.env.BaseEnv; +import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; +import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -50,8 +52,8 @@ import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2TableModel.class}) -public class IoTDBPipeExtractorIT extends AbstractPipeTableModelTestIT { +@Category({MultiClusterIT2DualTableManualBasic.class}) +public class IoTDBPipeExtractorIT extends AbstractPipeTableModelDualManualIT { @Before public void setUp() { diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeIsolationIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeIsolationIT.java similarity index 97% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeIsolationIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeIsolationIT.java index 57a3be38e217..b0bdb5dd6737 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeIsolationIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeIsolationIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.tablemodel; +package org.apache.iotdb.pipe.it.dual.tablemodel.manual.basic; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; import org.apache.iotdb.confignode.rpc.thrift.TAlterPipeReq; @@ -26,8 +26,10 @@ import org.apache.iotdb.confignode.rpc.thrift.TStopPipeReq; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2TableModel; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualBasic; import org.apache.iotdb.itbase.env.BaseEnv; +import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; +import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -43,8 +45,8 @@ import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2TableModel.class}) -public class IoTDBPipeIsolationIT extends AbstractPipeTableModelTestIT { +@Category({MultiClusterIT2DualTableManualBasic.class}) +public class IoTDBPipeIsolationIT extends AbstractPipeTableModelDualManualIT { @Test public void testWritePipeIsolation() throws Exception { diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeLifeCycleIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeLifeCycleIT.java similarity index 83% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeLifeCycleIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeLifeCycleIT.java index 88cfc1bffd77..ff03b5dbc694 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeLifeCycleIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeLifeCycleIT.java @@ -17,15 +17,19 @@ * under the License. */ -package org.apache.iotdb.pipe.it.tablemodel; +package org.apache.iotdb.pipe.it.dual.tablemodel.manual.basic; import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.auth.entity.PrivilegeType; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2TableModel; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualBasic; +import org.apache.iotdb.itbase.env.BaseEnv; +import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; +import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -34,15 +38,24 @@ import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; +import static org.apache.iotdb.db.it.utils.TestUtils.assertTableNonQueryTestFail; +import static org.apache.iotdb.db.it.utils.TestUtils.assertTableTestFail; +import static org.apache.iotdb.db.it.utils.TestUtils.createUser; +import static org.apache.iotdb.db.it.utils.TestUtils.executeNonQueriesWithRetry; +import static org.apache.iotdb.db.it.utils.TestUtils.executeNonQueryWithRetry; +import static org.apache.iotdb.db.it.utils.TestUtils.executeQueryWithRetry; +import static org.apache.iotdb.db.it.utils.TestUtils.grantUserSystemPrivileges; + @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2TableModel.class}) -public class IoTDBPipeLifeCycleIT extends AbstractPipeTableModelTestIT { +@Category({MultiClusterIT2DualTableManualBasic.class}) +public class IoTDBPipeLifeCycleIT extends AbstractPipeTableModelDualManualIT { @Test public void testLifeCycleWithHistoryEnabled() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); @@ -51,8 +64,8 @@ public void testLifeCycleWithHistoryEnabled() throws Exception { final int receiverPort = receiverDataNode.getPort(); final Consumer handleFailure = o -> { - TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); - TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); + executeNonQueryWithRetry(senderEnv, "flush"); + executeNonQueryWithRetry(receiverEnv, "flush"); }; boolean insertResult = true; @@ -121,8 +134,8 @@ public void testLifeCycleWithHistoryDisabled() throws Exception { final int receiverPort = receiverDataNode.getPort(); final Consumer handleFailure = o -> { - TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); - TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); + executeNonQueryWithRetry(senderEnv, "flush"); + executeNonQueryWithRetry(receiverEnv, "flush"); }; boolean insertResult = true; @@ -195,8 +208,8 @@ public void testLifeCycleLogMode() throws Exception { final int receiverPort = receiverDataNode.getPort(); final Consumer handleFailure = o -> { - TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); - TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); + executeNonQueryWithRetry(senderEnv, "flush"); + executeNonQueryWithRetry(receiverEnv, "flush"); }; boolean insertResult = true; @@ -257,8 +270,8 @@ public void testLifeCycleFileMode() throws Exception { final int receiverPort = receiverDataNode.getPort(); final Consumer handleFailure = o -> { - TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); - TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); + executeNonQueryWithRetry(senderEnv, "flush"); + executeNonQueryWithRetry(receiverEnv, "flush"); }; boolean insertResult = true; @@ -312,8 +325,8 @@ public void testLifeCycleHybridMode() throws Exception { final int receiverPort = receiverDataNode.getPort(); final Consumer handleFailure = o -> { - TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); - TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); + executeNonQueryWithRetry(senderEnv, "flush"); + executeNonQueryWithRetry(receiverEnv, "flush"); }; boolean insertResult = true; @@ -371,8 +384,8 @@ public void testLifeCycleWithClusterRestart() throws Exception { final int receiverPort = receiverDataNode.getPort(); final Consumer handleFailure = o -> { - TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); - TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); + executeNonQueryWithRetry(senderEnv, "flush"); + executeNonQueryWithRetry(receiverEnv, "flush"); }; boolean insertResult = true; @@ -440,8 +453,8 @@ public void testReceiverRestartWhenTransferring() throws Exception { final int receiverPort = receiverDataNode.getPort(); final Consumer handleFailure = o -> { - TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); - TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); + executeNonQueryWithRetry(senderEnv, "flush"); + executeNonQueryWithRetry(receiverEnv, "flush"); }; boolean insertResult = true; @@ -523,8 +536,8 @@ public void testReceiverAlreadyHaveTimeSeries() throws Exception { final int receiverPort = receiverDataNode.getPort(); final Consumer handleFailure = o -> { - TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); - TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); + executeNonQueryWithRetry(senderEnv, "flush"); + executeNonQueryWithRetry(receiverEnv, "flush"); }; boolean insertResult = true; @@ -583,8 +596,8 @@ public void testDoubleLiving() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); final Consumer handleFailure = o -> { - TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); - TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); + executeNonQueryWithRetry(receiverEnv, "flush"); + executeNonQueryWithRetry(senderEnv, "flush"); }; final String senderIp = senderDataNode.getIp(); @@ -691,4 +704,104 @@ public void testDoubleLiving() throws Exception { TableModelUtils.assertData("test", "test", 0, 500, receiverEnv, handleFailure); } + + @Test + public void testPermission() { + createUser(senderEnv, "test", "test123"); + + assertTableNonQueryTestFail( + senderEnv, + "create pipe testPipe\n" + + "with connector (\n" + + " 'connector'='iotdb-thrift-connector',\n" + + " 'connector.ip'='127.0.0.1',\n" + + " 'connector.port'='6668'\n" + + ")", + "803: Access Denied: No permissions for this operation, please add privilege MAINTAIN", + "test", + "test123", + null); + assertTableNonQueryTestFail( + senderEnv, + "drop pipe testPipe", + "803: Access Denied: No permissions for this operation, please add privilege MAINTAIN", + "test", + "test123", + null); + assertTableTestFail( + senderEnv, + "show pipes", + "803: Access Denied: No permissions for this operation, please add privilege MAINTAIN", + "test", + "test123", + null); + assertTableNonQueryTestFail( + senderEnv, + "start pipe testPipe", + "803: Access Denied: No permissions for this operation, please add privilege MAINTAIN", + "test", + "test123", + null); + assertTableNonQueryTestFail( + senderEnv, + "stop pipe testPipe", + "803: Access Denied: No permissions for this operation, please add privilege MAINTAIN", + "test", + "test123", + null); + + assertTableNonQueryTestFail( + senderEnv, + "create pipePlugin TestProcessor as 'org.apache.iotdb.db.pipe.example.TestProcessor' USING URI 'xxx'", + "803: Access Denied: No permissions for this operation, please add privilege MAINTAIN", + "test", + "test123", + null); + assertTableNonQueryTestFail( + senderEnv, + "drop pipePlugin TestProcessor", + "803: Access Denied: No permissions for this operation, please add privilege MAINTAIN", + "test", + "test123", + null); + assertTableTestFail( + senderEnv, + "show pipe plugins", + "803: Access Denied: No permissions for this operation, please add privilege MAINTAIN", + "test", + "test123", + null); + + grantUserSystemPrivileges(senderEnv, "test", PrivilegeType.MAINTAIN); + + executeNonQueryWithRetry( + senderEnv, + "create pipe testPipe\n" + + "with connector (\n" + + " 'connector'='write-back-connector'\n" + + ")", + "test", + "test123", + null, + BaseEnv.TABLE_SQL_DIALECT); + executeQueryWithRetry( + senderEnv, "show pipes", "test", "test123", null, BaseEnv.TABLE_SQL_DIALECT); + executeNonQueriesWithRetry( + senderEnv, + Arrays.asList("start pipe testPipe", "stop pipe testPipe", "drop pipe testPipe"), + "test", + "test123", + null, + BaseEnv.TABLE_SQL_DIALECT); + + assertTableNonQueryTestFail( + senderEnv, + "create pipePlugin TestProcessor as 'org.apache.iotdb.db.pipe.example.TestProcessor' USING URI 'xxx'", + "701: Untrusted uri xxx", + "test", + "test123", + null); + executeQueryWithRetry( + senderEnv, "show pipe plugins", "test", "test123", null, BaseEnv.TABLE_SQL_DIALECT); + } } diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeProtocolIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeProtocolIT.java similarity index 97% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeProtocolIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeProtocolIT.java index 046c5f68fece..3fe82b52b97c 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeProtocolIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeProtocolIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.tablemodel; +package org.apache.iotdb.pipe.it.dual.tablemodel.manual.basic; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; @@ -28,7 +28,9 @@ import org.apache.iotdb.it.env.MultiEnvFactory; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2TableModel; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualBasic; +import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; +import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -43,8 +45,8 @@ /** Test pipe's basic functionalities under multiple cluster and consensus protocol settings. */ @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2TableModel.class}) -public class IoTDBPipeProtocolIT extends AbstractPipeTableModelTestIT { +@Category({MultiClusterIT2DualTableManualBasic.class}) +public class IoTDBPipeProtocolIT extends AbstractPipeTableModelDualManualIT { @Override @Before public void setUp() { diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeSwitchStatusIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeSwitchStatusIT.java similarity index 97% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeSwitchStatusIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeSwitchStatusIT.java index 4c80f0ae6731..b6b29deaac2e 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeSwitchStatusIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeSwitchStatusIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.tablemodel; +package org.apache.iotdb.pipe.it.dual.tablemodel.manual.basic; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; @@ -27,7 +27,9 @@ import org.apache.iotdb.confignode.rpc.thrift.TShowPipeReq; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2TableModel; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualBasic; +import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; +import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -40,8 +42,8 @@ import java.util.Map; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2TableModel.class}) -public class IoTDBPipeSwitchStatusIT extends AbstractPipeTableModelTestIT { +@Category({MultiClusterIT2DualTableManualBasic.class}) +public class IoTDBPipeSwitchStatusIT extends AbstractPipeTableModelDualManualIT { @Test public void testPipeSwitchStatus() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeSyntaxIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeSyntaxIT.java similarity index 98% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeSyntaxIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeSyntaxIT.java index 96673a93a4a9..bc5233f1dd8c 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeSyntaxIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeSyntaxIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.tablemodel; +package org.apache.iotdb.pipe.it.dual.tablemodel.manual.basic; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; @@ -26,8 +26,9 @@ import org.apache.iotdb.confignode.rpc.thrift.TShowPipeReq; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2TableModel; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualBasic; import org.apache.iotdb.itbase.env.BaseEnv; +import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -46,8 +47,8 @@ import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2TableModel.class}) -public class IoTDBPipeSyntaxIT extends AbstractPipeTableModelTestIT { +@Category({MultiClusterIT2DualTableManualBasic.class}) +public class IoTDBPipeSyntaxIT extends AbstractPipeTableModelDualManualIT { @Test public void testValidPipeName() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeWithLoadIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeWithLoadIT.java similarity index 97% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeWithLoadIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeWithLoadIT.java index d4f689187bfb..72b87d3cc288 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeWithLoadIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeWithLoadIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.tablemodel; +package org.apache.iotdb.pipe.it.dual.tablemodel.manual.basic; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; @@ -27,8 +27,10 @@ import org.apache.iotdb.it.env.MultiEnvFactory; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2TableModel; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualBasic; import org.apache.iotdb.itbase.env.BaseEnv; +import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; +import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -48,8 +50,8 @@ import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2TableModel.class}) -public class IoTDBPipeWithLoadIT extends AbstractPipeTableModelTestIT { +@Category({MultiClusterIT2DualTableManualBasic.class}) +public class IoTDBPipeWithLoadIT extends AbstractPipeTableModelDualManualIT { @Before public void setUp() { diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBTablePatternFormatIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBTablePatternFormatIT.java similarity index 98% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBTablePatternFormatIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBTablePatternFormatIT.java index ac1e96ab4aa9..b3bd7cd85861 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBTablePatternFormatIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBTablePatternFormatIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.tablemodel; +package org.apache.iotdb.pipe.it.dual.tablemodel.manual.basic; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; @@ -25,7 +25,9 @@ import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2TableModel; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualBasic; +import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; +import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -40,8 +42,8 @@ import java.util.function.Consumer; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2TableModel.class}) -public class IoTDBTablePatternFormatIT extends AbstractPipeTableModelTestIT { +@Category({MultiClusterIT2DualTableManualBasic.class}) +public class IoTDBTablePatternFormatIT extends AbstractPipeTableModelDualManualIT { @Test public void testTableNamePattern() throws Exception { diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeAutoConflictIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeAutoConflictIT.java similarity index 96% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeAutoConflictIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeAutoConflictIT.java index 525be694815a..fc29e3e12a74 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeAutoConflictIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeAutoConflictIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.tablemodel; +package org.apache.iotdb.pipe.it.dual.tablemodel.manual.enhanced; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; @@ -27,8 +27,10 @@ import org.apache.iotdb.it.env.MultiEnvFactory; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2TableModel; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualEnhanced; import org.apache.iotdb.itbase.env.BaseEnv; +import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; +import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -44,8 +46,8 @@ import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2TableModel.class}) -public class IoTDBPipeAutoConflictIT extends AbstractPipeTableModelTestIT { +@Category({MultiClusterIT2DualTableManualEnhanced.class}) +public class IoTDBPipeAutoConflictIT extends AbstractPipeTableModelDualManualIT { @Before public void setUp() { MultiEnvFactory.createEnv(2); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeAutoDropIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeAutoDropIT.java similarity index 94% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeAutoDropIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeAutoDropIT.java index 8f19b7f32a7f..cbf2c3073978 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeAutoDropIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeAutoDropIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.tablemodel; +package org.apache.iotdb.pipe.it.dual.tablemodel.manual.enhanced; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; @@ -27,7 +27,9 @@ import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2TableModel; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualEnhanced; +import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; +import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -48,8 +50,8 @@ import static org.awaitility.Awaitility.await; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2TableModel.class}) -public class IoTDBPipeAutoDropIT extends AbstractPipeTableModelTestIT { +@Category({MultiClusterIT2DualTableManualEnhanced.class}) +public class IoTDBPipeAutoDropIT extends AbstractPipeTableModelDualManualIT { @Test public void testAutoDropInHistoricalTransfer() throws Exception { diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeClusterIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeClusterIT.java similarity index 98% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeClusterIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeClusterIT.java index cab7f31a7568..44fae78b362b 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeClusterIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeClusterIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.tablemodel; +package org.apache.iotdb.pipe.it.dual.tablemodel.manual.enhanced; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.exception.ClientManagerException; @@ -34,7 +34,9 @@ import org.apache.iotdb.it.env.cluster.env.AbstractEnv; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2TableModel; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualEnhanced; +import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; +import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; import org.apache.iotdb.rpc.TSStatusCode; import org.apache.thrift.TException; @@ -56,8 +58,8 @@ import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2TableModel.class}) -public class IoTDBPipeClusterIT extends AbstractPipeTableModelTestIT { +@Category({MultiClusterIT2DualTableManualEnhanced.class}) +public class IoTDBPipeClusterIT extends AbstractPipeTableModelDualManualIT { @Override @Before diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeConnectorCompressionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeConnectorCompressionIT.java similarity index 97% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeConnectorCompressionIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeConnectorCompressionIT.java index 3b462d0d6ae6..7bdbc0c2fca7 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeConnectorCompressionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeConnectorCompressionIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.tablemodel; +package org.apache.iotdb.pipe.it.dual.tablemodel.manual.enhanced; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; @@ -30,7 +30,9 @@ import org.apache.iotdb.it.env.MultiEnvFactory; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2TableModel; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualEnhanced; +import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; +import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -52,8 +54,8 @@ import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2TableModel.class}) -public class IoTDBPipeConnectorCompressionIT extends AbstractPipeTableModelTestIT { +@Category({MultiClusterIT2DualTableManualEnhanced.class}) +public class IoTDBPipeConnectorCompressionIT extends AbstractPipeTableModelDualManualIT { @Override @Before diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeConnectorParallelIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeConnectorParallelIT.java similarity index 91% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeConnectorParallelIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeConnectorParallelIT.java index 663d40c5b07c..7c2c56a0fd73 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeConnectorParallelIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeConnectorParallelIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.tablemodel; +package org.apache.iotdb.pipe.it.dual.tablemodel.manual.enhanced; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; @@ -25,7 +25,9 @@ import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2TableModel; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualEnhanced; +import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; +import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -41,8 +43,8 @@ import java.util.function.Consumer; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2TableModel.class}) -public class IoTDBPipeConnectorParallelIT extends AbstractPipeTableModelTestIT { +@Category({MultiClusterIT2DualTableManualEnhanced.class}) +public class IoTDBPipeConnectorParallelIT extends AbstractPipeTableModelDualManualIT { @Test public void testIoTConnectorParallel() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeDoubleLivingIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeDoubleLivingIT.java similarity index 96% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeDoubleLivingIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeDoubleLivingIT.java index f6573728cf29..55c2aae1290e 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeDoubleLivingIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeDoubleLivingIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.tablemodel; +package org.apache.iotdb.pipe.it.dual.tablemodel.manual.enhanced; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; import org.apache.iotdb.confignode.rpc.thrift.TDropPipeReq; @@ -26,8 +26,10 @@ import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2TableModel; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualEnhanced; import org.apache.iotdb.itbase.env.BaseEnv; +import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; +import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -46,8 +48,8 @@ import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2TableModel.class}) -public class IoTDBPipeDoubleLivingIT extends AbstractPipeTableModelTestIT { +@Category({MultiClusterIT2DualTableManualEnhanced.class}) +public class IoTDBPipeDoubleLivingIT extends AbstractPipeTableModelDualManualIT { @Test public void testDoubleLivingInvalidParameter() throws Exception { @@ -106,7 +108,8 @@ public void testDoubleLivingInvalidParameter() throws Exception { } } - // combination of org.apache.iotdb.pipe.it.tablemodel.IoTDBPipeLifeCycleIT.testDoubleLiving and + // combination of + // org.apache.iotdb.pipe.it.tablemodel.autocreate.IoTDBPipeLifeCycleIT.testDoubleLiving and // org.apache.iotdb.pipe.it.autocreate.IoTDBPipeLifeCycleIT.testDoubleLiving @Test public void testBasicDoubleLiving() { diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeIdempotentIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeIdempotentIT.java new file mode 100644 index 000000000000..c80a67c4d4f8 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeIdempotentIT.java @@ -0,0 +1,197 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.pipe.it.dual.tablemodel.manual.enhanced; + +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; +import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualEnhanced; +import org.apache.iotdb.itbase.env.BaseEnv; +import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; +import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2DualTableManualEnhanced.class}) +public class IoTDBPipeIdempotentIT extends AbstractPipeTableModelDualManualIT { + @Test + public void testCreateTableIdempotent() throws Exception { + testTableConfigIdempotent(Collections.emptyList(), "create table test()"); + } + + @Test + public void testAlterTableAddColumnIdempotent() throws Exception { + testTableConfigIdempotent( + Collections.singletonList("create table test()"), "alter table test add column a id"); + } + + @Test + public void testAlterTableSetPropertiesIdempotent() throws Exception { + testTableConfigIdempotent( + Collections.singletonList("create table test()"), + "alter table test set properties ttl=100"); + } + + @Test + public void testAlterTableDropColumnIdempotent() throws Exception { + testTableConfigIdempotent( + Collections.singletonList("create table test(a id, b attribute, c int32)"), + "alter table test drop column b"); + } + + @Test + public void testDropTableIdempotent() throws Exception { + testTableConfigIdempotent(Collections.singletonList("create table test()"), "drop table test"); + } + + @Test + public void testTableCreateUserIdempotent() throws Exception { + testTableConfigIdempotent(Collections.emptyList(), "create user newUser 'password'"); + } + + @Test + public void testTableDropUserIdempotent() throws Exception { + testTableConfigIdempotent( + Collections.singletonList("create user newUser 'password'"), "drop user newUser"); + } + + @Test + public void testTableCreateRoleIdempotent() throws Exception { + testTableConfigIdempotent(Collections.emptyList(), "create role newRole"); + } + + @Test + public void testTableDropRoleIdempotent() throws Exception { + testTableConfigIdempotent( + Collections.singletonList("create role newRole"), "drop role newRole"); + } + + @Test + public void testTableAlterUserIdempotent3() throws Exception { + testTableConfigIdempotent( + Collections.singletonList("create user newUser 'password'"), + "alter user newUser set password 'passwd'"); + } + + @Test + public void testTableGrantRoleToUserIdempotent() throws Exception { + testTableConfigIdempotent( + Arrays.asList("create user newUser 'password'", "create role newRole"), + "grant role newRole to newUser"); + } + + @Test + public void testTableRevokeRoleFromUserIdempotent() throws Exception { + testTableConfigIdempotent( + Arrays.asList( + "create user newUser 'password'", + "create role newRole", + "grant role newRole to newUser"), + "revoke role newRole from newUser"); + } + + @Test + public void testTableGrantIdempotent() throws Exception { + testTableConfigIdempotent( + Collections.singletonList("create user newUser 'password'"), + "grant all to user newUser with grant option"); + } + + @Test + public void testTableRevokeIdempotent() throws Exception { + testTableConfigIdempotent( + Arrays.asList( + "create user newUser 'password'", "grant all to user newUser with grant option"), + "revoke all from user newUser"); + } + + private void testTableConfigIdempotent(final List beforeSqlList, final String testSql) + throws Exception { + final String database = "test"; + TableModelUtils.createDatabase(senderEnv, database); + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("extractor.inclusion", "all"); + extractorAttributes.put("extractor.inclusion.exclusion", ""); + extractorAttributes.put("extractor.forwarding-pipe-requests", "false"); + extractorAttributes.put("extractor.capture.table", "true"); + extractorAttributes.put("extractor.capture.tree", "false"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.exception.conflict.resolve-strategy", "retry"); + connectorAttributes.put("connector.exception.conflict.retry-max-time-seconds", "-1"); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("testPipe", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + } + + if (!TestUtils.tryExecuteNonQueriesWithRetry( + database, BaseEnv.TABLE_SQL_DIALECT, senderEnv, beforeSqlList)) { + return; + } + + if (!TestUtils.tryExecuteNonQueryWithRetry( + database, BaseEnv.TABLE_SQL_DIALECT, receiverEnv, testSql)) { + return; + } + + // Create an idempotent conflict + if (!TestUtils.tryExecuteNonQueryWithRetry( + database, BaseEnv.TABLE_SQL_DIALECT, senderEnv, testSql)) { + return; + } + + TableModelUtils.createDatabase(senderEnv, "test2"); + + // Assume that the "database" is executed on receiverEnv + TestUtils.assertDataSizeEventuallyOnEnv(receiverEnv, "show databases", 3, null); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeTableManualIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeMetaIT.java similarity index 77% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeTableManualIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeMetaIT.java index f375bd35e81b..4e33f8369d62 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeTableManualIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeMetaIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.manual; +package org.apache.iotdb.pipe.it.dual.tablemodel.manual.enhanced; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; @@ -25,9 +25,10 @@ import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2ManualCreateSchema; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualEnhanced; import org.apache.iotdb.itbase.env.BaseEnv; -import org.apache.iotdb.pipe.it.tablemodel.TableModelUtils; +import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; +import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -42,8 +43,8 @@ import java.util.Map; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2ManualCreateSchema.class}) -public class IoTDBPipeTableManualIT extends AbstractPipeDualManualIT { +@Category({MultiClusterIT2DualTableManualEnhanced.class}) +public class IoTDBPipeMetaIT extends AbstractPipeTableModelDualManualIT { @Test public void testTableSync() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); @@ -290,4 +291,80 @@ public void testNoTable() throws Exception { dbName); } } + + @Test + public void testAuth() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + + if (!TestUtils.tryExecuteNonQueriesWithRetry( + senderEnv, + Arrays.asList( + "create user testUser 'password'", "grant all on root.** to user testUser"))) { + return; + } + + final String dbName = "test"; + if (!TestUtils.tryExecuteNonQueriesWithRetry( + dbName, + BaseEnv.TABLE_SQL_DIALECT, + senderEnv, + Arrays.asList( + "grant create on db.tb to user testUser", + "grant drop on database test to user testUser"))) { + return; + } + + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("extractor.inclusion", "all"); + extractorAttributes.put("extractor.capture.tree", "false"); + extractorAttributes.put("extractor.capture.table", "true"); + extractorAttributes.put("extractor.database-name", "test"); + extractorAttributes.put("extractor.table-name", "t.*[0-9]"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("testPipe", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("testPipe").getCode()); + + if (!TestUtils.tryExecuteNonQueryWithRetry( + dbName, + BaseEnv.TABLE_SQL_DIALECT, + senderEnv, + "grant alter on any to user testUser with grant option")) { + return; + } + + TestUtils.assertDataAlwaysOnEnv( + receiverEnv, + "list privileges of user testUser", + "Role,Scope,Privileges,GrantOption,", + new HashSet<>( + Arrays.asList( + ",,MANAGE_USER,false,", + ",,MANAGE_ROLE,false,", + ",,MAINTAIN,false,", + ",*.*,ALTER,true,", + ",test.*,DROP,false,")), + dbName); + } + } } diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeNullValueIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeNullValueIT.java similarity index 94% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeNullValueIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeNullValueIT.java index aae3626ee531..de59d7ef8a63 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeNullValueIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeNullValueIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.tablemodel; +package org.apache.iotdb.pipe.it.dual.tablemodel.manual.enhanced; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; @@ -25,7 +25,9 @@ import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2TableModel; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualEnhanced; +import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; +import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -38,8 +40,8 @@ import java.util.function.Consumer; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2TableModel.class}) -public class IoTDBPipeNullValueIT extends AbstractPipeTableModelTestIT { +@Category({MultiClusterIT2DualTableManualEnhanced.class}) +public class IoTDBPipeNullValueIT extends AbstractPipeTableModelDualManualIT { private enum InsertType { SESSION_INSERT_RECORD, diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeTypeConversionISessionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeTypeConversionISessionIT.java similarity index 96% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeTypeConversionISessionIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeTypeConversionISessionIT.java index dcd48928fdcc..606fba4d17e8 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeTypeConversionISessionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeTypeConversionISessionIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.tablemodel; +package org.apache.iotdb.pipe.it.dual.tablemodel.manual.enhanced; import org.apache.iotdb.commons.utils.function.CheckedTriConsumer; import org.apache.iotdb.db.it.utils.TestUtils; @@ -25,8 +25,9 @@ import org.apache.iotdb.isession.ITableSession; import org.apache.iotdb.isession.SessionDataSet; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2TableModel; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualEnhanced; import org.apache.iotdb.itbase.env.BaseEnv; +import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; import org.apache.iotdb.rpc.IoTDBConnectionException; import org.apache.iotdb.rpc.StatementExecutionException; @@ -53,6 +54,7 @@ import java.util.List; import java.util.Random; import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; import static org.awaitility.Awaitility.await; import static org.junit.Assert.assertEquals; @@ -60,8 +62,8 @@ import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2TableModel.class}) -public class IoTDBPipeTypeConversionISessionIT extends AbstractPipeTableModelTestIT { +@Category({MultiClusterIT2DualTableManualEnhanced.class}) +public class IoTDBPipeTypeConversionISessionIT extends AbstractPipeTableModelDualManualIT { private static final int generateDataSize = 1000; @Test @@ -122,6 +124,11 @@ private void prepareTypeConversionTest( senderSession.executeNonQueryStatement("flush"); } + final Consumer handleFailure = + o -> { + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); + TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); + }; // Verify receiver data long timeoutSeconds = 600; List> expectedValues = @@ -138,7 +145,8 @@ private void prepareTypeConversionTest( query(receiverSession, tablet.getSchemas(), tablet.getTableName()), expectedValues, tablet.getTimestamps()); - } catch (Exception e) { + } catch (Exception | Error e) { + handleFailure.accept(e.getMessage()); fail(e.getMessage()); } }); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeTypeConversionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeTypeConversionIT.java similarity index 97% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeTypeConversionIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeTypeConversionIT.java index 40e485b580db..d9597574c7b1 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeTypeConversionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeTypeConversionIT.java @@ -17,13 +17,14 @@ * under the License. */ -package org.apache.iotdb.pipe.it.tablemodel; +package org.apache.iotdb.pipe.it.dual.tablemodel.manual.enhanced; import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.db.pipe.receiver.transform.converter.ValueConverter; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2TableModel; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualEnhanced; import org.apache.iotdb.itbase.env.BaseEnv; +import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; import org.apache.iotdb.rpc.RpcUtils; import org.apache.tsfile.enums.TSDataType; @@ -45,10 +46,11 @@ import java.util.List; import java.util.Random; import java.util.Set; +import java.util.function.Consumer; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2TableModel.class}) -public class IoTDBPipeTypeConversionIT extends AbstractPipeTableModelTestIT { +@Category({MultiClusterIT2DualTableManualEnhanced.class}) +public class IoTDBPipeTypeConversionIT extends AbstractPipeTableModelDualManualIT { private static final int generateDataSize = 100; @@ -202,12 +204,18 @@ public void testStringToOtherTypeConversion() { private void executeAndVerifyTypeConversion(TSDataType source, TSDataType target) { List pairs = prepareTypeConversionTest(source, target); + final Consumer handleFailure = + o -> { + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); + TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); + }; TestUtils.assertDataEventuallyOnEnv( receiverEnv, String.format("select time,status,s1 from %s2%s", source.name(), target.name()), "time,status,s1,", createExpectedResultSet(pairs, source, target), - "test"); + "test", + handleFailure); } private List prepareTypeConversionTest(TSDataType sourceType, TSDataType targetType) { diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/AbstractPipeDualAutoIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/AbstractPipeDualTreeModelAutoIT.java similarity index 97% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/AbstractPipeDualAutoIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/AbstractPipeDualTreeModelAutoIT.java index 3ad55920d294..4aa06e7acf31 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/AbstractPipeDualAutoIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/AbstractPipeDualTreeModelAutoIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.autocreate; +package org.apache.iotdb.pipe.it.dual.treemodel.auto; import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.consensus.ConsensusFactory; @@ -34,7 +34,7 @@ import java.util.Objects; import java.util.concurrent.TimeUnit; -abstract class AbstractPipeDualAutoIT { +public abstract class AbstractPipeDualTreeModelAutoIT { protected BaseEnv senderEnv; protected BaseEnv receiverEnv; diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAlterIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeAlterIT.java similarity index 98% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAlterIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeAlterIT.java index a2f5c65cb73e..b1c8c4cf995b 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAlterIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeAlterIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.autocreate; +package org.apache.iotdb.pipe.it.dual.treemodel.auto.basic; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; import org.apache.iotdb.confignode.rpc.thrift.TShowPipeInfo; @@ -25,7 +25,8 @@ import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2AutoCreateSchema; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoBasic; +import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeDualTreeModelAutoIT; import org.junit.Assert; import org.junit.Test; @@ -44,8 +45,8 @@ import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2AutoCreateSchema.class}) -public class IoTDBPipeAlterIT extends AbstractPipeDualAutoIT { +@Category({MultiClusterIT2DualTreeAutoBasic.class}) +public class IoTDBPipeAlterIT extends AbstractPipeDualTreeModelAutoIT { @Test public void testBasicAlterPipe() throws Exception { diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeConnectorParallelIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeConnectorParallelIT.java similarity index 92% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeConnectorParallelIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeConnectorParallelIT.java index 87bb4b465e56..8857f1942179 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeConnectorParallelIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeConnectorParallelIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.autocreate; +package org.apache.iotdb.pipe.it.dual.treemodel.auto.basic; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; @@ -25,7 +25,8 @@ import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2AutoCreateSchema; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoBasic; +import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeDualTreeModelAutoIT; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -40,8 +41,8 @@ import java.util.Set; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2AutoCreateSchema.class}) -public class IoTDBPipeConnectorParallelIT extends AbstractPipeDualAutoIT { +@Category({MultiClusterIT2DualTreeAutoBasic.class}) +public class IoTDBPipeConnectorParallelIT extends AbstractPipeDualTreeModelAutoIT { @Test public void testIoTConnectorParallel() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeDataSinkIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeDataSinkIT.java similarity index 98% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeDataSinkIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeDataSinkIT.java index 7facee0bbf06..8855bf4e25f4 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeDataSinkIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeDataSinkIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.autocreate; +package org.apache.iotdb.pipe.it.dual.treemodel.auto.basic; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; @@ -25,7 +25,8 @@ import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2AutoCreateSchema; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoBasic; +import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeDualTreeModelAutoIT; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -40,8 +41,8 @@ import java.util.Map; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2AutoCreateSchema.class}) -public class IoTDBPipeDataSinkIT extends AbstractPipeDualAutoIT { +@Category({MultiClusterIT2DualTreeAutoBasic.class}) +public class IoTDBPipeDataSinkIT extends AbstractPipeDualTreeModelAutoIT { @Test public void testThriftConnectorWithRealtimeFirstDisabled() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeExtractorIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeExtractorIT.java similarity index 99% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeExtractorIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeExtractorIT.java index 0b4636c0d137..32075c4040cc 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeExtractorIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeExtractorIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.autocreate; +package org.apache.iotdb.pipe.it.dual.treemodel.auto.basic; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; @@ -29,8 +29,9 @@ import org.apache.iotdb.it.env.MultiEnvFactory; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2AutoCreateSchema; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoBasic; import org.apache.iotdb.itbase.env.BaseEnv; +import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeDualTreeModelAutoIT; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -52,8 +53,8 @@ import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2AutoCreateSchema.class}) -public class IoTDBPipeExtractorIT extends AbstractPipeDualAutoIT { +@Category({MultiClusterIT2DualTreeAutoBasic.class}) +public class IoTDBPipeExtractorIT extends AbstractPipeDualTreeModelAutoIT { @Before public void setUp() { diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeLifeCycleIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeLifeCycleIT.java similarity index 97% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeLifeCycleIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeLifeCycleIT.java index f151463b396c..9e03b3d80320 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeLifeCycleIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeLifeCycleIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.autocreate; +package org.apache.iotdb.pipe.it.dual.treemodel.auto.basic; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.auth.entity.PrivilegeType; @@ -26,7 +26,8 @@ import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2AutoCreateSchema; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoBasic; +import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeDualTreeModelAutoIT; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -45,14 +46,14 @@ import static org.apache.iotdb.db.it.utils.TestUtils.assertNonQueryTestFail; import static org.apache.iotdb.db.it.utils.TestUtils.assertTestFail; import static org.apache.iotdb.db.it.utils.TestUtils.createUser; +import static org.apache.iotdb.db.it.utils.TestUtils.executeNonQueriesWithRetry; +import static org.apache.iotdb.db.it.utils.TestUtils.executeNonQueryWithRetry; import static org.apache.iotdb.db.it.utils.TestUtils.executeQueryWithRetry; import static org.apache.iotdb.db.it.utils.TestUtils.grantUserSystemPrivileges; -import static org.apache.iotdb.db.it.utils.TestUtils.tryExecuteNonQueriesWithRetry; -import static org.apache.iotdb.db.it.utils.TestUtils.tryExecuteNonQueryWithRetry; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2AutoCreateSchema.class}) -public class IoTDBPipeLifeCycleIT extends AbstractPipeDualAutoIT { +@Category({MultiClusterIT2DualTreeAutoBasic.class}) +public class IoTDBPipeLifeCycleIT extends AbstractPipeDualTreeModelAutoIT { @Test public void testLifeCycleWithHistoryEnabled() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); @@ -826,18 +827,16 @@ public void testPermission() { grantUserSystemPrivileges(senderEnv, "test", PrivilegeType.USE_PIPE); - tryExecuteNonQueryWithRetry( + executeNonQueryWithRetry( senderEnv, "create pipe testPipe\n" + "with connector (\n" - + " 'connector'='iotdb-thrift-connector',\n" - + " 'connector.ip'='127.0.0.1',\n" - + " 'connector.port'='6668'\n" + + " 'connector'='write-back-connector'\n" + ")", "test", "test123"); executeQueryWithRetry(senderEnv, "show pipes", "test", "test123"); - tryExecuteNonQueriesWithRetry( + executeNonQueriesWithRetry( senderEnv, Arrays.asList("start pipe testPipe", "stop pipe testPipe", "drop pipe testPipe"), "test", @@ -849,7 +848,6 @@ public void testPermission() { "701: Untrusted uri xxx", "test", "test123"); - tryExecuteNonQueryWithRetry(senderEnv, "drop pipePlugin TestProcessor", "test", "test123"); executeQueryWithRetry(senderEnv, "show pipe plugins", "test", "test123"); } } diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeProcessorIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeProcessorIT.java similarity index 94% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeProcessorIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeProcessorIT.java index 27bc5be96599..4a17939065f4 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeProcessorIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeProcessorIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.autocreate; +package org.apache.iotdb.pipe.it.dual.treemodel.auto.basic; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; @@ -27,7 +27,8 @@ import org.apache.iotdb.it.env.MultiEnvFactory; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2AutoCreateSchema; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoBasic; +import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeDualTreeModelAutoIT; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -43,8 +44,8 @@ import java.util.Set; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2AutoCreateSchema.class}) -public class IoTDBPipeProcessorIT extends AbstractPipeDualAutoIT { +@Category({MultiClusterIT2DualTreeAutoBasic.class}) +public class IoTDBPipeProcessorIT extends AbstractPipeDualTreeModelAutoIT { @Before public void setUp() { MultiEnvFactory.createEnv(2); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeProtocolIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeProtocolIT.java similarity index 98% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeProtocolIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeProtocolIT.java index c5d41f001cb8..6035ed9c16a0 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeProtocolIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeProtocolIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.autocreate; +package org.apache.iotdb.pipe.it.dual.treemodel.auto.basic; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; @@ -28,7 +28,8 @@ import org.apache.iotdb.it.env.MultiEnvFactory; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2AutoCreateSchema; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoBasic; +import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeDualTreeModelAutoIT; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -44,8 +45,8 @@ /** Test pipe's basic functionalities under multiple cluster and consensus protocol settings. */ @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2AutoCreateSchema.class}) -public class IoTDBPipeProtocolIT extends AbstractPipeDualAutoIT { +@Category({MultiClusterIT2DualTreeAutoBasic.class}) +public class IoTDBPipeProtocolIT extends AbstractPipeDualTreeModelAutoIT { @Override @Before public void setUp() { diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeSwitchStatusIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeSwitchStatusIT.java similarity index 97% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeSwitchStatusIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeSwitchStatusIT.java index 03e52a982e61..ee70f1a3a3ba 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeSwitchStatusIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeSwitchStatusIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.autocreate; +package org.apache.iotdb.pipe.it.dual.treemodel.auto.basic; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; @@ -28,7 +28,8 @@ import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2AutoCreateSchema; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoBasic; +import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeDualTreeModelAutoIT; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -41,8 +42,8 @@ import java.util.Map; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2AutoCreateSchema.class}) -public class IoTDBPipeSwitchStatusIT extends AbstractPipeDualAutoIT { +@Category({MultiClusterIT2DualTreeAutoBasic.class}) +public class IoTDBPipeSwitchStatusIT extends AbstractPipeDualTreeModelAutoIT { @Test public void testPipeSwitchStatus() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeSyntaxIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeSyntaxIT.java similarity index 98% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeSyntaxIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeSyntaxIT.java index b9de28b71586..a0450942d473 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeSyntaxIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeSyntaxIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.autocreate; +package org.apache.iotdb.pipe.it.dual.treemodel.auto.basic; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; @@ -26,7 +26,8 @@ import org.apache.iotdb.confignode.rpc.thrift.TShowPipeReq; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2AutoCreateSchema; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoBasic; +import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeDualTreeModelAutoIT; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -45,8 +46,8 @@ import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2AutoCreateSchema.class}) -public class IoTDBPipeSyntaxIT extends AbstractPipeDualAutoIT { +@Category({MultiClusterIT2DualTreeAutoBasic.class}) +public class IoTDBPipeSyntaxIT extends AbstractPipeDualTreeModelAutoIT { @Test public void testValidPipeName() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBTreePatternFormatIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBTreePatternFormatIT.java similarity index 95% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBTreePatternFormatIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBTreePatternFormatIT.java index b62102f545df..28fa66213126 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBTreePatternFormatIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBTreePatternFormatIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.autocreate; +package org.apache.iotdb.pipe.it.dual.treemodel.auto.basic; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; @@ -25,7 +25,8 @@ import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2AutoCreateSchema; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoBasic; +import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeDualTreeModelAutoIT; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -40,8 +41,8 @@ import java.util.Set; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2AutoCreateSchema.class}) -public class IoTDBTreePatternFormatIT extends AbstractPipeDualAutoIT { +@Category({MultiClusterIT2DualTreeAutoBasic.class}) +public class IoTDBTreePatternFormatIT extends AbstractPipeDualTreeModelAutoIT { @Test public void testPrefixPattern() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAutoConflictIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeAutoConflictIT.java similarity index 98% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAutoConflictIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeAutoConflictIT.java index 65eb2458df77..65eb0d1ea140 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAutoConflictIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeAutoConflictIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.autocreate; +package org.apache.iotdb.pipe.it.dual.treemodel.auto.enhanced; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; @@ -27,7 +27,8 @@ import org.apache.iotdb.it.env.MultiEnvFactory; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2AutoCreateSchema; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoEnhanced; +import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeDualTreeModelAutoIT; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -44,8 +45,8 @@ import java.util.Set; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2AutoCreateSchema.class}) -public class IoTDBPipeAutoConflictIT extends AbstractPipeDualAutoIT { +@Category({MultiClusterIT2DualTreeAutoEnhanced.class}) +public class IoTDBPipeAutoConflictIT extends AbstractPipeDualTreeModelAutoIT { @Before public void setUp() { MultiEnvFactory.createEnv(2); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAutoDropIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeAutoDropIT.java similarity index 95% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAutoDropIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeAutoDropIT.java index adff5e5f53da..f1c3672e14a2 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAutoDropIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeAutoDropIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.autocreate; +package org.apache.iotdb.pipe.it.dual.treemodel.auto.enhanced; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; @@ -27,7 +27,8 @@ import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2AutoCreateSchema; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoEnhanced; +import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeDualTreeModelAutoIT; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -47,8 +48,8 @@ import static org.awaitility.Awaitility.await; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2AutoCreateSchema.class}) -public class IoTDBPipeAutoDropIT extends AbstractPipeDualAutoIT { +@Category({MultiClusterIT2DualTreeAutoEnhanced.class}) +public class IoTDBPipeAutoDropIT extends AbstractPipeDualTreeModelAutoIT { @Test public void testAutoDropInHistoricalTransfer() throws Exception { diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeClusterIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeClusterIT.java similarity index 98% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeClusterIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeClusterIT.java index ad8c86ea054c..ff827d6332b6 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeClusterIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeClusterIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.autocreate; +package org.apache.iotdb.pipe.it.dual.treemodel.auto.enhanced; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.exception.ClientManagerException; @@ -34,8 +34,9 @@ import org.apache.iotdb.it.env.cluster.env.AbstractEnv; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2AutoCreateSchema; -import org.apache.iotdb.pipe.it.tablemodel.TableModelUtils; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoEnhanced; +import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; +import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeDualTreeModelAutoIT; import org.apache.iotdb.rpc.TSStatusCode; import org.apache.thrift.TException; @@ -59,8 +60,8 @@ import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2AutoCreateSchema.class}) -public class IoTDBPipeClusterIT extends AbstractPipeDualAutoIT { +@Category({MultiClusterIT2DualTreeAutoEnhanced.class}) +public class IoTDBPipeClusterIT extends AbstractPipeDualTreeModelAutoIT { @Override @Before diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeConditionalOperationsIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeConditionalOperationsIT.java similarity index 96% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeConditionalOperationsIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeConditionalOperationsIT.java index e6dfd37a307d..36a12d5bcd6e 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeConditionalOperationsIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeConditionalOperationsIT.java @@ -17,14 +17,15 @@ * under the License. */ -package org.apache.iotdb.pipe.it.autocreate; +package org.apache.iotdb.pipe.it.dual.treemodel.auto.enhanced; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; import org.apache.iotdb.confignode.rpc.thrift.TShowPipeInfo; import org.apache.iotdb.confignode.rpc.thrift.TShowPipeReq; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2AutoCreateSchema; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoEnhanced; +import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeDualTreeModelAutoIT; import org.junit.Assert; import org.junit.Test; @@ -39,8 +40,8 @@ import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2AutoCreateSchema.class}) -public class IoTDBPipeConditionalOperationsIT extends AbstractPipeDualAutoIT { +@Category({MultiClusterIT2DualTreeAutoEnhanced.class}) +public class IoTDBPipeConditionalOperationsIT extends AbstractPipeDualTreeModelAutoIT { @Test public void testBasicCreatePipeIfNotExists() throws Exception { diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeConnectorCompressionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeConnectorCompressionIT.java similarity index 97% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeConnectorCompressionIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeConnectorCompressionIT.java index 2a2c83522715..37a542e6aa9f 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeConnectorCompressionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeConnectorCompressionIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.autocreate; +package org.apache.iotdb.pipe.it.dual.treemodel.auto.enhanced; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; @@ -30,7 +30,8 @@ import org.apache.iotdb.it.env.MultiEnvFactory; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2AutoCreateSchema; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoEnhanced; +import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeDualTreeModelAutoIT; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -51,8 +52,8 @@ import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2AutoCreateSchema.class}) -public class IoTDBPipeConnectorCompressionIT extends AbstractPipeDualAutoIT { +@Category({MultiClusterIT2DualTreeAutoEnhanced.class}) +public class IoTDBPipeConnectorCompressionIT extends AbstractPipeDualTreeModelAutoIT { @Override @Before diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeIdempotentIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeIdempotentIT.java similarity index 80% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeIdempotentIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeIdempotentIT.java index cbe1ad065578..e741ab82f6d9 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeIdempotentIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeIdempotentIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.autocreate; +package org.apache.iotdb.pipe.it.dual.treemodel.auto.enhanced; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; @@ -27,9 +27,8 @@ import org.apache.iotdb.it.env.MultiEnvFactory; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2AutoCreateSchema; -import org.apache.iotdb.itbase.env.BaseEnv; -import org.apache.iotdb.pipe.it.tablemodel.TableModelUtils; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoEnhanced; +import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeDualTreeModelAutoIT; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -46,8 +45,8 @@ import java.util.Set; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2AutoCreateSchema.class}) -public class IoTDBPipeIdempotentIT extends AbstractPipeDualAutoIT { +@Category({MultiClusterIT2DualTreeAutoEnhanced.class}) +public class IoTDBPipeIdempotentIT extends AbstractPipeDualTreeModelAutoIT { @Override @Before public void setUp() { @@ -396,36 +395,6 @@ public void testDropRoleIdempotent() throws Exception { // Table model - @Test - public void testCreateTableIdempotent() throws Exception { - testTableConfigIdempotent(Collections.emptyList(), "create table test()"); - } - - @Test - public void testAlterTableAddColumnIdempotent() throws Exception { - testTableConfigIdempotent( - Collections.singletonList("create table test()"), "alter table test add column a id"); - } - - @Test - public void testAlterTableSetPropertiesIdempotent() throws Exception { - testTableConfigIdempotent( - Collections.singletonList("create table test()"), - "alter table test set properties ttl=100"); - } - - @Test - public void testAlterTableDropColumnIdempotent() throws Exception { - testTableConfigIdempotent( - Collections.singletonList("create table test(a id, b attribute, c int32)"), - "alter table test drop column b"); - } - - @Test - public void testDropTableIdempotent() throws Exception { - testTableConfigIdempotent(Collections.singletonList("create table test()"), "drop table test"); - } - private void testIdempotent( final List beforeSqlList, final String testSql, @@ -483,63 +452,4 @@ private void testIdempotent( // Assume that the afterSql is executed on receiverEnv TestUtils.assertDataEventuallyOnEnv(receiverEnv, afterSqlQuery, expectedHeader, expectedResSet); } - - private void testTableConfigIdempotent(final List beforeSqlList, final String testSql) - throws Exception { - final String database = "test"; - TableModelUtils.createDatabase(senderEnv, database); - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - extractorAttributes.put("extractor.inclusion", "all"); - extractorAttributes.put("extractor.inclusion.exclusion", ""); - extractorAttributes.put("extractor.forwarding-pipe-requests", "false"); - extractorAttributes.put("extractor.capture.table", "true"); - extractorAttributes.put("extractor.capture.tree", "false"); - - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.exception.conflict.resolve-strategy", "retry"); - connectorAttributes.put("connector.exception.conflict.retry-max-time-seconds", "-1"); - - final TSStatus status = - client.createPipe( - new TCreatePipeReq("testPipe", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - } - - if (!TestUtils.tryExecuteNonQueriesWithRetry( - database, BaseEnv.TABLE_SQL_DIALECT, senderEnv, beforeSqlList)) { - return; - } - - if (!TestUtils.tryExecuteNonQueryWithRetry( - database, BaseEnv.TABLE_SQL_DIALECT, receiverEnv, testSql)) { - return; - } - - // Create an idempotent conflict - if (!TestUtils.tryExecuteNonQueryWithRetry( - database, BaseEnv.TABLE_SQL_DIALECT, senderEnv, testSql)) { - return; - } - - TableModelUtils.createDatabase(senderEnv, "test2"); - - // Assume that the "database" is executed on receiverEnv - TestUtils.assertDataSizeEventuallyOnEnv(receiverEnv, "show databases", 3, null); - } } diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeNullValueIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeNullValueIT.java similarity index 97% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeNullValueIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeNullValueIT.java index 3231fc523aab..765104e1f523 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeNullValueIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeNullValueIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.autocreate; +package org.apache.iotdb.pipe.it.dual.treemodel.auto.enhanced; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; @@ -26,7 +26,8 @@ import org.apache.iotdb.isession.ISession; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2AutoCreateSchema; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoEnhanced; +import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeDualTreeModelAutoIT; import org.apache.iotdb.rpc.IoTDBConnectionException; import org.apache.iotdb.rpc.StatementExecutionException; import org.apache.iotdb.rpc.TSStatusCode; @@ -51,8 +52,8 @@ import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2AutoCreateSchema.class}) -public class IoTDBPipeNullValueIT extends AbstractPipeDualAutoIT { +@Category({MultiClusterIT2DualTreeAutoEnhanced.class}) +public class IoTDBPipeNullValueIT extends AbstractPipeDualTreeModelAutoIT { // Test dimensions: // 1. is or not aligned diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeWithLoadIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeWithLoadIT.java similarity index 93% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeWithLoadIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeWithLoadIT.java index 563377c927ea..0b929e4daf88 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeWithLoadIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeWithLoadIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.autocreate; +package org.apache.iotdb.pipe.it.dual.treemodel.auto.enhanced; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; @@ -27,7 +27,8 @@ import org.apache.iotdb.it.env.MultiEnvFactory; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2AutoCreateSchema; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoEnhanced; +import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeDualTreeModelAutoIT; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -42,8 +43,8 @@ import java.util.Map; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2AutoCreateSchema.class}) -public class IoTDBPipeWithLoadIT extends AbstractPipeDualAutoIT { +@Category({MultiClusterIT2DualTreeAutoEnhanced.class}) +public class IoTDBPipeWithLoadIT extends AbstractPipeDualTreeModelAutoIT { @Before public void setUp() { diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/PipeNowFunctionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/PipeNowFunctionIT.java similarity index 97% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/PipeNowFunctionIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/PipeNowFunctionIT.java index 6a2106b554d1..471a1e815948 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/PipeNowFunctionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/PipeNowFunctionIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.autocreate; +package org.apache.iotdb.pipe.it.dual.treemodel.auto.enhanced; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; @@ -27,8 +27,9 @@ import org.apache.iotdb.confignode.rpc.thrift.TShowPipeReq; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2AutoCreateSchema; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoEnhanced; import org.apache.iotdb.itbase.env.BaseEnv; +import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeDualTreeModelAutoIT; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -46,8 +47,8 @@ import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2AutoCreateSchema.class}) -public class PipeNowFunctionIT extends AbstractPipeDualAutoIT { +@Category({MultiClusterIT2DualTreeAutoEnhanced.class}) +public class PipeNowFunctionIT extends AbstractPipeDualTreeModelAutoIT { @Test public void testPipeNowFunction() throws Exception { diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/AbstractPipeDualManualIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/AbstractPipeDualTreeModelManualIT.java similarity index 97% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/AbstractPipeDualManualIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/AbstractPipeDualTreeModelManualIT.java index c93f80a513a4..581d363cdb5c 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/AbstractPipeDualManualIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/AbstractPipeDualTreeModelManualIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.manual; +package org.apache.iotdb.pipe.it.dual.treemodel.manual; import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.consensus.ConsensusFactory; @@ -34,7 +34,7 @@ import java.util.Objects; import java.util.concurrent.TimeUnit; -abstract class AbstractPipeDualManualIT { +public abstract class AbstractPipeDualTreeModelManualIT { protected BaseEnv senderEnv; protected BaseEnv receiverEnv; diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeInclusionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeInclusionIT.java similarity index 97% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeInclusionIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeInclusionIT.java index 76affe08850f..4bc4505c1b6f 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeInclusionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeInclusionIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.manual; +package org.apache.iotdb.pipe.it.dual.treemodel.manual; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; @@ -25,7 +25,7 @@ import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2ManualCreateSchema; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeManual; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -40,8 +40,8 @@ import java.util.Map; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2ManualCreateSchema.class}) -public class IoTDBPipeInclusionIT extends AbstractPipeDualManualIT { +@Category({MultiClusterIT2DualTreeManual.class}) +public class IoTDBPipeInclusionIT extends AbstractPipeDualTreeModelManualIT { @Test public void testPureSchemaInclusion() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeManualConflictIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeManualConflictIT.java similarity index 97% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeManualConflictIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeManualConflictIT.java index dbb181092286..63bef4ff759c 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeManualConflictIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeManualConflictIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.manual; +package org.apache.iotdb.pipe.it.dual.treemodel.manual; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; @@ -25,7 +25,7 @@ import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2ManualCreateSchema; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeManual; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -40,8 +40,8 @@ import java.util.Map; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2ManualCreateSchema.class}) -public class IoTDBPipeManualConflictIT extends AbstractPipeDualManualIT { +@Category({MultiClusterIT2DualTreeManual.class}) +public class IoTDBPipeManualConflictIT extends AbstractPipeDualTreeModelManualIT { @Test public void testDoubleLivingTimeseries() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeMetaHistoricalIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeMetaHistoricalIT.java similarity index 98% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeMetaHistoricalIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeMetaHistoricalIT.java index 6c36de7b6966..c351aad274c4 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeMetaHistoricalIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeMetaHistoricalIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.manual; +package org.apache.iotdb.pipe.it.dual.treemodel.manual; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; @@ -28,7 +28,7 @@ import org.apache.iotdb.it.env.MultiEnvFactory; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2ManualCreateSchema; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeManual; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -44,8 +44,8 @@ import java.util.Map; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2ManualCreateSchema.class}) -public class IoTDBPipeMetaHistoricalIT extends AbstractPipeDualManualIT { +@Category({MultiClusterIT2DualTreeManual.class}) +public class IoTDBPipeMetaHistoricalIT extends AbstractPipeDualTreeModelManualIT { @Override @Before public void setUp() { diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeMetaLeaderChangeIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeMetaLeaderChangeIT.java similarity index 97% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeMetaLeaderChangeIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeMetaLeaderChangeIT.java index c03f56bad4ee..00281253abb0 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeMetaLeaderChangeIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeMetaLeaderChangeIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.manual; +package org.apache.iotdb.pipe.it.dual.treemodel.manual; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; @@ -27,7 +27,7 @@ import org.apache.iotdb.it.env.MultiEnvFactory; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2ManualCreateSchema; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeManual; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -41,8 +41,8 @@ import java.util.Map; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2ManualCreateSchema.class}) -public class IoTDBPipeMetaLeaderChangeIT extends AbstractPipeDualManualIT { +@Category({MultiClusterIT2DualTreeManual.class}) +public class IoTDBPipeMetaLeaderChangeIT extends AbstractPipeDualTreeModelManualIT { @Override @Before public void setUp() { diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeMetaRestartIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeMetaRestartIT.java similarity index 96% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeMetaRestartIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeMetaRestartIT.java index 76cd4a90e8b8..ec9930d2642b 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeMetaRestartIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeMetaRestartIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.manual; +package org.apache.iotdb.pipe.it.dual.treemodel.manual; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; @@ -25,7 +25,7 @@ import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2ManualCreateSchema; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeManual; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -38,8 +38,8 @@ import java.util.Map; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2ManualCreateSchema.class}) -public class IoTDBPipeMetaRestartIT extends AbstractPipeDualManualIT { +@Category({MultiClusterIT2DualTreeManual.class}) +public class IoTDBPipeMetaRestartIT extends AbstractPipeDualTreeModelManualIT { @Test public void testAutoRestartSchemaTask() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeMultiSchemaRegionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeMultiSchemaRegionIT.java similarity index 95% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeMultiSchemaRegionIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeMultiSchemaRegionIT.java index 769b940c78d2..8c210f64638d 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeMultiSchemaRegionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeMultiSchemaRegionIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.manual; +package org.apache.iotdb.pipe.it.dual.treemodel.manual; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; @@ -25,7 +25,7 @@ import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2ManualCreateSchema; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeManual; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -39,8 +39,8 @@ import java.util.Map; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2ManualCreateSchema.class}) -public class IoTDBPipeMultiSchemaRegionIT extends AbstractPipeDualManualIT { +@Category({MultiClusterIT2DualTreeManual.class}) +public class IoTDBPipeMultiSchemaRegionIT extends AbstractPipeDualTreeModelManualIT { @Test public void testMultiSchemaRegion() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipePermissionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipePermissionIT.java similarity index 97% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipePermissionIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipePermissionIT.java index 648899b13ea5..52320d52bd95 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipePermissionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipePermissionIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.manual; +package org.apache.iotdb.pipe.it.dual.treemodel.manual; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; @@ -27,7 +27,7 @@ import org.apache.iotdb.it.env.MultiEnvFactory; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2ManualCreateSchema; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeManual; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -46,8 +46,8 @@ import static org.junit.jupiter.api.Assertions.fail; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2ManualCreateSchema.class}) -public class IoTDBPipePermissionIT extends AbstractPipeDualManualIT { +@Category({MultiClusterIT2DualTreeManual.class}) +public class IoTDBPipePermissionIT extends AbstractPipeDualTreeModelManualIT { @Override @Before public void setUp() { diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeReqAutoSliceIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeReqAutoSliceIT.java similarity index 98% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeReqAutoSliceIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeReqAutoSliceIT.java index 953f1faec33c..55b5f72314da 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeReqAutoSliceIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeReqAutoSliceIT.java @@ -17,13 +17,13 @@ * under the License. */ -package org.apache.iotdb.pipe.it.manual; +package org.apache.iotdb.pipe.it.dual.treemodel.manual; import org.apache.iotdb.commons.utils.function.CheckedTriConsumer; import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.isession.ISession; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2ManualCreateSchema; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeManual; import org.apache.iotdb.rpc.IoTDBConnectionException; import org.apache.iotdb.rpc.StatementExecutionException; @@ -46,8 +46,8 @@ import java.util.Random; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2ManualCreateSchema.class}) -public class IoTDBPipeReqAutoSliceIT extends AbstractPipeDualManualIT { +@Category({MultiClusterIT2DualTreeManual.class}) +public class IoTDBPipeReqAutoSliceIT extends AbstractPipeDualTreeModelManualIT { private static final int generateDataSize = 10; @Override diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeTypeConversionISessionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTypeConversionISessionIT.java similarity index 99% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeTypeConversionISessionIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTypeConversionISessionIT.java index b94aea2f2371..23168924aed1 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeTypeConversionISessionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTypeConversionISessionIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.manual; +package org.apache.iotdb.pipe.it.dual.treemodel.manual; import org.apache.iotdb.commons.utils.function.CheckedTriConsumer; import org.apache.iotdb.db.it.utils.TestUtils; @@ -25,7 +25,7 @@ import org.apache.iotdb.isession.ISession; import org.apache.iotdb.isession.SessionDataSet; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2ManualCreateSchema; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeManual; import org.apache.iotdb.itbase.env.BaseEnv; import org.apache.iotdb.rpc.IoTDBConnectionException; import org.apache.iotdb.rpc.RpcUtils; @@ -62,8 +62,8 @@ import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2ManualCreateSchema.class}) -public class IoTDBPipeTypeConversionISessionIT extends AbstractPipeDualManualIT { +@Category({MultiClusterIT2DualTreeManual.class}) +public class IoTDBPipeTypeConversionISessionIT extends AbstractPipeDualTreeModelManualIT { private static final int generateDataSize = 100; @Test diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeTypeConversionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTypeConversionIT.java similarity index 98% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeTypeConversionIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTypeConversionIT.java index f7cb595acbf0..e87bda0f361d 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeTypeConversionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTypeConversionIT.java @@ -17,12 +17,12 @@ * under the License. */ -package org.apache.iotdb.pipe.it.manual; +package org.apache.iotdb.pipe.it.dual.treemodel.manual; import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.db.pipe.receiver.transform.converter.ValueConverter; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2ManualCreateSchema; +import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeManual; import org.apache.iotdb.itbase.env.BaseEnv; import org.apache.iotdb.rpc.RpcUtils; @@ -46,8 +46,8 @@ import java.util.Set; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2ManualCreateSchema.class}) -public class IoTDBPipeTypeConversionIT extends AbstractPipeDualManualIT { +@Category({MultiClusterIT2DualTreeManual.class}) +public class IoTDBPipeTypeConversionIT extends AbstractPipeDualTreeModelManualIT { private static final int generateDataSize = 100; diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBPipeOPCUAIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBPipeOPCUAIT.java index 9b8388659771..591881369ed4 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBPipeOPCUAIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBPipeOPCUAIT.java @@ -25,7 +25,7 @@ import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.MultiClusterIT1; import org.apache.iotdb.itbase.env.BaseEnv; -import org.apache.iotdb.pipe.it.tablemodel.TableModelUtils; +import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBAuthenticationTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBAuthenticationTableIT.java new file mode 100644 index 000000000000..aad3086b8080 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBAuthenticationTableIT.java @@ -0,0 +1,1001 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.relational.it.db.it; + +import org.apache.iotdb.isession.ITableSession; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.record.Tablet.ColumnCategory; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.util.Arrays; +import java.util.Locale; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +public class IoTDBAuthenticationTableIT { + @BeforeClass + public static void setUpClass() { + Locale.setDefault(Locale.ENGLISH); + EnvFactory.getEnv().initClusterEnvironment(); + } + + @After + public void tearDown() throws Exception { + try (ITableSession sessionRoot = EnvFactory.getEnv().getTableSessionConnection()) { + String[] sqls = + new String[] { + "DROP USER userA", + "DROP USER userB", + "DROP USER userC", + "DROP USER userD", + "DROP ROLE role1", + "DROP ROLE role2", + }; + for (String sql : sqls) { + try { + sessionRoot.executeNonQueryStatement(sql); + } catch (StatementExecutionException ignore) { + } + } + } + } + + @AfterClass + public static void tearDownClass() { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testInsert() throws IoTDBConnectionException, StatementExecutionException { + + try (ITableSession sessionRoot = EnvFactory.getEnv().getTableSessionConnection()) { + sessionRoot.executeNonQueryStatement("CREATE DATABASE test"); + sessionRoot.executeNonQueryStatement("USE test"); + + // insert by root + Tablet tablet = + new Tablet( + "table1", + Arrays.asList("id", "attr", "measurement"), + Arrays.asList(TSDataType.STRING, TSDataType.STRING, TSDataType.DOUBLE), + Arrays.asList(ColumnCategory.TAG, ColumnCategory.ATTRIBUTE, ColumnCategory.FIELD)); + tablet.addTimestamp(0, 0); + tablet.addValue(0, 0, "id1"); + tablet.addValue(0, 1, "attr1"); + tablet.addValue(0, 2, 0.1); + + sessionRoot.insert(tablet); + + sessionRoot.executeNonQueryStatement( + "INSERT INTO table1 (time, id, attr, measurement) VALUES (1, 'id2', 'attr2', 0.2)"); + + // revoke root + try { + sessionRoot.executeNonQueryStatement("REVOKE INSERT ON ANY FROM USER root"); + fail("Should have thrown an exception"); + } catch (StatementExecutionException e) { + assertEquals("803: Cannot grant/revoke privileges to/from admin", e.getMessage()); + } + try { + sessionRoot.executeNonQueryStatement("REVOKE INSERT ON DATABASE test FROM USER root"); + fail("Should have thrown an exception"); + } catch (StatementExecutionException e) { + assertEquals("803: Cannot grant/revoke privileges of admin user", e.getMessage()); + } + try { + sessionRoot.executeNonQueryStatement("REVOKE INSERT ON TABLE table1 FROM USER root"); + fail("Should have thrown an exception"); + } catch (StatementExecutionException e) { + assertEquals("803: Cannot grant/revoke privileges of admin user", e.getMessage()); + } + + // test users + sessionRoot.executeNonQueryStatement("CREATE USER userA 'userA'"); + sessionRoot.executeNonQueryStatement("CREATE USER userB 'userB'"); + // grant an irrelevant privilege so that the new users can use database + sessionRoot.executeNonQueryStatement("GRANT SELECT ON DATABASE test TO USER userA"); + sessionRoot.executeNonQueryStatement("GRANT SELECT ON DATABASE test TO USER userB"); + + try (ITableSession sessionA = + EnvFactory.getEnv().getTableSessionConnection("userA", "userA"); + ITableSession sessionB = + EnvFactory.getEnv().getTableSessionConnection("userB", "userB")) { + sessionA.executeNonQueryStatement("USE test"); + sessionB.executeNonQueryStatement("USE test"); + // userA no privilege + try { + sessionA.executeNonQueryStatement( + "INSERT INTO table1 (time, id, attr, measurement) VALUES (1, 'id2', 'attr2', 0.2)"); + fail("Should have thrown an exception"); + } catch (StatementExecutionException e) { + assertEquals( + "803: Access Denied: No permissions for this operation, please add privilege INSERT ON test.table1", + e.getMessage()); + } + + // grant and revoke - ALL + sessionRoot.executeNonQueryStatement("GRANT ALL TO USER userA"); + sessionA.executeNonQueryStatement( + "INSERT INTO table1 (time, id, attr, measurement) VALUES (1, 'id2', 'attr2', 0.2)"); + sessionRoot.executeNonQueryStatement("REVOKE ALL FROM USER userA"); + try { + sessionA.executeNonQueryStatement( + "INSERT INTO table1 (time, id, attr, measurement) VALUES (1, 'id2', 'attr2', 0.2)"); + fail("Should have thrown an exception"); + } catch (StatementExecutionException e) { + assertEquals( + "803: Access Denied: No permissions for this operation, please add privilege INSERT ON test.table1", + e.getMessage()); + } + + // grant and revoke - ANY + sessionRoot.executeNonQueryStatement("GRANT INSERT ON ANY TO USER userA"); + sessionA.executeNonQueryStatement( + "INSERT INTO table1 (time, id, attr, measurement) VALUES (1, 'id2', 'attr2', 0.2)"); + sessionRoot.executeNonQueryStatement("REVOKE INSERT ON ANY FROM USER userA"); + try { + sessionA.executeNonQueryStatement( + "INSERT INTO table1 (time, id, attr, measurement) VALUES (1, 'id2', 'attr2', 0.2)"); + fail("Should have thrown an exception"); + } catch (StatementExecutionException e) { + assertEquals( + "803: Access Denied: No permissions for this operation, please add privilege INSERT ON test.table1", + e.getMessage()); + } + + // grant and revoke - database + sessionRoot.executeNonQueryStatement("GRANT INSERT ON DATABASE test TO USER userA"); + sessionA.executeNonQueryStatement( + "INSERT INTO table1 (time, id, attr, measurement) VALUES (1, 'id2', 'attr2', 0.2)"); + sessionRoot.executeNonQueryStatement("REVOKE INSERT ON DATABASE test FROM USER userA"); + try { + sessionA.executeNonQueryStatement( + "INSERT INTO table1 (time, id, attr, measurement) VALUES (1, 'id2', 'attr2', 0.2)"); + fail("Should have thrown an exception"); + } catch (StatementExecutionException e) { + assertEquals( + "803: Access Denied: No permissions for this operation, please add privilege INSERT ON test.table1", + e.getMessage()); + } + + // grant and revoke - table + sessionRoot.executeNonQueryStatement("GRANT INSERT ON TABLE table1 TO USER userA"); + sessionA.executeNonQueryStatement( + "INSERT INTO table1 (time, id, attr, measurement) VALUES (1, 'id2', 'attr2', 0.2)"); + sessionRoot.executeNonQueryStatement("REVOKE INSERT ON TABLE table1 FROM USER userA"); + try { + sessionA.executeNonQueryStatement( + "INSERT INTO table1 (time, id, attr, measurement) VALUES (1, 'id2', 'attr2', 0.2)"); + fail("Should have thrown an exception"); + } catch (StatementExecutionException e) { + assertEquals( + "803: Access Denied: No permissions for this operation, please add privilege INSERT ON test.table1", + e.getMessage()); + } + + // can write but cannot auto-create + sessionRoot.executeNonQueryStatement("GRANT INSERT ON DATABASE test TO USER userA"); + tablet.setTableName("table2"); + try { + sessionA.insert(tablet); + fail("Should have thrown an exception"); + } catch (StatementExecutionException e) { + assertEquals( + "301: [EXECUTE_STATEMENT_ERROR(301)] Exception occurred: insertTablet failed. Access Denied: No permissions for this operation, please add privilege CREATE ON test.table2", + e.getMessage()); + } + sessionRoot.executeNonQueryStatement("GRANT CREATE ON DATABASE test TO USER userA"); + sessionA.insert(tablet); + sessionRoot.executeNonQueryStatement("REVOKE CREATE ON DATABASE test FROM USER userA"); + sessionRoot.executeNonQueryStatement("REVOKE INSERT ON DATABASE test FROM USER userA"); + + // can write but cannot add column + sessionRoot.executeNonQueryStatement("GRANT INSERT ON DATABASE test TO USER userA"); + tablet = + new Tablet( + "table2", + Arrays.asList("id", "attr", "measurement", "id2", "attr2", "measurement2"), + Arrays.asList( + TSDataType.STRING, + TSDataType.STRING, + TSDataType.DOUBLE, + TSDataType.STRING, + TSDataType.STRING, + TSDataType.DOUBLE), + Arrays.asList( + ColumnCategory.TAG, + ColumnCategory.ATTRIBUTE, + ColumnCategory.FIELD, + ColumnCategory.TAG, + ColumnCategory.ATTRIBUTE, + ColumnCategory.FIELD)); + tablet.addTimestamp(0, 0); + tablet.addValue(0, 0, "id1"); + tablet.addValue(0, 1, "attr1"); + tablet.addValue(0, 2, 0.1); + tablet.addValue(0, 3, "id2"); + tablet.addValue(0, 4, "attr2"); + tablet.addValue(0, 5, 0.2); + try { + sessionA.insert(tablet); + fail("Should have thrown an exception"); + } catch (StatementExecutionException e) { + assertEquals( + "301: [EXECUTE_STATEMENT_ERROR(301)] Exception occurred: insertTablet failed. Access Denied: No permissions for this operation, please add privilege ALTER ON test.table2", + e.getMessage()); + } + sessionRoot.executeNonQueryStatement("GRANT ALTER ON TABLE table2 TO USER userA"); + sessionA.insert(tablet); + sessionRoot.executeNonQueryStatement("REVOKE ALTER ON TABLE table2 FROM USER userA"); + sessionRoot.executeNonQueryStatement("REVOKE INSERT ON DATABASE test FROM USER userA"); + + // grant multiple and revoke one-by-one + sessionRoot.executeNonQueryStatement("GRANT INSERT ON ANY TO USER userA"); + sessionRoot.executeNonQueryStatement("GRANT INSERT ON DATABASE test TO USER userA"); + sessionRoot.executeNonQueryStatement("GRANT INSERT ON TABLE table1 TO USER userA"); + sessionA.executeNonQueryStatement( + "INSERT INTO table1 (time, id, attr, measurement) VALUES (1, 'id2', 'attr2', 0.2)"); + sessionRoot.executeNonQueryStatement("REVOKE INSERT ON ANY FROM USER userA"); + sessionA.executeNonQueryStatement( + "INSERT INTO table1 (time, id, attr, measurement) VALUES (1, 'id2', 'attr2', 0.2)"); + sessionRoot.executeNonQueryStatement("REVOKE INSERT ON DATABASE test FROM USER userA"); + sessionA.executeNonQueryStatement( + "INSERT INTO table1 (time, id, attr, measurement) VALUES (1, 'id2', 'attr2', 0.2)"); + sessionRoot.executeNonQueryStatement("REVOKE INSERT ON TABLE table1 FROM USER userA"); + try { + sessionA.executeNonQueryStatement( + "INSERT INTO table1 (time, id, attr, measurement) VALUES (1, 'id2', 'attr2', 0.2)"); + fail("Should have thrown an exception"); + } catch (StatementExecutionException e) { + assertEquals( + "803: Access Denied: No permissions for this operation, please add privilege INSERT ON test.table1", + e.getMessage()); + } + + // userA cannot revoke himself + sessionRoot.executeNonQueryStatement("GRANT INSERT ON ANY TO USER userA"); + try { + sessionA.executeNonQueryStatement("REVOKE INSERT ON ANY FROM USER userA"); + fail("Should have thrown an exception"); + } catch (StatementExecutionException e) { + assertEquals( + "803: Access Denied: No permissions for this operation, please add grant option to privilege INSERT", + e.getMessage()); + } + sessionRoot.executeNonQueryStatement("REVOKE INSERT ON ANY FROM USER userA"); + + // userA can revoke himself + sessionRoot.executeNonQueryStatement("GRANT INSERT ON ANY TO USER userA WITH GRANT OPTION"); + sessionA.executeNonQueryStatement("REVOKE INSERT ON ANY FROM USER userA"); + try { + sessionA.executeNonQueryStatement( + "INSERT INTO table1 (time, id, attr, measurement) VALUES (1, 'id2', 'attr2', 0.2)"); + fail("Should have thrown an exception"); + } catch (StatementExecutionException e) { + assertEquals( + "803: Access Denied: No permissions for this operation, please add privilege INSERT ON test.table1", + e.getMessage()); + } + // after revoked cannot revoke again + try { + sessionA.executeNonQueryStatement("REVOKE INSERT ON ANY FROM USER userA"); + fail("Should have thrown an exception"); + } catch (StatementExecutionException e) { + assertEquals( + "803: Access Denied: No permissions for this operation, please add grant option to privilege INSERT", + e.getMessage()); + } + + // userA cannot grant to userB + try { + sessionA.executeNonQueryStatement("GRANT INSERT ON ANY TO USER userB"); + fail("Should have thrown an exception"); + } catch (StatementExecutionException e) { + assertEquals( + "803: Access Denied: No permissions for this operation, please add grant option to privilege INSERT", + e.getMessage()); + } + sessionRoot.executeNonQueryStatement("GRANT INSERT ON ANY TO USER userA"); + try { + sessionA.executeNonQueryStatement("GRANT INSERT ON ANY TO USER userB"); + fail("Should have thrown an exception"); + } catch (StatementExecutionException e) { + assertEquals( + "803: Access Denied: No permissions for this operation, please add grant option to privilege INSERT", + e.getMessage()); + } + try { + sessionB.executeNonQueryStatement("REVOKE INSERT ON ANY FROM USER userA"); + fail("Should have thrown an exception"); + } catch (StatementExecutionException e) { + assertEquals( + "803: Access Denied: No permissions for this operation, please add grant option to privilege INSERT", + e.getMessage()); + } + + // userA can grant to userB + sessionRoot.executeNonQueryStatement("GRANT INSERT ON ANY TO USER userA WITH GRANT OPTION"); + sessionA.executeNonQueryStatement("GRANT INSERT ON ANY TO USER userB WITH GRANT OPTION"); + sessionB.executeNonQueryStatement( + "INSERT INTO table1 (time, id, attr, measurement) VALUES (1, 'id2', 'attr2', 0.2)"); + // userB can revoke userA + sessionB.executeNonQueryStatement("REVOKE INSERT ON ANY FROM USER userA"); + try { + sessionA.executeNonQueryStatement( + "INSERT INTO table1 (time, id, attr, measurement) VALUES (1, 'id2', 'attr2', 0.2)"); + fail("Should have thrown an exception"); + } catch (StatementExecutionException e) { + assertEquals( + "803: Access Denied: No permissions for this operation, please add privilege INSERT ON test.table1", + e.getMessage()); + } + // userB can revoke himself + sessionB.executeNonQueryStatement("REVOKE INSERT ON ANY FROM USER userB"); + try { + sessionB.executeNonQueryStatement("REVOKE INSERT ON ANY FROM USER userA"); + fail("Should have thrown an exception"); + } catch (StatementExecutionException e) { + assertEquals( + "803: Access Denied: No permissions for this operation, please add grant option to privilege INSERT", + e.getMessage()); + } + } + + // test role + sessionRoot.executeNonQueryStatement("CREATE USER userC 'userC'"); + sessionRoot.executeNonQueryStatement("CREATE USER userD 'userD'"); + sessionRoot.executeNonQueryStatement("CREATE ROLE role1"); + sessionRoot.executeNonQueryStatement("CREATE ROLE role2"); + sessionRoot.executeNonQueryStatement("GRANT ROLE role1 TO userC"); + sessionRoot.executeNonQueryStatement("GRANT ROLE role2 TO userD"); + + try (ITableSession sessionC = + EnvFactory.getEnv().getTableSessionConnection("userC", "userC"); + ITableSession sessionD = + EnvFactory.getEnv().getTableSessionConnection("userD", "userD")) { + // grant an irrelevant privilege so that the new users can use database + sessionRoot.executeNonQueryStatement("GRANT SELECT ON DATABASE test TO USER userC"); + sessionRoot.executeNonQueryStatement("GRANT SELECT ON DATABASE test TO USER userD"); + sessionC.executeNonQueryStatement("USE test"); + sessionD.executeNonQueryStatement("USE test"); + // userC no privilege + try { + sessionC.executeNonQueryStatement( + "INSERT INTO table1 (time, id, attr, measurement) VALUES (1, 'id2', 'attr2', 0.2)"); + fail("Should have thrown an exception"); + } catch (StatementExecutionException e) { + assertEquals( + "803: Access Denied: No permissions for this operation, please add privilege INSERT ON test.table1", + e.getMessage()); + } + + // grant and revoke - ALL + sessionRoot.executeNonQueryStatement("GRANT ALL TO ROLE role1"); + sessionC.executeNonQueryStatement( + "INSERT INTO table1 (time, id, attr, measurement) VALUES (1, 'id2', 'attr2', 0.2)"); + sessionRoot.executeNonQueryStatement("REVOKE ALL FROM ROLE role1"); + try { + sessionC.executeNonQueryStatement( + "INSERT INTO table1 (time, id, attr, measurement) VALUES (1, 'id2', 'attr2', 0.2)"); + fail("Should have thrown an exception"); + } catch (StatementExecutionException e) { + assertEquals( + "803: Access Denied: No permissions for this operation, please add privilege INSERT ON test.table1", + e.getMessage()); + } + + // grant and revoke - ANY + sessionRoot.executeNonQueryStatement("GRANT INSERT ON ANY TO ROLE role1"); + sessionC.executeNonQueryStatement( + "INSERT INTO table1 (time, id, attr, measurement) VALUES (1, 'id2', 'attr2', 0.2)"); + sessionRoot.executeNonQueryStatement("REVOKE INSERT ON ANY FROM ROLE role1"); + try { + sessionC.executeNonQueryStatement( + "INSERT INTO table1 (time, id, attr, measurement) VALUES (1, 'id2', 'attr2', 0.2)"); + fail("Should have thrown an exception"); + } catch (StatementExecutionException e) { + assertEquals( + "803: Access Denied: No permissions for this operation, please add privilege INSERT ON test.table1", + e.getMessage()); + } + + // grant and revoke - database + sessionRoot.executeNonQueryStatement("GRANT INSERT ON DATABASE test TO ROLE role1"); + sessionC.executeNonQueryStatement( + "INSERT INTO table1 (time, id, attr, measurement) VALUES (1, 'id2', 'attr2', 0.2)"); + sessionRoot.executeNonQueryStatement("REVOKE INSERT ON DATABASE test FROM ROLE role1"); + try { + sessionC.executeNonQueryStatement( + "INSERT INTO table1 (time, id, attr, measurement) VALUES (1, 'id2', 'attr2', 0.2)"); + fail("Should have thrown an exception"); + } catch (StatementExecutionException e) { + assertEquals( + "803: Access Denied: No permissions for this operation, please add privilege INSERT ON test.table1", + e.getMessage()); + } + + // grant and revoke - table + sessionRoot.executeNonQueryStatement("GRANT INSERT ON TABLE table1 TO ROLE role1"); + sessionC.executeNonQueryStatement( + "INSERT INTO table1 (time, id, attr, measurement) VALUES (1, 'id2', 'attr2', 0.2)"); + sessionRoot.executeNonQueryStatement("REVOKE INSERT ON TABLE table1 FROM ROLE role1"); + try { + sessionC.executeNonQueryStatement( + "INSERT INTO table1 (time, id, attr, measurement) VALUES (1, 'id2', 'attr2', 0.2)"); + fail("Should have thrown an exception"); + } catch (StatementExecutionException e) { + assertEquals( + "803: Access Denied: No permissions for this operation, please add privilege INSERT ON test.table1", + e.getMessage()); + } + + // grant multiple and revoke one-by-one + sessionRoot.executeNonQueryStatement("GRANT INSERT ON ANY TO ROLE role1"); + sessionRoot.executeNonQueryStatement("GRANT INSERT ON DATABASE test TO ROLE role1"); + sessionRoot.executeNonQueryStatement("GRANT INSERT ON TABLE table1 TO ROLE role1"); + sessionC.executeNonQueryStatement( + "INSERT INTO table1 (time, id, attr, measurement) VALUES (1, 'id2', 'attr2', 0.2)"); + sessionRoot.executeNonQueryStatement("REVOKE INSERT ON ANY FROM ROLE role1"); + sessionC.executeNonQueryStatement( + "INSERT INTO table1 (time, id, attr, measurement) VALUES (1, 'id2', 'attr2', 0.2)"); + sessionRoot.executeNonQueryStatement("REVOKE INSERT ON DATABASE test FROM ROLE role1"); + sessionC.executeNonQueryStatement( + "INSERT INTO table1 (time, id, attr, measurement) VALUES (1, 'id2', 'attr2', 0.2)"); + sessionRoot.executeNonQueryStatement("REVOKE INSERT ON TABLE table1 FROM ROLE role1"); + try { + sessionC.executeNonQueryStatement( + "INSERT INTO table1 (time, id, attr, measurement) VALUES (1, 'id2', 'attr2', 0.2)"); + fail("Should have thrown an exception"); + } catch (StatementExecutionException e) { + assertEquals( + "803: Access Denied: No permissions for this operation, please add privilege INSERT ON test.table1", + e.getMessage()); + } + + // role1 cannot revoke himself + sessionRoot.executeNonQueryStatement("GRANT INSERT ON ANY TO ROLE role1"); + try { + sessionC.executeNonQueryStatement("REVOKE INSERT ON ANY FROM ROLE role1"); + fail("Should have thrown an exception"); + } catch (StatementExecutionException e) { + assertEquals( + "803: Access Denied: No permissions for this operation, please add grant option to privilege INSERT", + e.getMessage()); + } + sessionRoot.executeNonQueryStatement("REVOKE INSERT ON ANY FROM ROLE role1"); + + // role1 can revoke himself + sessionRoot.executeNonQueryStatement("GRANT INSERT ON ANY TO ROLE role1 WITH GRANT OPTION"); + sessionC.executeNonQueryStatement("REVOKE INSERT ON ANY FROM ROLE role1"); + try { + sessionC.executeNonQueryStatement( + "INSERT INTO table1 (time, id, attr, measurement) VALUES (1, 'id2', 'attr2', 0.2)"); + fail("Should have thrown an exception"); + } catch (StatementExecutionException e) { + assertEquals( + "803: Access Denied: No permissions for this operation, please add privilege INSERT ON test.table1", + e.getMessage()); + } + // after revoked cannot revoke again + try { + sessionC.executeNonQueryStatement("REVOKE INSERT ON ANY FROM ROLE role1"); + fail("Should have thrown an exception"); + } catch (StatementExecutionException e) { + assertEquals( + "803: Access Denied: No permissions for this operation, please add grant option to privilege INSERT", + e.getMessage()); + } + + // role1 cannot grant to role2 + try { + sessionC.executeNonQueryStatement("GRANT INSERT ON ANY TO ROLE role2"); + fail("Should have thrown an exception"); + } catch (StatementExecutionException e) { + assertEquals( + "803: Access Denied: No permissions for this operation, please add grant option to privilege INSERT", + e.getMessage()); + } + sessionRoot.executeNonQueryStatement("GRANT INSERT ON ANY TO ROLE role1"); + try { + sessionC.executeNonQueryStatement("GRANT INSERT ON ANY TO ROLE role2"); + fail("Should have thrown an exception"); + } catch (StatementExecutionException e) { + assertEquals( + "803: Access Denied: No permissions for this operation, please add grant option to privilege INSERT", + e.getMessage()); + } + try { + sessionD.executeNonQueryStatement( + "INSERT INTO table1 (time, id, attr, measurement) VALUES (1, 'id2', 'attr2', 0.2)"); + fail("Should have thrown an exception"); + } catch (StatementExecutionException e) { + assertEquals( + "803: Access Denied: No permissions for this operation, please add privilege INSERT ON test.table1", + e.getMessage()); + } + + // userC can grant to userD + sessionRoot.executeNonQueryStatement("GRANT INSERT ON ANY TO ROLE role1 WITH GRANT OPTION"); + sessionC.executeNonQueryStatement("GRANT INSERT ON ANY TO ROLE role2 WITH GRANT OPTION"); + sessionD.executeNonQueryStatement( + "INSERT INTO table1 (time, id, attr, measurement) VALUES (1, 'id2', 'attr2', 0.2)"); + // userD can revoke userC + sessionD.executeNonQueryStatement("REVOKE INSERT ON ANY FROM ROLE role1"); + try { + sessionC.executeNonQueryStatement( + "INSERT INTO table1 (time, id, attr, measurement) VALUES (1, 'id2', 'attr2', 0.2)"); + fail("Should have thrown an exception"); + } catch (StatementExecutionException e) { + assertEquals( + "803: Access Denied: No permissions for this operation, please add privilege INSERT ON test.table1", + e.getMessage()); + } + // userD can revoke himself + sessionD.executeNonQueryStatement("REVOKE INSERT ON ANY FROM ROLE role2"); + try { + sessionD.executeNonQueryStatement("REVOKE INSERT ON ANY FROM ROLE role1"); + fail("Should have thrown an exception"); + } catch (StatementExecutionException e) { + assertEquals( + "803: Access Denied: No permissions for this operation, please add grant option to privilege INSERT", + e.getMessage()); + } + + // lose privilege after role is revoked + sessionRoot.executeNonQueryStatement("GRANT INSERT ON ANY TO ROLE role1"); + sessionRoot.executeNonQueryStatement("REVOKE ROLE role1 FROM userC"); + try { + sessionC.executeNonQueryStatement( + "INSERT INTO table1 (time, id, attr, measurement) VALUES (1, 'id2', 'attr2', 0.2)"); + fail("Should have thrown an exception"); + } catch (StatementExecutionException e) { + assertEquals( + "803: Access Denied: No permissions for this operation, please add privilege INSERT ON test.table1", + e.getMessage()); + } + } + } + } + + @Test + public void testDelete() throws IoTDBConnectionException, StatementExecutionException { + + try (ITableSession sessionRoot = EnvFactory.getEnv().getTableSessionConnection()) { + sessionRoot.executeNonQueryStatement("CREATE DATABASE test2"); + sessionRoot.executeNonQueryStatement("USE test2"); + + // insert by root + Tablet tablet = + new Tablet( + "table1", + Arrays.asList("id", "attr", "measurement"), + Arrays.asList(TSDataType.STRING, TSDataType.STRING, TSDataType.DOUBLE), + Arrays.asList(ColumnCategory.TAG, ColumnCategory.ATTRIBUTE, ColumnCategory.FIELD)); + tablet.addTimestamp(0, 0); + tablet.addValue(0, 0, "id1"); + tablet.addValue(0, 1, "attr1"); + tablet.addValue(0, 2, 0.1); + + sessionRoot.insert(tablet); + + sessionRoot.executeNonQueryStatement("DELETE FROM table1"); + + // revoke root + try { + sessionRoot.executeNonQueryStatement("REVOKE DELETE ON ANY FROM USER root"); + fail("Should have thrown an exception"); + } catch (StatementExecutionException e) { + assertEquals("803: Cannot grant/revoke privileges to/from admin", e.getMessage()); + } + try { + sessionRoot.executeNonQueryStatement("REVOKE DELETE ON DATABASE test FROM USER root"); + fail("Should have thrown an exception"); + } catch (StatementExecutionException e) { + assertEquals("803: Cannot grant/revoke privileges of admin user", e.getMessage()); + } + try { + sessionRoot.executeNonQueryStatement("REVOKE DELETE ON TABLE table1 FROM USER root"); + fail("Should have thrown an exception"); + } catch (StatementExecutionException e) { + assertEquals("803: Cannot grant/revoke privileges of admin user", e.getMessage()); + } + + // test users + sessionRoot.executeNonQueryStatement("CREATE USER userA 'userA'"); + sessionRoot.executeNonQueryStatement("CREATE USER userB 'userB'"); + + try (ITableSession sessionA = + EnvFactory.getEnv().getTableSessionConnection("userA", "userA"); + ITableSession sessionB = + EnvFactory.getEnv().getTableSessionConnection("userB", "userB")) { + // grant an irrelevant privilege so that the new users can use database + sessionRoot.executeNonQueryStatement("GRANT SELECT ON DATABASE test2 TO USER userA"); + sessionRoot.executeNonQueryStatement("GRANT SELECT ON DATABASE test2 TO USER userB"); + sessionA.executeNonQueryStatement("USE test2"); + sessionB.executeNonQueryStatement("USE test2"); + // userA no privilege + try { + sessionA.executeNonQueryStatement("DELETE FROM table1"); + fail("Should have thrown an exception"); + } catch (StatementExecutionException e) { + assertEquals( + "803: Access Denied: No permissions for this operation, please add privilege DELETE ON test2.table1", + e.getMessage()); + } + + // grant and revoke - ALL + sessionRoot.executeNonQueryStatement("GRANT ALL TO USER userA"); + sessionA.executeNonQueryStatement("DELETE FROM table1"); + sessionRoot.executeNonQueryStatement("REVOKE ALL FROM USER userA"); + try { + sessionA.executeNonQueryStatement("DELETE FROM table1"); + fail("Should have thrown an exception"); + } catch (StatementExecutionException e) { + assertEquals( + "803: Access Denied: No permissions for this operation, please add privilege DELETE ON test2.table1", + e.getMessage()); + } + + // grant and revoke - ANY + sessionRoot.executeNonQueryStatement("GRANT DELETE ON ANY TO USER userA"); + sessionA.executeNonQueryStatement("DELETE FROM table1"); + sessionRoot.executeNonQueryStatement("REVOKE DELETE ON ANY FROM USER userA"); + try { + sessionA.executeNonQueryStatement("DELETE FROM table1"); + fail("Should have thrown an exception"); + } catch (StatementExecutionException e) { + assertEquals( + "803: Access Denied: No permissions for this operation, please add privilege DELETE ON test2.table1", + e.getMessage()); + } + + // grant and revoke - database + sessionRoot.executeNonQueryStatement("GRANT DELETE ON DATABASE test2 TO USER userA"); + sessionA.executeNonQueryStatement("DELETE FROM table1"); + sessionRoot.executeNonQueryStatement("REVOKE DELETE ON DATABASE test2 FROM USER userA"); + try { + sessionA.executeNonQueryStatement("DELETE FROM table1"); + fail("Should have thrown an exception"); + } catch (StatementExecutionException e) { + assertEquals( + "803: Access Denied: No permissions for this operation, please add privilege DELETE ON test2.table1", + e.getMessage()); + } + + // grant and revoke - table + sessionRoot.executeNonQueryStatement("GRANT DELETE ON TABLE table1 TO USER userA"); + sessionA.executeNonQueryStatement("DELETE FROM table1"); + sessionRoot.executeNonQueryStatement("REVOKE DELETE ON TABLE table1 FROM USER userA"); + try { + sessionA.executeNonQueryStatement("DELETE FROM table1"); + fail("Should have thrown an exception"); + } catch (StatementExecutionException e) { + assertEquals( + "803: Access Denied: No permissions for this operation, please add privilege DELETE ON test2.table1", + e.getMessage()); + } + + // grant multiple and revoke one-by-one + sessionRoot.executeNonQueryStatement("GRANT DELETE ON ANY TO USER userA"); + sessionRoot.executeNonQueryStatement("GRANT DELETE ON DATABASE test2 TO USER userA"); + sessionRoot.executeNonQueryStatement("GRANT DELETE ON TABLE table1 TO USER userA"); + sessionA.executeNonQueryStatement("DELETE FROM table1"); + sessionRoot.executeNonQueryStatement("REVOKE DELETE ON ANY FROM USER userA"); + sessionA.executeNonQueryStatement("DELETE FROM table1"); + sessionRoot.executeNonQueryStatement("REVOKE DELETE ON DATABASE test2 FROM USER userA"); + sessionA.executeNonQueryStatement("DELETE FROM table1"); + sessionRoot.executeNonQueryStatement("REVOKE DELETE ON TABLE table1 FROM USER userA"); + try { + sessionA.executeNonQueryStatement("DELETE FROM table1"); + fail("Should have thrown an exception"); + } catch (StatementExecutionException e) { + assertEquals( + "803: Access Denied: No permissions for this operation, please add privilege DELETE ON test2.table1", + e.getMessage()); + } + + // userA cannot revoke himself + sessionRoot.executeNonQueryStatement("GRANT DELETE ON ANY TO USER userA"); + try { + sessionA.executeNonQueryStatement("REVOKE DELETE ON ANY FROM USER userA"); + fail("Should have thrown an exception"); + } catch (StatementExecutionException e) { + assertEquals( + "803: Access Denied: No permissions for this operation, please add grant option to privilege DELETE", + e.getMessage()); + } + sessionRoot.executeNonQueryStatement("REVOKE DELETE ON ANY FROM USER userA"); + + // userA can revoke himself + sessionRoot.executeNonQueryStatement("GRANT DELETE ON ANY TO USER userA WITH GRANT OPTION"); + sessionA.executeNonQueryStatement("REVOKE DELETE ON ANY FROM USER userA"); + try { + sessionA.executeNonQueryStatement("DELETE FROM table1"); + fail("Should have thrown an exception"); + } catch (StatementExecutionException e) { + assertEquals( + "803: Access Denied: No permissions for this operation, please add privilege DELETE ON test2.table1", + e.getMessage()); + } + // after revoked cannot revoke again + try { + sessionA.executeNonQueryStatement("REVOKE DELETE ON ANY FROM USER userA"); + fail("Should have thrown an exception"); + } catch (StatementExecutionException e) { + assertEquals( + "803: Access Denied: No permissions for this operation, please add grant option to privilege DELETE", + e.getMessage()); + } + + // userA cannot grant to userB + try { + sessionA.executeNonQueryStatement("GRANT DELETE ON ANY TO USER userB"); + fail("Should have thrown an exception"); + } catch (StatementExecutionException e) { + assertEquals( + "803: Access Denied: No permissions for this operation, please add grant option to privilege DELETE", + e.getMessage()); + } + sessionRoot.executeNonQueryStatement("GRANT DELETE ON ANY TO USER userA"); + try { + sessionA.executeNonQueryStatement("GRANT DELETE ON ANY TO USER userB"); + fail("Should have thrown an exception"); + } catch (StatementExecutionException e) { + assertEquals( + "803: Access Denied: No permissions for this operation, please add grant option to privilege DELETE", + e.getMessage()); + } + try { + sessionB.executeNonQueryStatement("REVOKE DELETE ON ANY FROM USER userA"); + fail("Should have thrown an exception"); + } catch (StatementExecutionException e) { + assertEquals( + "803: Access Denied: No permissions for this operation, please add grant option to privilege DELETE", + e.getMessage()); + } + + // userA can grant to userB + sessionRoot.executeNonQueryStatement("GRANT DELETE ON ANY TO USER userA WITH GRANT OPTION"); + sessionA.executeNonQueryStatement("GRANT DELETE ON ANY TO USER userB WITH GRANT OPTION"); + sessionB.executeNonQueryStatement("DELETE FROM table1"); + // userB can revoke userA + sessionB.executeNonQueryStatement("REVOKE DELETE ON ANY FROM USER userA"); + try { + sessionA.executeNonQueryStatement("DELETE FROM table1"); + fail("Should have thrown an exception"); + } catch (StatementExecutionException e) { + assertEquals( + "803: Access Denied: No permissions for this operation, please add privilege DELETE ON test2.table1", + e.getMessage()); + } + // userB can revoke himself + sessionB.executeNonQueryStatement("REVOKE DELETE ON ANY FROM USER userB"); + try { + sessionB.executeNonQueryStatement("REVOKE DELETE ON ANY FROM USER userA"); + fail("Should have thrown an exception"); + } catch (StatementExecutionException e) { + assertEquals( + "803: Access Denied: No permissions for this operation, please add grant option to privilege DELETE", + e.getMessage()); + } + } + + // test role + sessionRoot.executeNonQueryStatement("CREATE USER userC 'userC'"); + sessionRoot.executeNonQueryStatement("CREATE USER userD 'userD'"); + sessionRoot.executeNonQueryStatement("CREATE ROLE role1"); + sessionRoot.executeNonQueryStatement("CREATE ROLE role2"); + sessionRoot.executeNonQueryStatement("GRANT ROLE role1 TO userC"); + sessionRoot.executeNonQueryStatement("GRANT ROLE role2 TO userD"); + + try (ITableSession sessionC = + EnvFactory.getEnv().getTableSessionConnection("userC", "userC"); + ITableSession sessionD = + EnvFactory.getEnv().getTableSessionConnection("userD", "userD")) { + // grant an irrelevant privilege so that the new users can use database + sessionRoot.executeNonQueryStatement("GRANT SELECT ON DATABASE test2 TO USER userC"); + sessionRoot.executeNonQueryStatement("GRANT SELECT ON DATABASE test2 TO USER userD"); + sessionC.executeNonQueryStatement("USE test2"); + sessionD.executeNonQueryStatement("USE test2"); + // userC no privilege + try { + sessionC.executeNonQueryStatement("DELETE FROM table1"); + fail("Should have thrown an exception"); + } catch (StatementExecutionException e) { + assertEquals( + "803: Access Denied: No permissions for this operation, please add privilege DELETE ON test2.table1", + e.getMessage()); + } + + // grant and revoke - ALL + sessionRoot.executeNonQueryStatement("GRANT ALL TO ROLE role1"); + sessionC.executeNonQueryStatement("DELETE FROM table1"); + sessionRoot.executeNonQueryStatement("REVOKE ALL FROM ROLE role1"); + try { + sessionC.executeNonQueryStatement("DELETE FROM table1"); + fail("Should have thrown an exception"); + } catch (StatementExecutionException e) { + assertEquals( + "803: Access Denied: No permissions for this operation, please add privilege DELETE ON test2.table1", + e.getMessage()); + } + + // grant and revoke - ANY + sessionRoot.executeNonQueryStatement("GRANT DELETE ON ANY TO ROLE role1"); + sessionC.executeNonQueryStatement("DELETE FROM table1"); + sessionRoot.executeNonQueryStatement("REVOKE DELETE ON ANY FROM ROLE role1"); + try { + sessionC.executeNonQueryStatement("DELETE FROM table1"); + fail("Should have thrown an exception"); + } catch (StatementExecutionException e) { + assertEquals( + "803: Access Denied: No permissions for this operation, please add privilege DELETE ON test2.table1", + e.getMessage()); + } + + // grant and revoke - database + sessionRoot.executeNonQueryStatement("GRANT DELETE ON DATABASE test2 TO ROLE role1"); + sessionC.executeNonQueryStatement("DELETE FROM table1"); + sessionRoot.executeNonQueryStatement("REVOKE DELETE ON DATABASE test2 FROM ROLE role1"); + try { + sessionC.executeNonQueryStatement("DELETE FROM table1"); + fail("Should have thrown an exception"); + } catch (StatementExecutionException e) { + assertEquals( + "803: Access Denied: No permissions for this operation, please add privilege DELETE ON test2.table1", + e.getMessage()); + } + + // grant and revoke - table + sessionRoot.executeNonQueryStatement("GRANT DELETE ON TABLE table1 TO ROLE role1"); + sessionC.executeNonQueryStatement("DELETE FROM table1"); + sessionRoot.executeNonQueryStatement("REVOKE DELETE ON TABLE table1 FROM ROLE role1"); + try { + sessionC.executeNonQueryStatement("DELETE FROM table1"); + fail("Should have thrown an exception"); + } catch (StatementExecutionException e) { + assertEquals( + "803: Access Denied: No permissions for this operation, please add privilege DELETE ON test2.table1", + e.getMessage()); + } + + // grant multiple and revoke one-by-one + sessionRoot.executeNonQueryStatement("GRANT DELETE ON ANY TO ROLE role1"); + sessionRoot.executeNonQueryStatement("GRANT DELETE ON DATABASE test2 TO ROLE role1"); + sessionRoot.executeNonQueryStatement("GRANT DELETE ON TABLE table1 TO ROLE role1"); + sessionC.executeNonQueryStatement("DELETE FROM table1"); + sessionRoot.executeNonQueryStatement("REVOKE DELETE ON ANY FROM ROLE role1"); + sessionC.executeNonQueryStatement("DELETE FROM table1"); + sessionRoot.executeNonQueryStatement("REVOKE DELETE ON DATABASE test2 FROM ROLE role1"); + sessionC.executeNonQueryStatement("DELETE FROM table1"); + sessionRoot.executeNonQueryStatement("REVOKE DELETE ON TABLE table1 FROM ROLE role1"); + try { + sessionC.executeNonQueryStatement("DELETE FROM table1"); + fail("Should have thrown an exception"); + } catch (StatementExecutionException e) { + assertEquals( + "803: Access Denied: No permissions for this operation, please add privilege DELETE ON test2.table1", + e.getMessage()); + } + + // role1 cannot revoke himself + sessionRoot.executeNonQueryStatement("GRANT DELETE ON ANY TO ROLE role1"); + try { + sessionC.executeNonQueryStatement("REVOKE DELETE ON ANY FROM ROLE role1"); + fail("Should have thrown an exception"); + } catch (StatementExecutionException e) { + assertEquals( + "803: Access Denied: No permissions for this operation, please add grant option to privilege DELETE", + e.getMessage()); + } + sessionRoot.executeNonQueryStatement("REVOKE DELETE ON ANY FROM ROLE role1"); + + // role1 can revoke himself + sessionRoot.executeNonQueryStatement("GRANT DELETE ON ANY TO ROLE role1 WITH GRANT OPTION"); + sessionC.executeNonQueryStatement("REVOKE DELETE ON ANY FROM ROLE role1"); + try { + sessionC.executeNonQueryStatement("DELETE FROM table1"); + fail("Should have thrown an exception"); + } catch (StatementExecutionException e) { + assertEquals( + "803: Access Denied: No permissions for this operation, please add privilege DELETE ON test2.table1", + e.getMessage()); + } + // after revoked cannot revoke again + try { + sessionC.executeNonQueryStatement("REVOKE DELETE ON ANY FROM ROLE role1"); + fail("Should have thrown an exception"); + } catch (StatementExecutionException e) { + assertEquals( + "803: Access Denied: No permissions for this operation, please add grant option to privilege DELETE", + e.getMessage()); + } + + // role1 cannot grant to role2 + try { + sessionC.executeNonQueryStatement("GRANT DELETE ON ANY TO ROLE role2"); + fail("Should have thrown an exception"); + } catch (StatementExecutionException e) { + assertEquals( + "803: Access Denied: No permissions for this operation, please add grant option to privilege DELETE", + e.getMessage()); + } + sessionRoot.executeNonQueryStatement("GRANT DELETE ON ANY TO ROLE role1"); + try { + sessionC.executeNonQueryStatement("GRANT DELETE ON ANY TO ROLE role2"); + fail("Should have thrown an exception"); + } catch (StatementExecutionException e) { + assertEquals( + "803: Access Denied: No permissions for this operation, please add grant option to privilege DELETE", + e.getMessage()); + } + try { + sessionD.executeNonQueryStatement("DELETE FROM table1"); + fail("Should have thrown an exception"); + } catch (StatementExecutionException e) { + assertEquals( + "803: Access Denied: No permissions for this operation, please add privilege DELETE ON test2.table1", + e.getMessage()); + } + + // role1 can grant to role2 + sessionRoot.executeNonQueryStatement("GRANT DELETE ON ANY TO ROLE role1 WITH GRANT OPTION"); + sessionC.executeNonQueryStatement("GRANT DELETE ON ANY TO ROLE role2 WITH GRANT OPTION"); + sessionD.executeNonQueryStatement("DELETE FROM table1"); + // role2 can revoke role1 + sessionD.executeNonQueryStatement("REVOKE DELETE ON ANY FROM ROLE role1"); + try { + sessionC.executeNonQueryStatement("DELETE FROM table1"); + fail("Should have thrown an exception"); + } catch (StatementExecutionException e) { + assertEquals( + "803: Access Denied: No permissions for this operation, please add privilege DELETE ON test2.table1", + e.getMessage()); + } + // role2 can revoke himself + sessionD.executeNonQueryStatement("REVOKE DELETE ON ANY FROM ROLE role2"); + try { + sessionD.executeNonQueryStatement("REVOKE DELETE ON ANY FROM ROLE role1"); + fail("Should have thrown an exception"); + } catch (StatementExecutionException e) { + assertEquals( + "803: Access Denied: No permissions for this operation, please add grant option to privilege DELETE", + e.getMessage()); + } + + // lose privilege after role is revoked + sessionRoot.executeNonQueryStatement("GRANT DELETE ON ANY TO ROLE role1"); + sessionRoot.executeNonQueryStatement("REVOKE ROLE role1 FROM userC"); + try { + sessionC.executeNonQueryStatement("DELETE FROM table1"); + fail("Should have thrown an exception"); + } catch (StatementExecutionException e) { + assertEquals( + "803: Access Denied: No permissions for this operation, please add privilege DELETE ON test2.table1", + e.getMessage()); + } + } + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java index 95eee41d007a..ab4f64366143 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java @@ -24,8 +24,6 @@ import org.apache.iotdb.isession.SessionDataSet; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.ClusterIT; -import org.apache.iotdb.itbase.category.LocalStandaloneIT; import org.apache.iotdb.itbase.category.ManualIT; import org.apache.iotdb.itbase.category.TableClusterIT; import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; @@ -1647,7 +1645,6 @@ private List collectDataRanges(Statement statement, long timeUpperBou } @Test - @Category({LocalStandaloneIT.class, ClusterIT.class}) public void deleteTableOfTheSameNameTest() throws IoTDBConnectionException, StatementExecutionException { int testNum = 24; @@ -1776,6 +1773,47 @@ public void testConcurrentFlushAndRandomDeviceDeletion() } } + @Test + public void testCaseSensitivity() throws IoTDBConnectionException, StatementExecutionException { + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("CREATE DATABASE IF NOT EXISTS db1"); + session.executeNonQueryStatement("USE db1"); + session.executeNonQueryStatement("CREATE TABLE case_sensitivity (tag1 TAG, s1 INT32)"); + + session.executeNonQueryStatement( + "INSERT INTO case_sensitivity (time, tag1, s1) VALUES (1, 'd1', 1)"); + session.executeNonQueryStatement( + "INSERT INTO case_sensitivity (time, tag1, s1) VALUES (2, 'd2', 2)"); + session.executeNonQueryStatement( + "INSERT INTO case_sensitivity (time, tag1, s1) VALUES (3, 'd3', 3)"); + + session.executeNonQueryStatement("DELETE FROM DB1.case_sensitivity where time = 1"); + SessionDataSet dataSet = + session.executeQueryStatement("select * from db1.case_sensitivity order by time"); + RowRecord rec = dataSet.next(); + assertEquals(2, rec.getFields().get(0).getLongV()); + assertEquals("d2", rec.getFields().get(1).toString()); + assertEquals(2, rec.getFields().get(2).getIntV()); + rec = dataSet.next(); + assertEquals(3, rec.getFields().get(0).getLongV()); + assertEquals("d3", rec.getFields().get(1).toString()); + assertEquals(3, rec.getFields().get(2).getIntV()); + assertFalse(dataSet.hasNext()); + + session.executeNonQueryStatement("DELETE FROM db1.CASE_sensitivity where time = 2"); + dataSet = session.executeQueryStatement("select * from db1.case_sensitivity order by time"); + rec = dataSet.next(); + assertEquals(3, rec.getFields().get(0).getLongV()); + assertEquals("d3", rec.getFields().get(1).toString()); + assertEquals(3, rec.getFields().get(2).getIntV()); + assertFalse(dataSet.hasNext()); + + session.executeNonQueryStatement("DELETE FROM db1.CASE_sensitivity where TAG1 = 'd3'"); + dataSet = session.executeQueryStatement("select * from db1.case_sensitivity order by time"); + assertFalse(dataSet.hasNext()); + } + } + @Ignore("performance") @Test public void testDeletionWritePerformance() throws SQLException, IOException { diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBInsertTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBInsertTableIT.java index e751ab6423f2..af51ab3fedd2 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBInsertTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBInsertTableIT.java @@ -491,6 +491,17 @@ public void testInsertMultiRowWithWrongTimestampPrecision() throws SQLException assertTrue(e.getMessage().contains("Current system timestamp precision is ms")); } } + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement st1 = connection.createStatement()) { + try { + st1.execute("use \"test\""); + st1.execute( + "insert into wf16(tag1, time, status) values('wt01', -1618283005586000, true), ('wt01', -1618283005586001, false)"); + fail(); + } catch (SQLException e) { + assertTrue(e.getMessage().contains("Current system timestamp precision is ms")); + } + } } @Test diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLoadConfigurationTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLoadConfigurationTableIT.java new file mode 100644 index 000000000000..53e8bc9487e3 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLoadConfigurationTableIT.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.db.it; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.itbase.env.BaseEnv; + +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.nio.channels.FileChannel; +import java.nio.file.StandardOpenOption; +import java.sql.Connection; +import java.sql.Statement; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBLoadConfigurationTableIT { + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void loadConfiguration() throws IOException { + DataNodeWrapper dataNodeWrapper = EnvFactory.getEnv().getDataNodeWrapper(0); + String confPath = + dataNodeWrapper.getNodePath() + + File.separator + + "conf" + + File.separator + + "iotdb-system.properties"; + long length = new File(confPath).length(); + try (FileWriter fileWriter = new FileWriter(confPath, true)) { + fileWriter.write(System.lineSeparator()); + fileWriter.write("target_compaction_file_size=t"); + } + + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("LOAD CONFIGURATION"); + Assert.fail(); + } catch (Exception e) { + Assert.assertTrue(e.getMessage().contains("NumberFormatException")); + } finally { + try (FileChannel fileChannel = + FileChannel.open(new File(confPath).toPath(), StandardOpenOption.WRITE)) { + fileChannel.truncate(length); + } + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBSetSystemStatusTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBSetSystemStatusTableIT.java new file mode 100644 index 000000000000..5a3c435fb8b4 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBSetSystemStatusTableIT.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.db.it; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.itbase.env.BaseEnv; +import org.apache.iotdb.itbase.exception.InconsistentDataException; + +import org.awaitility.Awaitility; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.concurrent.TimeUnit; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBSetSystemStatusTableIT { + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void setSystemStatus() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("SET SYSTEM TO READONLY ON CLUSTER"); + Awaitility.await() + .atMost(10, TimeUnit.SECONDS) + .pollDelay(1, TimeUnit.SECONDS) + .until( + () -> { + ResultSet resultSet = statement.executeQuery("SHOW DATANODES"); + int num = 0; + try { + while (resultSet.next()) { + String status = resultSet.getString("Status"); + if (status.equals("ReadOnly")) { + num++; + } + } + } catch (InconsistentDataException e) { + return false; + } + return num == EnvFactory.getEnv().getDataNodeWrapperList().size(); + }); + + statement.execute("SET SYSTEM TO RUNNING ON CLUSTER"); + Awaitility.await() + .atMost(10, TimeUnit.SECONDS) + .pollDelay(1, TimeUnit.SECONDS) + .until( + () -> { + ResultSet resultSet = statement.executeQuery("SHOW DATANODES"); + int num = 0; + try { + while (resultSet.next()) { + String status = resultSet.getString("Status"); + if (status.equals("Running")) { + num++; + } + } + } catch (InconsistentDataException e) { + return false; + } + return num == EnvFactory.getEnv().getDataNodeWrapperList().size(); + }); + } catch (Exception e) { + Assert.fail(e.getMessage()); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/query/IoTDBGreatestLeastTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/query/IoTDBGreatestLeastTableIT.java new file mode 100644 index 000000000000..529b4daeabe8 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/query/IoTDBGreatestLeastTableIT.java @@ -0,0 +1,305 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.relational.it.query.old.query; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.itbase.env.BaseEnv; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.Statement; + +import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBGreatestLeastTableIT { + + private static final String DATABASE_NAME = "db"; + + private static final String[] SQLs = + new String[] { + "CREATE DATABASE " + DATABASE_NAME, + "USE " + DATABASE_NAME, + "CREATE TABLE boolean_table(device_id STRING TAG, bool1 BOOLEAN FIELD, bool2 BOOLEAN FIELD)", + "CREATE TABLE number_table(device_id STRING TAG, int1 INT32 FIELD, int2 INT32 FIELD, long1 INT64 FIELD, long2 INT64 FIELD, float1 FLOAT FIELD, float2 FLOAT FIELD, double1 DOUBLE FIELD, double2 DOUBLE FIELD)", + "CREATE TABLE string_table(device_id STRING TAG, string1 STRING FIELD, string2 STRING FIELD, text1 TEXT FIELD, text2 TEXT FIELD)", + "CREATE TABLE mix_type_table(device_id STRING TAG, s1 INT32 FIELD, s2 INT64 FIELD, s3 FLOAT FIELD, s4 DOUBLE FIELD, s5 BOOLEAN FIELD, s6 STRING FIELD, s7 TEXT FIELD)", + "CREATE TABLE null_table(device_id STRING TAG, string1 STRING FIELD, string2 STRING FIELD, int1 INT32 FIELD, int2 INT32 FIELD, double1 DOUBLE FIELD, double2 DOUBLE FIELD, timestamp1 TIMESTAMP FIELD, timestamp2 TIMESTAMP FIELD)", + "CREATE TABLE any_null_table(device_id STRING TAG, string1 STRING FIELD, string2 STRING FIELD, int1 INT32 FIELD, int2 INT32 FIELD, double1 DOUBLE FIELD, double2 DOUBLE FIELD, timestamp1 TIMESTAMP FIELD, timestamp2 TIMESTAMP FIELD)", + // normal case + "INSERT INTO number_table(time, device_id, int1, int2, long1, long2, float1, float2, double1, double2) VALUES (10, 'd1', 1000000, 2000000, 1000000, 2000000, 10.1, 20.2, 10.1, 20.2)", + "INSERT INTO string_table(time, device_id, string1, string2, text1, text2) VALUES(10, 'd1', 'aaa', 'bbb', 'aaa', 'bbb')", + "INSERT INTO boolean_table(time, device_id, bool1, bool2) VALUES(10, 'd1', true, false)", + "INSERT INTO mix_type_table(time, device_id, s1, s2, s3, s4, s5, s6, s7) VALUES(10, 'd1', 1, 1, 1.0, 1.0, true, 'a', 'a')", + "INSERT INTO null_table(time, device_id) VALUES(10, 'd1')", + "INSERT INTO any_null_table(time, device_id, string2, int2, double2, timestamp2) VALUES(10, 'd1', 'test', 10, 10.0, 10)", + }; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + insertData(); + } + + protected static void insertData() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + + for (String sql : SQLs) { + statement.execute(sql); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testNumberTypeGreatestFunction() { + + tableResultSetEqualTest( + "SELECT GREATEST(int1, int2) FROM number_table", + new String[] {"_col0"}, + new String[] {"2000000,"}, + DATABASE_NAME); + + tableResultSetEqualTest( + "SELECT GREATEST(long1, long2) FROM number_table", + new String[] {"_col0"}, + new String[] {"2000000,"}, + DATABASE_NAME); + + tableResultSetEqualTest( + "SELECT GREATEST(float1, float2) FROM number_table", + new String[] {"_col0"}, + new String[] {"20.2,"}, + DATABASE_NAME); + + tableResultSetEqualTest( + "SELECT GREATEST(double1, double2) FROM number_table", + new String[] {"_col0"}, + new String[] {"20.2,"}, + DATABASE_NAME); + } + + @Test + public void testNumberTypeLeastFunction() { + tableResultSetEqualTest( + "SELECT LEAST(int1, int2) FROM number_table", + new String[] {"_col0"}, + new String[] {"1000000,"}, + DATABASE_NAME); + + tableResultSetEqualTest( + "SELECT LEAST(long1, long2) FROM number_table", + new String[] {"_col0"}, + new String[] {"1000000,"}, + DATABASE_NAME); + + tableResultSetEqualTest( + "SELECT LEAST(float1, float2) FROM number_table", + new String[] {"_col0"}, + new String[] {"10.1,"}, + DATABASE_NAME); + + tableResultSetEqualTest( + "SELECT LEAST(double1, double2) FROM number_table", + new String[] {"_col0"}, + new String[] {"10.1,"}, + DATABASE_NAME); + } + + @Test + public void testStringTypeGreatestFunction() { + tableResultSetEqualTest( + "SELECT GREATEST(string1, string2) FROM string_table", + new String[] {"_col0"}, + new String[] {"bbb,"}, + DATABASE_NAME); + + tableResultSetEqualTest( + "SELECT GREATEST(text1, text2) FROM string_table", + new String[] {"_col0"}, + new String[] {"bbb,"}, + DATABASE_NAME); + } + + @Test + public void testStringTypeLeastFunction() { + tableResultSetEqualTest( + "SELECT LEAST(string1, string2) FROM string_table", + new String[] {"_col0"}, + new String[] {"aaa,"}, + DATABASE_NAME); + + tableResultSetEqualTest( + "SELECT LEAST(text1, text2) FROM string_table", + new String[] {"_col0"}, + new String[] {"aaa,"}, + DATABASE_NAME); + } + + @Test + public void testBooleanTypeGreatestFunction() { + tableResultSetEqualTest( + "SELECT GREATEST(bool1, bool2) FROM boolean_table", + new String[] {"_col0"}, + new String[] {"true,"}, + DATABASE_NAME); + } + + @Test + public void testBooleanTypeLeastFunction() { + tableResultSetEqualTest( + "SELECT LEAST(bool1, bool2) FROM boolean_table", + new String[] {"_col0"}, + new String[] {"false,"}, + DATABASE_NAME); + } + + @Test + public void testAllNullValue() { + tableResultSetEqualTest( + "SELECT GREATEST(string1, string2) FROM null_table", + new String[] {"_col0"}, + new String[] {"null,"}, + DATABASE_NAME); + + tableResultSetEqualTest( + "SELECT LEAST(string1, string2) FROM null_table", + new String[] {"_col0"}, + new String[] {"null,"}, + DATABASE_NAME); + + tableResultSetEqualTest( + "SELECT GREATEST(int1, int2) FROM null_table", + new String[] {"_col0"}, + new String[] {"null,"}, + DATABASE_NAME); + + tableResultSetEqualTest( + "SELECT LEAST(int1, int2) FROM null_table", + new String[] {"_col0"}, + new String[] {"null,"}, + DATABASE_NAME); + + tableResultSetEqualTest( + "SELECT GREATEST(double1, double2) FROM null_table", + new String[] {"_col0"}, + new String[] {"null,"}, + DATABASE_NAME); + + tableResultSetEqualTest( + "SELECT LEAST(double1, double2) FROM null_table", + new String[] {"_col0"}, + new String[] {"null,"}, + DATABASE_NAME); + + tableResultSetEqualTest( + "SELECT GREATEST(timestamp1, timestamp2) FROM null_table", + new String[] {"_col0"}, + new String[] {"null,"}, + DATABASE_NAME); + + tableResultSetEqualTest( + "SELECT LEAST(timestamp1, timestamp2) FROM null_table", + new String[] {"_col0"}, + new String[] {"null,"}, + DATABASE_NAME); + } + + @Test + public void testAnyNullValue() { + tableResultSetEqualTest( + "SELECT GREATEST(string1, string2) FROM any_null_table", + new String[] {"_col0"}, + new String[] {"test,"}, + DATABASE_NAME); + + tableResultSetEqualTest( + "SELECT LEAST(string1, string2) FROM any_null_table", + new String[] {"_col0"}, + new String[] {"test,"}, + DATABASE_NAME); + + tableResultSetEqualTest( + "SELECT GREATEST(int1, int2) FROM any_null_table", + new String[] {"_col0"}, + new String[] {"10,"}, + DATABASE_NAME); + + tableResultSetEqualTest( + "SELECT LEAST(int1, int2) FROM any_null_table", + new String[] {"_col0"}, + new String[] {"10,"}, + DATABASE_NAME); + + tableResultSetEqualTest( + "SELECT GREATEST(double1, double2) FROM any_null_table", + new String[] {"_col0"}, + new String[] {"10.0,"}, + DATABASE_NAME); + + tableResultSetEqualTest( + "SELECT LEAST(double1, double2) FROM any_null_table", + new String[] {"_col0"}, + new String[] {"10.0,"}, + DATABASE_NAME); + + tableResultSetEqualTest( + "SELECT GREATEST(timestamp1, timestamp2) FROM any_null_table", + new String[] {"_col0"}, + new String[] {"1970-01-01T00:00:00.010Z,"}, + DATABASE_NAME); + + tableResultSetEqualTest( + "SELECT LEAST(timestamp1, timestamp2) FROM any_null_table", + new String[] {"_col0"}, + new String[] {"1970-01-01T00:00:00.010Z,"}, + DATABASE_NAME); + } + + @Test + public void testAnomalies() { + // do not support different type + for (int i = 1; i <= 7; i++) { + for (int j = i + 1; j <= 7; j++) { + tableAssertTestFail( + String.format("SELECT LEAST(s%d, s%d) FROM mix_type_table", i, j), + "701: Scalar function least must have at least two arguments, and all type must be the same.", + DATABASE_NAME); + + tableAssertTestFail( + String.format("SELECT GREATEST(s%d, s%d) FROM mix_type_table", i, j), + "701: Scalar function greatest must have at least two arguments, and all type must be the same.", + DATABASE_NAME); + } + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBMaintainAuthIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBMaintainAuthIT.java new file mode 100644 index 000000000000..a67113c30e90 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBMaintainAuthIT.java @@ -0,0 +1,289 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.recent; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import static org.apache.iotdb.db.auth.AuthorityChecker.ONLY_ADMIN_ALLOWED; +import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; +import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; +import static org.apache.iotdb.db.it.utils.TestUtils.tableExecuteTest; +import static org.apache.iotdb.db.it.utils.TestUtils.tableQueryNoVerifyResultTest; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBMaintainAuthIT { + private static final String DATABASE_NAME = "test"; + private static final String CREATE_USER_FORMAT = "create user %s '%s'"; + private static final String USER_1 = "user1"; + private static final String USER_2 = "user2"; + private static final String PASSWORD = "password"; + + private static final String[] createSqls = + new String[] { + "CREATE DATABASE " + DATABASE_NAME, + "USE " + DATABASE_NAME, + "CREATE TABLE table1(device_id STRING TAG, s1 INT32 FIELD)", + "INSERT INTO table1(time,device_id,s1) values(1, 'd1', 1)", + String.format(CREATE_USER_FORMAT, USER_1, PASSWORD), + "GRANT MAINTAIN TO USER " + USER_1, + "GRANT SELECT ON TABLE table1 TO USER " + USER_1, + "GRANT SELECT ON information_schema.queries TO USER " + USER_1, + String.format(CREATE_USER_FORMAT, USER_2, PASSWORD) + }; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + prepareTableData(createSqls); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void maintainAuthTest() { + // case 1: explain + // user1 with select on table1 + String[] expectedHeader = new String[] {"distribution plan"}; + tableQueryNoVerifyResultTest( + "explain select * from test.table1", expectedHeader, USER_1, PASSWORD); + // user2 without select on table1 + tableAssertTestFail( + "explain select * from test.table1", + TSStatusCode.NO_PERMISSION.getStatusCode() + + ": Access Denied: No permissions for this operation, please add privilege SELECT ON test.table1", + USER_2, + PASSWORD); + + // case 2: explain analyze [verbose] + // user1 with select on table1 + expectedHeader = new String[] {"Explain Analyze"}; + tableQueryNoVerifyResultTest( + "explain analyze select * from test.table1", expectedHeader, USER_1, PASSWORD); + tableQueryNoVerifyResultTest( + "explain analyze verbose select * from test.table1", expectedHeader, USER_1, PASSWORD); + // user2 without select on table1 + tableAssertTestFail( + "explain analyze select * from test.table1", + TSStatusCode.NO_PERMISSION.getStatusCode() + + ": Access Denied: No permissions for this operation, please add privilege SELECT ON test.table1", + USER_2, + PASSWORD); + tableAssertTestFail( + "explain analyze verbose select * from test.table1", + TSStatusCode.NO_PERMISSION.getStatusCode() + + ": Access Denied: No permissions for this operation, please add privilege SELECT ON test.table1", + USER_2, + PASSWORD); + + // case 3: show current_sql_dialect + expectedHeader = new String[] {"CurrentSqlDialect"}; + tableQueryNoVerifyResultTest("SHOW CURRENT_SQL_DIALECT", expectedHeader, USER_2, PASSWORD); + + // case 4: show current_user + expectedHeader = new String[] {"CurrentUser"}; + tableQueryNoVerifyResultTest("SHOW CURRENT_USER", expectedHeader, USER_2, PASSWORD); + + // case 5: show version + expectedHeader = new String[] {"Version", "BuildInfo"}; + tableQueryNoVerifyResultTest("SHOW VERSION", expectedHeader, USER_2, PASSWORD); + + // case 6: show current_timestamp + expectedHeader = new String[] {"CurrentTimestamp"}; + tableQueryNoVerifyResultTest("SHOW CURRENT_TIMESTAMP", expectedHeader, USER_2, PASSWORD); + + // case 7: show variables + expectedHeader = new String[] {"Variable", "Value"}; + // user1 with MAINTAIN + tableQueryNoVerifyResultTest("SHOW VARIABLES", expectedHeader, USER_1, PASSWORD); + // user2 without MAINTAIN + tableAssertTestFail( + "SHOW VARIABLES", + TSStatusCode.NO_PERMISSION.getStatusCode() + + ": Access Denied: No permissions for this operation, please add privilege MAINTAIN", + USER_2, + PASSWORD); + + // case 8: show cluster_id + expectedHeader = new String[] {"ClusterId"}; + // user1 with MAINTAIN + tableQueryNoVerifyResultTest("SHOW CLUSTER_ID", expectedHeader, USER_1, PASSWORD); + // user2 without MAINTAIN + tableAssertTestFail( + "SHOW CLUSTER_ID", + TSStatusCode.NO_PERMISSION.getStatusCode() + + ": Access Denied: No permissions for this operation, please add privilege MAINTAIN", + USER_2, + PASSWORD); + + // case 9: flush + // user1 with MAINTAIN + tableExecuteTest("FLUSH", USER_1, PASSWORD); + // user2 without MAINTAIN + tableAssertTestFail( + "FLUSH", + TSStatusCode.NO_PERMISSION.getStatusCode() + + ": Access Denied: No permissions for this operation, please add privilege MAINTAIN", + USER_2, + PASSWORD); + + // case 10: clear cache + // user1 with MAINTAIN + tableExecuteTest("CLEAR CACHE", USER_1, PASSWORD); + // user2 without MAINTAIN + tableAssertTestFail( + "CLEAR CACHE", + TSStatusCode.NO_PERMISSION.getStatusCode() + + ": Access Denied: No permissions for this operation, please add privilege MAINTAIN", + USER_2, + PASSWORD); + + // case 11: set configuration + // user1 with MAINTAIN + tableExecuteTest("SET CONFIGURATION query_timeout_threshold='100000'", USER_1, PASSWORD); + // user2 without MAINTAIN + tableAssertTestFail( + "SET CONFIGURATION query_timeout_threshold='100000'", + TSStatusCode.NO_PERMISSION.getStatusCode() + + ": Access Denied: No permissions for this operation, please add privilege MAINTAIN", + USER_2, + PASSWORD); + + // case 12: show queries + // user1 with select on information_schema.queries + expectedHeader = + new String[] {"query_id", "start_time", "datanode_id", "elapsed_time", "statement", "user"}; + tableQueryNoVerifyResultTest("SHOW QUERIES", expectedHeader, USER_1, PASSWORD); + // user2 without select on information_schema.queries + tableAssertTestFail( + "SHOW QUERIES", + TSStatusCode.NO_PERMISSION.getStatusCode() + + ": Access Denied: No permissions for this operation, please add privilege SELECT ON information_schema.queries", + USER_2, + PASSWORD); + + // case 13: kill query + // user1 with MAINTAIN + tableAssertTestFail( + "kill query '20250206_093300_00001_1'", + TSStatusCode.NO_SUCH_QUERY.getStatusCode() + ": No such query", + USER_1, + PASSWORD); + // user2 without MAINTAIN + tableAssertTestFail( + "kill query '20250206_093300_00001_1'", + TSStatusCode.NO_PERMISSION.getStatusCode() + + ": Access Denied: No permissions for this operation, please add privilege MAINTAIN", + USER_2, + PASSWORD); + + // case 14: load configuration + // user1 with MAINTAIN + tableExecuteTest("LOAD CONFIGURATION", USER_1, PASSWORD); + // user2 without MAINTAIN + tableAssertTestFail( + "LOAD CONFIGURATION", + TSStatusCode.NO_PERMISSION.getStatusCode() + + ": Access Denied: No permissions for this operation, please add privilege MAINTAIN", + USER_2, + PASSWORD); + + // case 15: set system status + // user1 with MAINTAIN + tableExecuteTest("SET SYSTEM TO RUNNING", USER_1, PASSWORD); + // user2 without MAINTAIN + tableAssertTestFail( + "SET SYSTEM TO RUNNING", + TSStatusCode.NO_PERMISSION.getStatusCode() + + ": Access Denied: No permissions for this operation, please add privilege MAINTAIN", + USER_2, + PASSWORD); + + // case 16: start repair data + // user1 with MAINTAIN + tableExecuteTest("START REPAIR DATA", USER_1, PASSWORD); + // user2 without MAINTAIN + tableAssertTestFail( + "START REPAIR DATA", + TSStatusCode.NO_PERMISSION.getStatusCode() + + ": Access Denied: No permissions for this operation, please add privilege MAINTAIN", + USER_2, + PASSWORD); + + // case 17: stop repair data + // user1 with MAINTAIN + tableExecuteTest("STOP REPAIR DATA", USER_1, PASSWORD); + // user2 without MAINTAIN + tableAssertTestFail( + "STOP REPAIR DATA", + TSStatusCode.NO_PERMISSION.getStatusCode() + + ": Access Denied: No permissions for this operation, please add privilege MAINTAIN", + USER_2, + PASSWORD); + + // case 18: create function + // user1 with MAINTAIN + tableAssertTestFail( + "create function udsf as 'org.apache.iotdb.db.query.udf.example.relational.ContainNull'", + TSStatusCode.NO_PERMISSION.getStatusCode() + ": Access Denied: " + ONLY_ADMIN_ALLOWED, + USER_1, + PASSWORD); + // user2 without MAINTAIN + tableAssertTestFail( + "create function udsf as 'org.apache.iotdb.db.query.udf.example.relational.ContainNull'", + TSStatusCode.NO_PERMISSION.getStatusCode() + ": Access Denied: " + ONLY_ADMIN_ALLOWED, + USER_2, + PASSWORD); + + // case 19: show functions + // user1 with MAINTAIN + expectedHeader = new String[] {"FunctionName", "FunctionType", "ClassName(UDF)", "State"}; + tableQueryNoVerifyResultTest("SHOW FUNCTIONS", expectedHeader, USER_1, PASSWORD); + // user2 without MAINTAIN + tableQueryNoVerifyResultTest("SHOW FUNCTIONS", expectedHeader, USER_2, PASSWORD); + + // case 20: create function + // user1 with MAINTAIN + tableAssertTestFail( + "drop function udsf", + TSStatusCode.NO_PERMISSION.getStatusCode() + ": Access Denied: " + ONLY_ADMIN_ALLOWED, + USER_1, + PASSWORD); + // user2 without MAINTAIN + tableAssertTestFail( + "drop function udsf", + TSStatusCode.NO_PERMISSION.getStatusCode() + ": Access Denied: " + ONLY_ADMIN_ALLOWED, + USER_2, + PASSWORD); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBNullIdQueryIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBNullIdQueryIT.java index 6f067df19dff..eba15523aa23 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBNullIdQueryIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBNullIdQueryIT.java @@ -34,9 +34,11 @@ import java.sql.Connection; import java.sql.ResultSet; import java.sql.ResultSetMetaData; +import java.sql.SQLException; import java.sql.Statement; import java.sql.Types; +import static org.apache.iotdb.db.it.utils.TestUtils.createUser; import static org.apache.iotdb.db.it.utils.TestUtils.defaultFormatDataTime; import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; @@ -502,4 +504,35 @@ public void showStatementTest() { expectedHeader = new String[] {"CurrentTimestamp"}; tableResultSetFuzzyTest("show current_timestamp", expectedHeader, 1, DATABASE_NAME); } + + @Test + public void setSqlDialectTest() throws SQLException { + createUser("tempuser", "temppw"); + + try (Connection userCon = EnvFactory.getEnv().getConnection("tempuser", "temppw"); + Statement userStmt = userCon.createStatement()) { + assertCurrentSqlDialect(true, userStmt); + + // set Tree to Table + userStmt.execute("set sql_dialect=table"); + assertCurrentSqlDialect(false, userStmt); + + // set Table to Tree + userStmt.execute("set sql_dialect=tree"); + assertCurrentSqlDialect(true, userStmt); + } + } + + public static void assertCurrentSqlDialect(boolean expectedTree, Statement statement) + throws SQLException { + ResultSet resultSet = statement.executeQuery("show current_sql_dialect"); + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + assertEquals("CurrentSqlDialect", resultSetMetaData.getColumnName(1)); + int count = 0; + while (resultSet.next()) { + assertEquals(expectedTree ? "TREE" : "TABLE", resultSet.getString(1)); + count++; + } + assertEquals(1, count); + } } diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBNullValueIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBNullValueIT.java new file mode 100644 index 000000000000..531d88101ba6 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBNullValueIT.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.recent; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBNullValueIT { + private static final String DATABASE_NAME = "test"; + + private static final String[] createSqls = + new String[] { + "CREATE DATABASE " + DATABASE_NAME, + "USE " + DATABASE_NAME, + "create table table1(id1 tag, s1 string)", + "insert into table1 values(0, 'd1', null), (1,'d1', 1)", + "flush", + "insert into table1 values(0, 'd1', 0)", + "flush" + }; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().getConfig().getCommonConfig().setEnableCrossSpaceCompaction(false); + EnvFactory.getEnv().initClusterEnvironment(); + prepareTableData(createSqls); + } + + @AfterClass + public static void tearDown() { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void nullTest() { + + // case 1: all without time filter using previous fill without timeDuration + String[] expectedHeader = new String[] {"time", "id1", "s1"}; + String[] retArray = + new String[] { + "1970-01-01T00:00:00.000Z,d1,0,", "1970-01-01T00:00:00.001Z,d1,1,", + }; + tableResultSetEqualTest("select * from table1", expectedHeader, retArray, DATABASE_NAME); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBQueryAuthIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBQueryAuthIT.java new file mode 100644 index 000000000000..4557a66e7d14 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBQueryAuthIT.java @@ -0,0 +1,320 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.recent; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; +import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBQueryAuthIT { + + private static final String DATABASE_NAME = "test"; + private static final String CREATE_USER_FORMAT = "create user %s '%s'"; + private static final String USER_1 = "user1"; + private static final String USER_2 = "user2"; + private static final String USER_3 = "user3"; + private static final String USER_4 = "user4"; + private static final String USER_5 = "user5"; + private static final String USER_6 = "user6"; + private static final String USER_7 = "user7"; + private static final String USER_8 = "user8"; + private static final String USER_9 = "user9"; + private static final String PASSWORD = "password"; + + private static final String[] createSqls = + new String[] { + "CREATE DATABASE " + DATABASE_NAME, + "USE " + DATABASE_NAME, + "CREATE TABLE table1(device_id STRING TAG, s1 INT32 FIELD)", + "INSERT INTO table1(time,device_id,s1) values(1, 'd1', 1)", + "CREATE TABLE table2(device_id STRING TAG, s1 INT32 FIELD)", + "INSERT INTO table2(time,device_id,s1) values(1, 'd1', 1)", + String.format(CREATE_USER_FORMAT, USER_1, PASSWORD), + "GRANT SELECT ON ANY TO USER " + USER_1, + String.format(CREATE_USER_FORMAT, USER_2, PASSWORD), + "GRANT SELECT ON DATABASE " + DATABASE_NAME + " TO USER " + USER_2, + String.format(CREATE_USER_FORMAT, USER_3, PASSWORD), + "GRANT SELECT ON TABLE table1 TO USER " + USER_3, + String.format(CREATE_USER_FORMAT, USER_4, PASSWORD), + "GRANT DROP ON TABLE table2 TO USER " + USER_4, + String.format(CREATE_USER_FORMAT, USER_5, PASSWORD), + "GRANT INSERT ON TABLE table2 TO USER " + USER_5, + String.format(CREATE_USER_FORMAT, USER_6, PASSWORD), + "GRANT CREATE ON TABLE table2 TO USER " + USER_6, + String.format(CREATE_USER_FORMAT, USER_7, PASSWORD), + "GRANT ALTER ON TABLE table2 TO USER " + USER_7, + String.format(CREATE_USER_FORMAT, USER_8, PASSWORD), + "GRANT DELETE ON TABLE table2 TO USER " + USER_8, + String.format(CREATE_USER_FORMAT, USER_9, PASSWORD), + }; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + prepareTableData(createSqls); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void queryAuthTest() { + // case 1: user1 with SELECT ON ANY + String[] expectedHeader1 = new String[] {"time", "device_id", "s1"}; + String[] retArray1 = + new String[] { + "1970-01-01T00:00:00.001Z,d1,1,", + }; + String[] expectedHeader2 = new String[] {"device_id"}; + String[] retArray2 = new String[] {"d1,"}; + String[] expectedHeader3 = new String[] {"count(devices)"}; + String[] retArray3 = new String[] {"1,"}; + + tableResultSetEqualTest( + "select * from table1", expectedHeader1, retArray1, USER_1, PASSWORD, DATABASE_NAME); + tableResultSetEqualTest( + "show devices from table1", expectedHeader2, retArray2, USER_1, PASSWORD, DATABASE_NAME); + tableResultSetEqualTest( + "count devices from table1", expectedHeader3, retArray3, USER_1, PASSWORD, DATABASE_NAME); + + // case 2: user2 with SELECT ON database + tableResultSetEqualTest( + "select * from table1", expectedHeader1, retArray1, USER_2, PASSWORD, DATABASE_NAME); + tableResultSetEqualTest( + "show devices from table1", expectedHeader2, retArray2, USER_2, PASSWORD, DATABASE_NAME); + tableResultSetEqualTest( + "count devices from table1", expectedHeader3, retArray3, USER_2, PASSWORD, DATABASE_NAME); + + // case 3: user3 with SELECT ON table1 + tableResultSetEqualTest( + "select * from table1", expectedHeader1, retArray1, USER_3, PASSWORD, DATABASE_NAME); + tableResultSetEqualTest( + "show devices from table1", expectedHeader2, retArray2, USER_3, PASSWORD, DATABASE_NAME); + tableResultSetEqualTest( + "count devices from table1", expectedHeader3, retArray3, USER_3, PASSWORD, DATABASE_NAME); + + // case 4: user3 with SELECT ON table1, without SELECT ON table2 + tableAssertTestFail( + "select * from table2", + TSStatusCode.NO_PERMISSION.getStatusCode() + + ": Access Denied: No permissions for this operation, please add privilege SELECT ON test.table2", + USER_3, + PASSWORD, + DATABASE_NAME); + tableAssertTestFail( + "show devices from table2", + TSStatusCode.NO_PERMISSION.getStatusCode() + + ": Access Denied: No permissions for this operation, please add privilege SELECT ON test.table2", + USER_3, + PASSWORD, + DATABASE_NAME); + tableAssertTestFail( + "count devices from table2", + TSStatusCode.NO_PERMISSION.getStatusCode() + + ": Access Denied: No permissions for this operation, please add privilege SELECT ON test.table2", + USER_3, + PASSWORD, + DATABASE_NAME); + + // case 5: user4 with only DROP ON table2 + tableAssertTestFail( + "select * from table2", + TSStatusCode.NO_PERMISSION.getStatusCode() + + ": Access Denied: No permissions for this operation, please add privilege SELECT ON test.table2", + USER_4, + PASSWORD, + DATABASE_NAME); + tableAssertTestFail( + "show devices from table2", + TSStatusCode.NO_PERMISSION.getStatusCode() + + ": Access Denied: No permissions for this operation, please add privilege SELECT ON test.table2", + USER_4, + PASSWORD, + DATABASE_NAME); + tableAssertTestFail( + "count devices from table2", + TSStatusCode.NO_PERMISSION.getStatusCode() + + ": Access Denied: No permissions for this operation, please add privilege SELECT ON test.table2", + USER_4, + PASSWORD, + DATABASE_NAME); + + // case 6: user5 with only INSERT ON table2 + tableAssertTestFail( + "select * from table2", + TSStatusCode.NO_PERMISSION.getStatusCode() + + ": Access Denied: No permissions for this operation, please add privilege SELECT ON test.table2", + USER_5, + PASSWORD, + DATABASE_NAME); + tableAssertTestFail( + "show devices from table2", + TSStatusCode.NO_PERMISSION.getStatusCode() + + ": Access Denied: No permissions for this operation, please add privilege SELECT ON test.table2", + USER_5, + PASSWORD, + DATABASE_NAME); + tableAssertTestFail( + "count devices from table2", + TSStatusCode.NO_PERMISSION.getStatusCode() + + ": Access Denied: No permissions for this operation, please add privilege SELECT ON test.table2", + USER_5, + PASSWORD, + DATABASE_NAME); + + // case 7: user6 with only CREATE ON table2 + tableAssertTestFail( + "select * from table2", + TSStatusCode.NO_PERMISSION.getStatusCode() + + ": Access Denied: No permissions for this operation, please add privilege SELECT ON test.table2", + USER_6, + PASSWORD, + DATABASE_NAME); + tableAssertTestFail( + "show devices from table2", + TSStatusCode.NO_PERMISSION.getStatusCode() + + ": Access Denied: No permissions for this operation, please add privilege SELECT ON test.table2", + USER_6, + PASSWORD, + DATABASE_NAME); + tableAssertTestFail( + "count devices from table2", + TSStatusCode.NO_PERMISSION.getStatusCode() + + ": Access Denied: No permissions for this operation, please add privilege SELECT ON test.table2", + USER_6, + PASSWORD, + DATABASE_NAME); + + // case 8: user7 with only ALTER ON table2 + tableAssertTestFail( + "select * from table2", + TSStatusCode.NO_PERMISSION.getStatusCode() + + ": Access Denied: No permissions for this operation, please add privilege SELECT ON test.table2", + USER_7, + PASSWORD, + DATABASE_NAME); + tableAssertTestFail( + "show devices from table2", + TSStatusCode.NO_PERMISSION.getStatusCode() + + ": Access Denied: No permissions for this operation, please add privilege SELECT ON test.table2", + USER_7, + PASSWORD, + DATABASE_NAME); + tableAssertTestFail( + "count devices from table2", + TSStatusCode.NO_PERMISSION.getStatusCode() + + ": Access Denied: No permissions for this operation, please add privilege SELECT ON test.table2", + USER_7, + PASSWORD, + DATABASE_NAME); + + // case 9: user8 with only DELETE ON table2 + tableAssertTestFail( + "select * from table2", + TSStatusCode.NO_PERMISSION.getStatusCode() + + ": Access Denied: No permissions for this operation, please add privilege SELECT ON test.table2", + USER_8, + PASSWORD, + DATABASE_NAME); + tableAssertTestFail( + "show devices from table2", + TSStatusCode.NO_PERMISSION.getStatusCode() + + ": Access Denied: No permissions for this operation, please add privilege SELECT ON test.table2", + USER_8, + PASSWORD, + DATABASE_NAME); + tableAssertTestFail( + "count devices from table2", + TSStatusCode.NO_PERMISSION.getStatusCode() + + ": Access Denied: No permissions for this operation, please add privilege SELECT ON test.table2", + USER_8, + PASSWORD, + DATABASE_NAME); + + // case 10: user9 with nothing + tableAssertTestFail( + "select * from test.table2", + TSStatusCode.NO_PERMISSION.getStatusCode() + + ": Access Denied: No permissions for this operation, please add privilege SELECT ON test.table2", + USER_9, + PASSWORD); + tableAssertTestFail( + "show devices from test.table2", + TSStatusCode.NO_PERMISSION.getStatusCode() + + ": Access Denied: No permissions for this operation, please add privilege SELECT ON test.table2", + USER_9, + PASSWORD); + tableAssertTestFail( + "count devices from test.table2", + TSStatusCode.NO_PERMISSION.getStatusCode() + + ": Access Denied: No permissions for this operation, please add privilege SELECT ON test.table2", + USER_9, + PASSWORD); + + // case 11: user1 with SELECT ON ANY + String[] expectedHeader4 = new String[] {"time", "device_id", "table1_s1", "table2_s1"}; + String[] retArray4 = + new String[] { + "1970-01-01T00:00:00.001Z,d1,1,1,", + }; + + tableResultSetEqualTest( + "select table1.time as time, table1.device_id as device_id, table1.s1 as table1_s1, table2.s1 as table2_s1 from table1 inner join table2 on table1.time=table2.time and table1.device_id=table2.device_id", + expectedHeader4, + retArray4, + USER_1, + PASSWORD, + DATABASE_NAME); + + // case 12: user2 with SELECT ON database + tableResultSetEqualTest( + "select table1.time as time, table1.device_id as device_id, table1.s1 as table1_s1, table2.s1 as table2_s1 from table1 inner join table2 on table1.time=table2.time and table1.device_id=table2.device_id", + expectedHeader4, + retArray4, + USER_2, + PASSWORD, + DATABASE_NAME); + + // case 3: user3 with SELECT ON just table1 + tableAssertTestFail( + "select table1.time as time, table1.device_id as device_id, table1.s1 as table1_s1, table2.s1 as table2_s1 from table1 inner join table2 on table1.time=table2.time and table1.device_id=table2.device_id", + TSStatusCode.NO_PERMISSION.getStatusCode() + + ": Access Denied: No permissions for this operation, please add privilege SELECT ON test.table2", + USER_3, + PASSWORD, + DATABASE_NAME); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDatabaseIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDatabaseIT.java index 160495e40fb7..219088b74f06 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDatabaseIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDatabaseIT.java @@ -27,6 +27,7 @@ import org.apache.iotdb.itbase.env.BaseEnv; import org.junit.After; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -362,7 +363,16 @@ public void testInformationSchema() throws SQLException { statement.executeQuery("show tables"), "TableName,TTL(ms),", new HashSet<>( - Arrays.asList("databases,INF,", "tables,INF,", "columns,INF,", "queries,INF,"))); + Arrays.asList( + "databases,INF,", + "tables,INF,", + "columns,INF,", + "queries,INF,", + "regions,INF,", + "topics,INF,", + "pipe_plugins,INF,", + "pipes,INF,", + "subscriptions,INF,"))); TestUtils.assertResultSetEqual( statement.executeQuery("desc databases"), @@ -405,7 +415,44 @@ public void testInformationSchema() throws SQLException { "start_time,TIMESTAMP,ATTRIBUTE,", "datanode_id,INT32,ATTRIBUTE,", "elapsed_time,FLOAT,ATTRIBUTE,", - "statement,STRING,ATTRIBUTE,"))); + "statement,STRING,ATTRIBUTE,", + "user,STRING,ATTRIBUTE,"))); + TestUtils.assertResultSetEqual( + statement.executeQuery("desc pipes"), + "ColumnName,DataType,Category,", + new HashSet<>( + Arrays.asList( + "id,STRING,TAG,", + "creation_time,TIMESTAMP,ATTRIBUTE,", + "state,STRING,ATTRIBUTE,", + "pipe_source,STRING,ATTRIBUTE,", + "pipe_processor,STRING,ATTRIBUTE,", + "pipe_sink,STRING,ATTRIBUTE,", + "exception_message,STRING,ATTRIBUTE,", + "remaining_event_count,INT64,ATTRIBUTE,", + "estimated_remaining_seconds,DOUBLE,ATTRIBUTE,"))); + TestUtils.assertResultSetEqual( + statement.executeQuery("desc pipe_plugins"), + "ColumnName,DataType,Category,", + new HashSet<>( + Arrays.asList( + "plugin_name,STRING,TAG,", + "plugin_type,STRING,ATTRIBUTE,", + "class_name,STRING,ATTRIBUTE,", + "plugin_jar,STRING,ATTRIBUTE,"))); + TestUtils.assertResultSetEqual( + statement.executeQuery("desc topics"), + "ColumnName,DataType,Category,", + new HashSet<>( + Arrays.asList("topic_name,STRING,TAG,", "topic_configs,STRING,ATTRIBUTE,"))); + TestUtils.assertResultSetEqual( + statement.executeQuery("desc subscriptions"), + "ColumnName,DataType,Category,", + new HashSet<>( + Arrays.asList( + "topic_name,STRING,TAG,", + "consumer_group_name,STRING,TAG,", + "subscribed_consumers,STRING,ATTRIBUTE,"))); // Test table query statement.execute("create database test"); @@ -427,11 +474,16 @@ public void testInformationSchema() throws SQLException { "information_schema,tables,INF,USING,", "information_schema,columns,INF,USING,", "information_schema,queries,INF,USING,", + "information_schema,regions,INF,USING,", + "information_schema,topics,INF,USING,", + "information_schema,pipe_plugins,INF,USING,", + "information_schema,pipes,INF,USING,", + "information_schema,subscriptions,INF,USING,", "test,test,INF,USING,"))); TestUtils.assertResultSetEqual( statement.executeQuery("count devices from tables where status = 'USING'"), "count(devices),", - Collections.singleton("5,")); + Collections.singleton("10,")); TestUtils.assertResultSetEqual( statement.executeQuery( "select * from columns where table_name = 'queries' or database = 'test'"), @@ -443,10 +495,36 @@ public void testInformationSchema() throws SQLException { "information_schema,queries,datanode_id,INT32,ATTRIBUTE,USING,", "information_schema,queries,elapsed_time,FLOAT,ATTRIBUTE,USING,", "information_schema,queries,statement,STRING,ATTRIBUTE,USING,", + "information_schema,queries,user,STRING,ATTRIBUTE,USING,", "test,test,time,TIMESTAMP,TIME,USING,", "test,test,a,STRING,TAG,USING,", "test,test,b,STRING,ATTRIBUTE,USING,", "test,test,c,INT32,FIELD,USING,"))); + + statement.execute( + "create pipe a2b with source('double-living'='true') with sink ('sink'='write-back-sink')"); + TestUtils.assertResultSetEqual( + statement.executeQuery("select id, pipe_sink from pipes where creation_time > 0"), + "id,pipe_sink,", + Collections.singleton("a2b,{sink=write-back-sink},")); + TestUtils.assertResultSetEqual( + statement.executeQuery("select * from pipe_plugins"), + "plugin_name,plugin_type,class_name,plugin_jar,", + new HashSet<>( + Arrays.asList( + "IOTDB-THRIFT-SSL-SINK,Builtin,org.apache.iotdb.commons.pipe.agent.plugin.builtin.connector.iotdb.thrift.IoTDBThriftSslConnector,null,", + "IOTDB-AIR-GAP-SINK,Builtin,org.apache.iotdb.commons.pipe.agent.plugin.builtin.connector.iotdb.airgap.IoTDBAirGapConnector,null,", + "DO-NOTHING-SINK,Builtin,org.apache.iotdb.commons.pipe.agent.plugin.builtin.connector.donothing.DoNothingConnector,null,", + "DO-NOTHING-PROCESSOR,Builtin,org.apache.iotdb.commons.pipe.agent.plugin.builtin.processor.donothing.DoNothingProcessor,null,", + "IOTDB-THRIFT-SINK,Builtin,org.apache.iotdb.commons.pipe.agent.plugin.builtin.connector.iotdb.thrift.IoTDBThriftConnector,null,", + "IOTDB-SOURCE,Builtin,org.apache.iotdb.commons.pipe.agent.plugin.builtin.extractor.iotdb.IoTDBExtractor,null,"))); + + statement.execute("create topic tp with ('start-time'='2025-01-13T10:03:19.229+08:00')"); + TestUtils.assertResultSetEqual( + statement.executeQuery("select * from topics where topic_name = 'tp'"), + "topic_name,topic_configs,", + Collections.singleton( + "tp,{__system.sql-dialect=table, start-time=2025-01-13T10:03:19.229+08:00},")); } } @@ -508,4 +586,65 @@ public void testMixedDatabase() throws SQLException { TestUtils.assertResultSetSize(statement.executeQuery("show databases"), 1); } } + + @Test + public void testDBAuth() throws SQLException { + try (final Connection adminCon = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement adminStmt = adminCon.createStatement()) { + adminStmt.execute("create user test 'password'"); + adminStmt.execute("create database db"); + } + + try (final Connection userCon = + EnvFactory.getEnv().getConnection("test", "password", BaseEnv.TABLE_SQL_DIALECT); + final Statement userStmt = userCon.createStatement()) { + TestUtils.assertResultSetEqual( + userStmt.executeQuery("show databases"), + "Database,TTL(ms),SchemaReplicationFactor,DataReplicationFactor,TimePartitionInterval,", + Collections.emptySet()); + } + + try (final Connection adminCon = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement adminStmt = adminCon.createStatement()) { + adminStmt.execute("GRANT SELECT ON DATABASE DB to user test"); + } + + try (final Connection userCon = + EnvFactory.getEnv().getConnection("test", "password", BaseEnv.TABLE_SQL_DIALECT); + final Statement userStmt = userCon.createStatement()) { + try (final ResultSet resultSet = userStmt.executeQuery("SHOW DATABASES")) { + final ResultSetMetaData metaData = resultSet.getMetaData(); + assertEquals(showDBColumnHeaders.size(), metaData.getColumnCount()); + for (int i = 0; i < showDBColumnHeaders.size(); i++) { + assertEquals(showDBColumnHeaders.get(i).getColumnName(), metaData.getColumnName(i + 1)); + } + Assert.assertTrue(resultSet.next()); + assertEquals("db", resultSet.getString(1)); + Assert.assertFalse(resultSet.next()); + } + + Assert.assertThrows( + SQLException.class, + () -> { + userStmt.execute("alter database db set properties ttl=6600000"); + }); + + Assert.assertThrows( + SQLException.class, + () -> { + userStmt.execute("drop database db"); + }); + } + + try (final Connection adminCon = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement adminStmt = adminCon.createStatement()) { + adminStmt.execute("GRANT DROP ON ANY to user test"); + } + + try (final Connection userCon = + EnvFactory.getEnv().getConnection("test", "password", BaseEnv.TABLE_SQL_DIALECT); + final Statement userStmt = userCon.createStatement()) { + userStmt.execute("drop database db"); + } + } } diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java index d27e298f8776..901917743102 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java @@ -27,6 +27,7 @@ import org.apache.iotdb.itbase.env.BaseEnv; import org.junit.AfterClass; +import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -41,6 +42,7 @@ import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.describeTableColumnHeaders; import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.describeTableDetailsColumnHeaders; +import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.showDBColumnHeaders; import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.showTablesColumnHeaders; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -560,4 +562,59 @@ public void testManageTable() { fail(e.getMessage()); } } + + @Test + public void testTableAuth() throws SQLException { + try (final Connection adminCon = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement adminStmt = adminCon.createStatement()) { + adminStmt.execute("create user test 'password'"); + adminStmt.execute("create database db"); + adminStmt.execute("use db"); + adminStmt.execute("create table test (a tag, b attribute, c int32)"); + } + + try (final Connection userCon = + EnvFactory.getEnv().getConnection("test", "password", BaseEnv.TABLE_SQL_DIALECT); + final Statement userStmt = userCon.createStatement()) { + Assert.assertThrows( + SQLException.class, + () -> { + userStmt.execute("select * from db.test"); + }); + } + + try (final Connection adminCon = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement adminStmt = adminCon.createStatement()) { + adminStmt.execute("GRANT SELECT ON db.test to user test"); + } + + try (final Connection userCon = + EnvFactory.getEnv().getConnection("test", "password", BaseEnv.TABLE_SQL_DIALECT); + final Statement userStmt = userCon.createStatement()) { + try (final ResultSet resultSet = userStmt.executeQuery("SHOW DATABASES")) { + final ResultSetMetaData metaData = resultSet.getMetaData(); + assertEquals(showDBColumnHeaders.size(), metaData.getColumnCount()); + for (int i = 0; i < showDBColumnHeaders.size(); i++) { + assertEquals(showDBColumnHeaders.get(i).getColumnName(), metaData.getColumnName(i + 1)); + } + Assert.assertTrue(resultSet.next()); + assertEquals("db", resultSet.getString(1)); + Assert.assertFalse(resultSet.next()); + } + + userStmt.execute("select * from db.test"); + } + + try (final Connection adminCon = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement adminStmt = adminCon.createStatement()) { + adminStmt.execute("GRANT DROP ON DATABASE DB to user test"); + } + + try (final Connection userCon = + EnvFactory.getEnv().getConnection("test", "password", BaseEnv.TABLE_SQL_DIALECT); + final Statement userStmt = userCon.createStatement()) { + userStmt.execute("use db"); + userStmt.execute("drop table test"); + } + } } diff --git a/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionInsertNullIT.java b/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionInsertNullIT.java index bbee36782fc4..4eb14e35c939 100644 --- a/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionInsertNullIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionInsertNullIT.java @@ -35,6 +35,7 @@ import org.apache.tsfile.write.record.Tablet; import org.apache.tsfile.write.schema.MeasurementSchema; import org.junit.After; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -425,4 +426,61 @@ public void insertAlignedTabletNullTest() { fail(e.getMessage()); } } + + @Test + public void insertTabletNullMeasurementTest() { + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + String deviceId = "root.sg1.clsu.aligned_d1"; + Tablet tablet = + new Tablet( + deviceId, + Arrays.asList( + new MeasurementSchema("s1", TSDataType.BOOLEAN), + new MeasurementSchema(null, TSDataType.INT32)), + 1); + tablet.addTimestamp(0, 300); + tablet.addValue("s1", 0, true); + tablet.addValue(null, 0, 1); + session.insertAlignedTablet(tablet); + fail(); + } catch (Exception e) { + Assert.assertEquals("measurement should be non null value", e.getMessage()); + } + + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + String deviceId = "root.sg1.clsu.aligned_d1"; + Tablet tablet = + new Tablet( + deviceId, + Arrays.asList( + new MeasurementSchema("s1", TSDataType.BOOLEAN), + new MeasurementSchema(null, TSDataType.INT32)), + 1); + tablet.addTimestamp(0, 300); + tablet.addValue(0, 0, true); + tablet.addValue(0, 1, 1); + session.insertAlignedTablet(tablet); + fail(); + } catch (Exception e) { + Assert.assertEquals("measurement should be non null value", e.getMessage()); + } + + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + String deviceId = "root.sg1.clsu.aligned_d1"; + Tablet tablet = + new Tablet( + deviceId, + Arrays.asList( + new MeasurementSchema("s1", TSDataType.BOOLEAN), + new MeasurementSchema(null, TSDataType.INT32)), + 1); + tablet.addTimestamp(0, 300); + tablet.addValue("s1", 0, true); + // doesn't insert 2nd measurement + session.insertAlignedTablet(tablet); + fail(); + } catch (Exception e) { + Assert.assertEquals("measurement should be non null value", e.getMessage()); + } + } } diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/IoTDBSubscriptionBasicIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/IoTDBSubscriptionBasicIT.java index be478548c311..93ed3d484065 100644 --- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/IoTDBSubscriptionBasicIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/IoTDBSubscriptionBasicIT.java @@ -19,10 +19,12 @@ package org.apache.iotdb.subscription.it.local; +import org.apache.iotdb.commons.auth.entity.PrivilegeType; import org.apache.iotdb.isession.ISession; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.LocalStandaloneIT; +import org.apache.iotdb.itbase.env.BaseEnv; import org.apache.iotdb.rpc.subscription.config.TopicConstant; import org.apache.iotdb.session.subscription.SubscriptionTreeSession; import org.apache.iotdb.session.subscription.consumer.AckStrategy; @@ -48,6 +50,7 @@ import java.io.IOException; import java.time.Duration; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Properties; @@ -57,6 +60,12 @@ import java.util.concurrent.locks.LockSupport; import java.util.stream.Collectors; +import static org.apache.iotdb.db.it.utils.TestUtils.assertTableNonQueryTestFail; +import static org.apache.iotdb.db.it.utils.TestUtils.assertTableTestFail; +import static org.apache.iotdb.db.it.utils.TestUtils.createUser; +import static org.apache.iotdb.db.it.utils.TestUtils.executeNonQueryWithRetry; +import static org.apache.iotdb.db.it.utils.TestUtils.executeQueriesWithRetry; +import static org.apache.iotdb.db.it.utils.TestUtils.grantUserSystemPrivileges; import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; import static org.junit.Assert.fail; @@ -621,4 +630,62 @@ public void testMissingConsumerId() { fail(e.getMessage()); } } + + @Test + public void testTablePermission() { + createUser(EnvFactory.getEnv(), "test", "test123"); + + assertTableNonQueryTestFail( + EnvFactory.getEnv(), + "create topic topic1", + "803: Access Denied: No permissions for this operation, please add privilege MAINTAIN", + "test", + "test123", + null); + assertTableTestFail( + EnvFactory.getEnv(), + "show topics", + "803: Access Denied: No permissions for this operation, please add privilege MAINTAIN", + "test", + "test123", + null); + assertTableTestFail( + EnvFactory.getEnv(), + "show subscriptions", + "803: Access Denied: No permissions for this operation, please add privilege MAINTAIN", + "test", + "test123", + null); + assertTableNonQueryTestFail( + EnvFactory.getEnv(), + "drop topic topic1", + "803: Access Denied: No permissions for this operation, please add privilege MAINTAIN", + "test", + "test123", + null); + + grantUserSystemPrivileges(EnvFactory.getEnv(), "test", PrivilegeType.MAINTAIN); + + executeNonQueryWithRetry( + EnvFactory.getEnv(), + "create topic topic1", + "test", + "test123", + null, + BaseEnv.TABLE_SQL_DIALECT); + executeQueriesWithRetry( + EnvFactory.getEnv(), + Arrays.asList("show topics", "show subscriptions"), + "test", + "test123", + null, + BaseEnv.TABLE_SQL_DIALECT); + executeNonQueryWithRetry( + EnvFactory.getEnv(), + "drop topic topic1", + "test", + "test123", + null, + BaseEnv.TABLE_SQL_DIALECT); + } } diff --git a/iotdb-api/external-api/pom.xml b/iotdb-api/external-api/pom.xml index 18e171c3aec2..3622d5482147 100644 --- a/iotdb-api/external-api/pom.xml +++ b/iotdb-api/external-api/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-api - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT external-api IoTDB: API: External API diff --git a/iotdb-api/pipe-api/pom.xml b/iotdb-api/pipe-api/pom.xml index b0368594e5e3..e587605019ba 100644 --- a/iotdb-api/pipe-api/pom.xml +++ b/iotdb-api/pipe-api/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-api - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT pipe-api IoTDB: API: Pipe API diff --git a/iotdb-api/pom.xml b/iotdb-api/pom.xml index 01aba4586534..82c7d055d5b0 100644 --- a/iotdb-api/pom.xml +++ b/iotdb-api/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-parent - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT iotdb-api pom diff --git a/iotdb-api/trigger-api/pom.xml b/iotdb-api/trigger-api/pom.xml index 64560a259bb3..22419d80ea17 100644 --- a/iotdb-api/trigger-api/pom.xml +++ b/iotdb-api/trigger-api/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-api - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT trigger-api IoTDB: API: Trigger API diff --git a/iotdb-api/udf-api/pom.xml b/iotdb-api/udf-api/pom.xml index 8a66274237ff..72355686168a 100644 --- a/iotdb-api/udf-api/pom.xml +++ b/iotdb-api/udf-api/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-api - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT udf-api IoTDB: API: UDF API diff --git a/iotdb-client/cli/pom.xml b/iotdb-client/cli/pom.xml index a735626e9eaf..600684bc4ec7 100644 --- a/iotdb-client/cli/pom.xml +++ b/iotdb-client/cli/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-client - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT iotdb-cli IoTDB: Client: CLI @@ -37,37 +37,37 @@ org.apache.iotdb iotdb-session - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.apache.iotdb iotdb-jdbc - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.apache.iotdb iotdb-antlr - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.apache.iotdb node-commons - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.apache.iotdb iotdb-server - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.apache.iotdb isession - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.apache.iotdb service-rpc - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.apache.tsfile @@ -82,17 +82,17 @@ org.apache.iotdb iotdb-thrift - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.apache.iotdb iotdb-thrift-commons - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.apache.iotdb pipe-api - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.slf4j diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/common/ImportTsFileOperation.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/common/ImportTsFileOperation.java new file mode 100644 index 000000000000..3769d855d161 --- /dev/null +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/common/ImportTsFileOperation.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.tool.common; + +import org.apache.iotdb.cli.utils.IoTPrinter; + +public enum ImportTsFileOperation { + NONE, + MV, + HARDLINK, + CP, + DELETE, + ; + + public static boolean isValidOperation(String operation) { + return "none".equalsIgnoreCase(operation) + || "mv".equalsIgnoreCase(operation) + || "cp".equalsIgnoreCase(operation) + || "delete".equalsIgnoreCase(operation); + } + + public static ImportTsFileOperation getOperation(String operation, boolean isFileStoreEquals) { + switch (operation.toLowerCase()) { + case "none": + return ImportTsFileOperation.NONE; + case "mv": + return ImportTsFileOperation.MV; + case "cp": + if (isFileStoreEquals) { + return ImportTsFileOperation.HARDLINK; + } else { + return ImportTsFileOperation.CP; + } + case "delete": + return ImportTsFileOperation.DELETE; + default: + new IoTPrinter(System.out).println("Args error: os/of must be one of none, mv, cp, delete"); + System.exit(Constants.CODE_ERROR); + return null; + } + } +} diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/AbstractDataTool.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/AbstractDataTool.java index 388993900eb5..eeaf5a542b0d 100644 --- a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/AbstractDataTool.java +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/AbstractDataTool.java @@ -31,7 +31,7 @@ import org.apache.iotdb.rpc.StatementExecutionException; import org.apache.iotdb.session.Session; import org.apache.iotdb.tool.common.Constants; -import org.apache.iotdb.tool.tsfile.ImportTsFile; +import org.apache.iotdb.tool.common.ImportTsFileOperation; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.HelpFormatter; @@ -110,9 +110,9 @@ public abstract class AbstractDataTool { protected static String successDir = "success/"; protected static String timestampPrecision = "ms"; protected static String failedFileDirectory = null; - protected static ImportTsFile.Operation failOperation; + protected static ImportTsFileOperation failOperation; protected static ZoneId zoneId = ZoneId.systemDefault(); - protected static ImportTsFile.Operation successOperation; + protected static ImportTsFileOperation successOperation; protected static String targetFile = Constants.DUMP_FILE_NAME_DEFAULT; protected static final LongAdder loadFileFailedNum = new LongAdder(); protected static final LongAdder loadFileSuccessfulNum = new LongAdder(); diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/ImportData.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/ImportData.java index e0c4c4d72b1e..2985d4c2e49d 100644 --- a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/ImportData.java +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/ImportData.java @@ -27,8 +27,8 @@ import org.apache.iotdb.rpc.StatementExecutionException; import org.apache.iotdb.session.Session; import org.apache.iotdb.tool.common.Constants; +import org.apache.iotdb.tool.common.ImportTsFileOperation; import org.apache.iotdb.tool.common.OptionsUtil; -import org.apache.iotdb.tool.tsfile.ImportTsFile; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.CommandLineParser; @@ -307,27 +307,27 @@ private static void parseSpecialParams(CommandLine commandLine) throws ArgsError final String of = commandLine.getOptionValue(Constants.ON_FAIL_ARGS); final String onFail = StringUtils.isNotBlank(of) ? of.trim().toLowerCase() : null; if (Constants.TSFILE_SUFFIXS.equalsIgnoreCase(fileType) - && (!ImportTsFile.Operation.isValidOperation(onSuccess) - || !ImportTsFile.Operation.isValidOperation(onFail))) { + && (!ImportTsFileOperation.isValidOperation(onSuccess) + || !ImportTsFileOperation.isValidOperation(onFail))) { ioTPrinter.println("Args error: os/of must be one of none, mv, cp, delete"); System.exit(Constants.CODE_ERROR); + } + if (Constants.TSFILE_SUFFIXS.equalsIgnoreCase(fileType)) { boolean isSuccessDirEqualsSourceDir = false; - if (ImportTsFile.Operation.MV.name().equalsIgnoreCase(onSuccess) - || ImportTsFile.Operation.CP.name().equalsIgnoreCase(onSuccess)) { + if (ImportTsFileOperation.MV.name().equalsIgnoreCase(onSuccess) + || ImportTsFileOperation.CP.name().equalsIgnoreCase(onSuccess)) { File dir = createSuccessDir(commandLine); isSuccessDirEqualsSourceDir = isFileStoreEquals(targetPath, dir); } boolean isFailDirEqualsSourceDir = false; - if (ImportTsFile.Operation.MV.name().equalsIgnoreCase(onFail) - || ImportTsFile.Operation.CP.name().equalsIgnoreCase(onFail)) { + if (ImportTsFileOperation.MV.name().equalsIgnoreCase(onFail) + || ImportTsFileOperation.CP.name().equalsIgnoreCase(onFail)) { File dir = createFailDir(commandLine); isFailDirEqualsSourceDir = isFileStoreEquals(targetPath, dir); } - - successOperation = - ImportTsFile.Operation.getOperation(onSuccess, isSuccessDirEqualsSourceDir); - failOperation = ImportTsFile.Operation.getOperation(onFail, isFailDirEqualsSourceDir); + successOperation = ImportTsFileOperation.getOperation(onSuccess, isSuccessDirEqualsSourceDir); + failOperation = ImportTsFileOperation.getOperation(onFail, isFailDirEqualsSourceDir); } } diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/tsfile/ImportTsFile.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/tsfile/ImportTsFile.java index cdfbd5b4437a..c017ab2b0e03 100644 --- a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/tsfile/ImportTsFile.java +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/tsfile/ImportTsFile.java @@ -22,6 +22,7 @@ import org.apache.iotdb.cli.utils.IoTPrinter; import org.apache.iotdb.commons.utils.NodeUrlUtils; import org.apache.iotdb.session.pool.SessionPool; +import org.apache.iotdb.tool.common.ImportTsFileOperation; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.CommandLineParser; @@ -77,8 +78,8 @@ public class ImportTsFile extends AbstractTsFileTool { private static String successDir = "success/"; private static String failDir = "fail/"; - private static Operation successOperation; - private static Operation failOperation; + private static ImportTsFileOperation successOperation; + private static ImportTsFileOperation failOperation; private static int threadNum = 8; @@ -267,27 +268,28 @@ private static void parseSpecialParams(CommandLine commandLine) { final String onSuccess = commandLine.getOptionValue(ON_SUCCESS_ARGS).trim().toLowerCase(); final String onFail = commandLine.getOptionValue(ON_FAIL_ARGS).trim().toLowerCase(); - if (!Operation.isValidOperation(onSuccess) || !Operation.isValidOperation(onFail)) { + if (!ImportTsFileOperation.isValidOperation(onSuccess) + || !ImportTsFileOperation.isValidOperation(onFail)) { IOT_PRINTER.println("Args error: os/of must be one of none, mv, cp, delete"); System.exit(CODE_ERROR); } boolean isSuccessDirEqualsSourceDir = false; - if (Operation.MV.name().equalsIgnoreCase(onSuccess) - || Operation.CP.name().equalsIgnoreCase(onSuccess)) { + if (ImportTsFileOperation.MV.name().equalsIgnoreCase(onSuccess) + || ImportTsFileOperation.CP.name().equalsIgnoreCase(onSuccess)) { File dir = createSuccessDir(commandLine); isSuccessDirEqualsSourceDir = isFileStoreEquals(source, dir); } boolean isFailDirEqualsSourceDir = false; - if (Operation.MV.name().equalsIgnoreCase(onFail) - || Operation.CP.name().equalsIgnoreCase(onFail)) { + if (ImportTsFileOperation.MV.name().equalsIgnoreCase(onFail) + || ImportTsFileOperation.CP.name().equalsIgnoreCase(onFail)) { File dir = createFailDir(commandLine); isFailDirEqualsSourceDir = isFileStoreEquals(source, dir); } - successOperation = Operation.getOperation(onSuccess, isSuccessDirEqualsSourceDir); - failOperation = Operation.getOperation(onFail, isFailDirEqualsSourceDir); + successOperation = ImportTsFileOperation.getOperation(onSuccess, isSuccessDirEqualsSourceDir); + failOperation = ImportTsFileOperation.getOperation(onFail, isFailDirEqualsSourceDir); if (commandLine.getOptionValue(THREAD_NUM_ARGS) != null) { threadNum = Integer.parseInt(commandLine.getOptionValue(THREAD_NUM_ARGS)); @@ -430,41 +432,4 @@ public static void asyncImportTsFiles() { } }); } - - public enum Operation { - NONE, - MV, - HARDLINK, - CP, - DELETE, - ; - - public static boolean isValidOperation(String operation) { - return "none".equalsIgnoreCase(operation) - || "mv".equalsIgnoreCase(operation) - || "cp".equalsIgnoreCase(operation) - || "delete".equalsIgnoreCase(operation); - } - - public static Operation getOperation(String operation, boolean isFileStoreEquals) { - switch (operation.toLowerCase()) { - case "none": - return Operation.NONE; - case "mv": - return Operation.MV; - case "cp": - if (isFileStoreEquals) { - return Operation.HARDLINK; - } else { - return Operation.CP; - } - case "delete": - return Operation.DELETE; - default: - IOT_PRINTER.println("Args error: os/of must be one of none, mv, cp, delete"); - System.exit(CODE_ERROR); - return null; - } - } - } } diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/tsfile/ImportTsFileBase.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/tsfile/ImportTsFileBase.java index 6d6f95dcb827..f1fd3bf2d30f 100644 --- a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/tsfile/ImportTsFileBase.java +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/tsfile/ImportTsFileBase.java @@ -20,6 +20,7 @@ package org.apache.iotdb.tool.tsfile; import org.apache.iotdb.cli.utils.IoTPrinter; +import org.apache.iotdb.tool.common.ImportTsFileOperation; import java.io.File; import java.nio.file.FileAlreadyExistsException; @@ -40,9 +41,9 @@ public abstract class ImportTsFileBase implements Runnable { private static final LongAdder processingLoadFailedFileSuccessfulNum = new LongAdder(); private static String timePrecision = "ms"; private static String successDir; - private static ImportTsFile.Operation successOperation; + private static ImportTsFileOperation successOperation; private static String failDir; - private static ImportTsFile.Operation failOperation; + private static ImportTsFileOperation failOperation; @Override public void run() { @@ -232,9 +233,9 @@ protected static void printResult(final long startTime) { public static void setSuccessAndFailDirAndOperation( final String successDir, - final ImportTsFile.Operation successOperation, + final ImportTsFileOperation successOperation, final String failDir, - final ImportTsFile.Operation failOperation) { + final ImportTsFileOperation failOperation) { ImportTsFileBase.successDir = successDir; ImportTsFileBase.successOperation = successOperation; ImportTsFileBase.failDir = failDir; diff --git a/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/ImportTsFileOperationTest.java b/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/ImportTsFileOperationTest.java new file mode 100644 index 000000000000..63541f586362 --- /dev/null +++ b/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/ImportTsFileOperationTest.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.tool; + +import org.apache.iotdb.tool.common.ImportTsFileOperation; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class ImportTsFileOperationTest { + + @Test + public void testIsValidOperation() { + assertTrue(ImportTsFileOperation.isValidOperation("none")); + assertTrue(ImportTsFileOperation.isValidOperation("mv")); + assertTrue(ImportTsFileOperation.isValidOperation("cp")); + assertTrue(ImportTsFileOperation.isValidOperation("delete")); + assertFalse(ImportTsFileOperation.isValidOperation("invalid")); + } + + @Test + public void testGetOperation() { + assertEquals(ImportTsFileOperation.NONE, ImportTsFileOperation.getOperation("none", false)); + assertEquals(ImportTsFileOperation.MV, ImportTsFileOperation.getOperation("mv", false)); + assertEquals(ImportTsFileOperation.HARDLINK, ImportTsFileOperation.getOperation("cp", true)); + assertEquals(ImportTsFileOperation.CP, ImportTsFileOperation.getOperation("cp", false)); + assertEquals(ImportTsFileOperation.DELETE, ImportTsFileOperation.getOperation("delete", false)); + } +} diff --git a/iotdb-client/client-cpp/pom.xml b/iotdb-client/client-cpp/pom.xml index 2196fb5f2346..7cfe4b0d8a07 100644 --- a/iotdb-client/client-cpp/pom.xml +++ b/iotdb-client/client-cpp/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-client - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT client-cpp pom @@ -43,7 +43,7 @@ org.apache.iotdb iotdb-thrift-commons - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT provided diff --git a/iotdb-client/client-py/pom.xml b/iotdb-client/client-py/pom.xml index f411afcbb92e..a7f3ed35a11d 100644 --- a/iotdb-client/client-py/pom.xml +++ b/iotdb-client/client-py/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-client - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT iotdb-python-api IoTDB: Client: Python-API @@ -34,13 +34,13 @@ org.apache.iotdb iotdb-thrift-commons - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT provided org.apache.iotdb iotdb-thrift - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT provided diff --git a/iotdb-client/isession/pom.xml b/iotdb-client/isession/pom.xml index ed8306daa381..e51dc149ab3b 100644 --- a/iotdb-client/isession/pom.xml +++ b/iotdb-client/isession/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-client - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT isession IoTDB: Client: isession @@ -32,7 +32,7 @@ org.apache.iotdb service-rpc - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.apache.tsfile @@ -47,12 +47,12 @@ org.apache.iotdb iotdb-thrift - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.apache.iotdb iotdb-thrift-commons - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.apache.thrift diff --git a/iotdb-client/jdbc/pom.xml b/iotdb-client/jdbc/pom.xml index 625c75ca1407..ebd2651c84c1 100644 --- a/iotdb-client/jdbc/pom.xml +++ b/iotdb-client/jdbc/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-client - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT iotdb-jdbc IoTDB: Client: Jdbc @@ -38,12 +38,12 @@ org.apache.iotdb service-rpc - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.apache.iotdb iotdb-thrift-commons - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.apache.tsfile @@ -58,7 +58,7 @@ org.apache.iotdb iotdb-thrift - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.apache.thrift diff --git a/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/Constant.java b/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/Constant.java index 66b1e57d092a..cf1f0a2fe7fe 100644 --- a/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/Constant.java +++ b/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/Constant.java @@ -42,6 +42,9 @@ private Constant() {} public static final String STATISTICS_RESULT_LINES = "* Lines of result: %d"; public static final String STATISTICS_PRC_INFO = "* Num of RPC: %d, avg cost: %d ms"; + public static final String TREE = "tree"; + public static final String TABLE = "table"; + // version number public enum Version { V_0_12, diff --git a/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBConnection.java b/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBConnection.java index f0949429270a..07d30370999c 100644 --- a/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBConnection.java +++ b/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBConnection.java @@ -617,6 +617,10 @@ protected void changeDefaultDatabase(String database) { params.setDb(database); } + protected void changeDefaultSqlDialect(String sqlDialect) { + params.setSqlDialect(sqlDialect); + } + public int getTimeFactor() { return timeFactor; } diff --git a/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBConnectionParams.java b/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBConnectionParams.java index a31d2198d279..1112caabd4e2 100644 --- a/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBConnectionParams.java +++ b/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBConnectionParams.java @@ -27,6 +27,8 @@ import java.time.ZoneId; import java.util.Optional; +import static org.apache.iotdb.jdbc.Constant.TREE; + public class IoTDBConnectionParams { private String host = Config.IOTDB_DEFAULT_HOST; @@ -50,7 +52,7 @@ public class IoTDBConnectionParams { private String trustStore; private String trustStorePwd; - private String sqlDialect = "tree"; + private String sqlDialect = TREE; private String db; diff --git a/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBStatement.java b/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBStatement.java index f29b18031f11..a3b6205512b7 100644 --- a/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBStatement.java +++ b/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBStatement.java @@ -46,6 +46,9 @@ import java.util.BitSet; import java.util.List; +import static org.apache.iotdb.jdbc.Constant.TABLE; +import static org.apache.iotdb.jdbc.Constant.TREE; + public class IoTDBStatement implements Statement { private final IoTDBConnection connection; @@ -326,6 +329,10 @@ private boolean executeSQL(String sql) throws TException, SQLException { connection.changeDefaultDatabase(execResp.getDatabase()); } + if (execResp.isSetTableModel()) { + connection.changeDefaultSqlDialect(execResp.tableModel ? TABLE : TREE); + } + if (execResp.isSetColumns()) { queryId = execResp.getQueryId(); if (execResp.queryResult == null) { diff --git a/iotdb-client/pom.xml b/iotdb-client/pom.xml index d4ea94352634..167f8a5f07e5 100644 --- a/iotdb-client/pom.xml +++ b/iotdb-client/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-parent - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT iotdb-client pom diff --git a/iotdb-client/service-rpc/pom.xml b/iotdb-client/service-rpc/pom.xml index 9515493257e9..0b9ccd0af091 100644 --- a/iotdb-client/service-rpc/pom.xml +++ b/iotdb-client/service-rpc/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-client - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT service-rpc IoTDB: Client: Service-RPC @@ -60,12 +60,12 @@ org.apache.iotdb iotdb-thrift - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.apache.iotdb iotdb-thrift-commons - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.apache.thrift diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/RpcUtils.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/RpcUtils.java index d41c1c8fd970..7541f500e45d 100644 --- a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/RpcUtils.java +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/RpcUtils.java @@ -374,6 +374,20 @@ public static boolean isUseDatabase(String sql) { return sql.length() > 4 && "use ".equalsIgnoreCase(sql.substring(0, 4)); } + public static boolean isSetSqlDialect(String sql) { + // check if startWith 'set ' + if (sql.length() <= 15 || !"set ".equalsIgnoreCase(sql.substring(0, 4))) { + return false; + } + + // check if the following content of sql is 'sql_dialect' + sql = sql.substring(4).trim(); + if (sql.length() <= 11) { + return false; + } + return sql.substring(0, 11).equalsIgnoreCase("sql_dialect"); + } + public static long getMilliSecond(long time, int timeFactor) { return time / timeFactor * 1_000; } diff --git a/iotdb-client/service-rpc/src/test/java/org/apache/iotdb/rpc/RpcUtilsTest.java b/iotdb-client/service-rpc/src/test/java/org/apache/iotdb/rpc/RpcUtilsTest.java index 12969e96904f..a69802760745 100644 --- a/iotdb-client/service-rpc/src/test/java/org/apache/iotdb/rpc/RpcUtilsTest.java +++ b/iotdb-client/service-rpc/src/test/java/org/apache/iotdb/rpc/RpcUtilsTest.java @@ -64,4 +64,14 @@ public void parseLongToDateWithPrecision() { "1970-01-01T07:59:59.999+08:00", RpcUtils.parseLongToDateWithPrecision(formatter, -1, zoneId, "ms")); } + + @Test + public void testIsSetSqlDialect() { + Assert.assertTrue(RpcUtils.isSetSqlDialect("set sql_dialect=table")); + Assert.assertTrue(RpcUtils.isSetSqlDialect("set sql_dialect =table")); + Assert.assertTrue(RpcUtils.isSetSqlDialect("set sql_dialect =table")); + Assert.assertTrue(RpcUtils.isSetSqlDialect("set sql_dialect =table")); + Assert.assertFalse(RpcUtils.isSetSqlDialect("setsql_dialect =table")); + Assert.assertFalse(RpcUtils.isSetSqlDialect("set sql_dia")); + } } diff --git a/iotdb-client/session/pom.xml b/iotdb-client/session/pom.xml index a79d3c25bafe..40ed5ab0a96d 100644 --- a/iotdb-client/session/pom.xml +++ b/iotdb-client/session/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-client - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT iotdb-session IoTDB: Client: Session @@ -37,17 +37,17 @@ org.apache.iotdb service-rpc - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.apache.iotdb isession - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.apache.iotdb iotdb-thrift-commons - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.apache.tsfile @@ -62,7 +62,7 @@ org.apache.iotdb iotdb-thrift - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.slf4j diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/Session.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/Session.java index 1e65be39082f..6908cad0632e 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/Session.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/Session.java @@ -192,7 +192,7 @@ public class Session implements ISession { protected long retryIntervalInMs = SessionConfig.RETRY_INTERVAL_IN_MS; - protected String sqlDialect = SessionConfig.SQL_DIALECT; + protected volatile String sqlDialect = SessionConfig.SQL_DIALECT; // may be null protected volatile String database; @@ -215,6 +215,9 @@ public class Session implements ISession { "All values are null and this submission is ignored,deviceIds are [{}],times are [{}],measurements are [{}]"; private static final String ALL_INSERT_DATA_IS_NULL = "All inserted data is null."; + protected static final String TABLE = "table"; + protected static final String TREE = "tree"; + public Session(String host, int rpcPort) { this( host, @@ -994,8 +997,10 @@ private SessionConnection getQuerySessionConnection() { public void executeNonQueryStatement(String sql) throws IoTDBConnectionException, StatementExecutionException { String previousDB = database; + String previousDialect = sqlDialect; defaultSessionConnection.executeNonQueryStatement(sql); - if (!Objects.equals(previousDB, database) && endPointToSessionConnection != null) { + if ((!Objects.equals(previousDB, database) || !Objects.equals(previousDialect, sqlDialect)) + && endPointToSessionConnection != null) { Iterator> iterator = endPointToSessionConnection.entrySet().iterator(); while (iterator.hasNext()) { @@ -1005,7 +1010,7 @@ public void executeNonQueryStatement(String sql) try { sessionConnection.executeNonQueryStatement(sql); } catch (Throwable t) { - logger.warn("failed to change database for {}", entry.getKey()); + logger.warn("failed to execute '{}' for {}", sql, entry.getKey()); iterator.remove(); } } @@ -2960,6 +2965,9 @@ private TSInsertTabletReq genTSInsertTabletReq(Tablet tablet, boolean sorted, bo TSInsertTabletReq request = new TSInsertTabletReq(); for (IMeasurementSchema measurementSchema : tablet.getSchemas()) { + if (measurementSchema.getMeasurementName() == null) { + throw new IllegalArgumentException("measurement should be non null value"); + } request.addToMeasurements(measurementSchema.getMeasurementName()); request.addToTypes(measurementSchema.getType().ordinal()); } @@ -3084,6 +3092,9 @@ private void updateTSInsertTabletsReq( List dataTypes = new ArrayList<>(); request.setIsAligned(isAligned); for (IMeasurementSchema measurementSchema : tablet.getSchemas()) { + if (measurementSchema.getMeasurementName() == null) { + throw new IllegalArgumentException("measurement should be non null value"); + } measurements.add(measurementSchema.getMeasurementName()); dataTypes.add(measurementSchema.getType().ordinal()); } @@ -4150,6 +4161,14 @@ public String getDatabase() { return database; } + protected void changeSqlDialect(String sqlDialect) { + this.sqlDialect = sqlDialect; + } + + public String getSqlDialect() { + return sqlDialect; + } + public static class Builder extends AbstractSessionBuilder { public Builder host(String host) { diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/SessionConnection.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/SessionConnection.java index c0dc516ba114..31abf69ec2ed 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/SessionConnection.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/SessionConnection.java @@ -85,6 +85,9 @@ import java.util.concurrent.TimeUnit; import java.util.function.Supplier; +import static org.apache.iotdb.session.Session.TABLE; +import static org.apache.iotdb.session.Session.TREE; + @SuppressWarnings("java:S2142") public class SessionConnection { @@ -107,7 +110,7 @@ public class SessionConnection { private final long retryIntervalInMs; - private final String sqlDialect; + private String sqlDialect; private String database; @@ -471,6 +474,13 @@ private TSStatus executeNonQueryStatementInternal(TSExecuteStatementReq request) session.changeDatabase(dbName); this.database = dbName; } + if (resp.isSetTableModel()) { + String sqlDialect = resp.tableModel ? TABLE : TREE; + if (!sqlDialect.equalsIgnoreCase(this.sqlDialect)) { + session.changeSqlDialect(sqlDialect); + this.sqlDialect = sqlDialect; + } + } return resp.status; } diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/TableSessionBuilder.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/TableSessionBuilder.java index 9929f4b9cdc7..fd59846c6e68 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/TableSessionBuilder.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/TableSessionBuilder.java @@ -27,6 +27,8 @@ import java.util.Collections; import java.util.List; +import static org.apache.iotdb.session.Session.TABLE; + /** * A builder class for constructing instances of {@link ITableSession}. * @@ -267,7 +269,7 @@ public ITableSession build() throws IoTDBConnectionException { this.nodeUrls = Collections.singletonList(SessionConfig.DEFAULT_HOST + ":" + SessionConfig.DEFAULT_PORT); } - this.sqlDialect = "table"; + this.sqlDialect = TABLE; Session newSession = new Session(this); newSession.open(enableCompression, connectionTimeoutInMs); return new TableSession(newSession); diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/pool/SessionPool.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/pool/SessionPool.java index 86b4700c90eb..e7515d60e476 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/pool/SessionPool.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/pool/SessionPool.java @@ -59,6 +59,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; +import static org.apache.iotdb.rpc.RpcUtils.isSetSqlDialect; import static org.apache.iotdb.rpc.RpcUtils.isUseDatabase; /** @@ -3080,8 +3081,8 @@ public SessionDataSetWrapper executeQueryStatement(String sql, long timeoutInMs) public void executeNonQueryStatement(String sql) throws StatementExecutionException, IoTDBConnectionException { - // use XXX is forbidden in SessionPool.executeNonQueryStatement - if (isUseDatabase(sql)) { + // 'use XXX' and 'set sql_dialect' is forbidden in SessionPool.executeNonQueryStatement + if (isUseDatabase(sql) || isSetSqlDialect(sql)) { throw new IllegalArgumentException( String.format("SessionPool doesn't support executing %s directly", sql)); } diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/pool/TableSessionWrapper.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/pool/TableSessionWrapper.java index c595b0feb219..242ab1a63532 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/pool/TableSessionWrapper.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/pool/TableSessionWrapper.java @@ -121,6 +121,21 @@ public void close() throws IoTDBConnectionException { return; } } + + if (!Objects.equals(session.getSqlDialect(), sessionPool.sqlDialect)) { + try { + session.executeNonQueryStatement("set sql_dialect=" + sessionPool.sqlDialect); + } catch (StatementExecutionException e) { + LOGGER.warn( + "Failed to change back sql_dialect by executing: set sql_dialect={}", + sessionPool.sqlDialect, + e); + session.close(); + session = null; + return; + } + } + sessionPool.putBack(session); session = null; } diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionConsumer.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionConsumer.java index c783626fc20a..7dfae59a935d 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionConsumer.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionConsumer.java @@ -391,7 +391,9 @@ AbstractSubscriptionProvider constructProviderAndHandshake(final TEndPoint endPo } catch (final Exception ignored) { } throw new SubscriptionConnectionException( - String.format("Failed to handshake with subscription provider %s", provider), e); + String.format( + "Failed to handshake with subscription provider %s because of %s", provider, e), + e); } // update consumer id and consumer group id if not exist diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionProviders.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionProviders.java index a9d239d5886f..142719df8b1e 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionProviders.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionProviders.java @@ -85,7 +85,8 @@ void openProviders(final AbstractSubscriptionConsumer consumer) throws Subscript try { defaultProvider = consumer.constructProviderAndHandshake(endPoint); } catch (final Exception e) { - LOGGER.warn("Failed to create connection with {}", endPoint, e); + LOGGER.warn( + "{} failed to create connection with {} because of {}", consumer, endPoint, e, e); continue; // try next endpoint } defaultDataNodeId = defaultProvider.getDataNodeId(); @@ -95,8 +96,9 @@ void openProviders(final AbstractSubscriptionConsumer consumer) throws Subscript try { allEndPoints = defaultProvider.getSessionConnection().fetchAllEndPoints(); } catch (final Exception e) { - LOGGER.warn("Failed to fetch all endpoints from {}, will retry later...", endPoint, e); - break; // retry later + LOGGER.warn( + "{} failed to fetch all endpoints from {} because of {}", consumer, endPoint, e, e); + break; } for (final Map.Entry entry : allEndPoints.entrySet()) { @@ -109,8 +111,12 @@ void openProviders(final AbstractSubscriptionConsumer consumer) throws Subscript provider = consumer.constructProviderAndHandshake(entry.getValue()); } catch (final Exception e) { LOGGER.warn( - "Failed to create connection with {}, will retry later...", entry.getValue(), e); - continue; // retry later + "{} failed to create connection with {} because of {}", + consumer, + entry.getValue(), + e, + e); + continue; } addProvider(entry.getKey(), provider); } @@ -134,7 +140,7 @@ void closeProviders() { try { provider.close(); } catch (final Exception e) { - LOGGER.warn(e.getMessage()); + LOGGER.warn("Failed to close subscription provider {} because of {}", provider, e, e); } } subscriptionProviders.clear(); @@ -242,8 +248,10 @@ private void heartbeatInternal(final AbstractSubscriptionConsumer consumer) { provider.setAvailable(); } catch (final Exception e) { LOGGER.warn( - "something unexpected happened when sending heartbeat to subscription provider {}, set subscription provider unavailable", + "{} failed to sending heartbeat to subscription provider {} because of {}, set subscription provider unavailable", + consumer, provider, + e, e); provider.setUnavailable(); } @@ -270,7 +278,7 @@ private void syncInternal(final AbstractSubscriptionConsumer consumer) { try { openProviders(consumer); } catch (final Exception e) { - LOGGER.warn("something unexpected happened when syncing subscription endpoints...", e); + LOGGER.warn("Failed to open providers for consumer {} because of {}", consumer, e, e); return; } } @@ -279,8 +287,8 @@ private void syncInternal(final AbstractSubscriptionConsumer consumer) { try { allEndPoints = consumer.fetchAllEndPointsWithRedirection(); } catch (final Exception e) { - LOGGER.warn("Failed to fetch all endpoints, will retry later...", e); - return; // retry later + LOGGER.warn("Failed to fetch all endpoints for consumer {} because of {}", consumer, e, e); + return; } // add new providers or handshake existing providers @@ -294,8 +302,8 @@ private void syncInternal(final AbstractSubscriptionConsumer consumer) { newProvider = consumer.constructProviderAndHandshake(endPoint); } catch (final Exception e) { LOGGER.warn( - "Failed to create connection with endpoint {}, will retry later...", endPoint, e); - continue; // retry later + "{} failed to create connection with {} because of {}", consumer, endPoint, e, e); + continue; } addProvider(entry.getKey(), newProvider); } else { @@ -305,8 +313,10 @@ private void syncInternal(final AbstractSubscriptionConsumer consumer) { provider.setAvailable(); } catch (final Exception e) { LOGGER.warn( - "something unexpected happened when sending heartbeat to subscription provider {}, set subscription provider unavailable", + "{} failed to sending heartbeat to subscription provider {} because of {}, set subscription provider unavailable", + consumer, provider, + e, e); provider.setUnavailable(); } @@ -316,8 +326,10 @@ private void syncInternal(final AbstractSubscriptionConsumer consumer) { closeAndRemoveProvider(entry.getKey()); } catch (final Exception e) { LOGGER.warn( - "Exception occurred when closing and removing subscription provider with data node id {}", - entry.getKey(), + "Exception occurred when {} closing and removing subscription provider {} because of {}", + consumer, + provider, + e, e); } } @@ -332,8 +344,10 @@ private void syncInternal(final AbstractSubscriptionConsumer consumer) { closeAndRemoveProvider(dataNodeId); } catch (final Exception e) { LOGGER.warn( - "Exception occurred when closing and removing subscription provider with data node id {}", - dataNodeId, + "Exception occurred when {} closing and removing subscription provider {} because of {}", + consumer, + provider, + e, e); } } diff --git a/iotdb-collector/collector-core/pom.xml b/iotdb-collector/collector-core/pom.xml index e317f3b35c47..d5d7a4dc47fc 100644 --- a/iotdb-collector/collector-core/pom.xml +++ b/iotdb-collector/collector-core/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-collector - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT collector-core IoTDB: Collector: Core @@ -32,15 +32,48 @@ org.apache.iotdb collector-openapi - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT - org.eclipse.jetty - jetty-http + org.apache.iotdb + service-rpc + 2.0.2-SNAPSHOT - org.eclipse.jetty - jetty-util + org.apache.iotdb + pipe-api + 2.0.2-SNAPSHOT + + + org.apache.iotdb + node-commons + 2.0.2-SNAPSHOT + + + io.netty + netty-common + + + io.netty + netty-handler + + + net.minidev + json-smart + + + + + jakarta.servlet + jakarta.servlet-api + + + jakarta.ws.rs + jakarta.ws.rs-api + + + org.slf4j + slf4j-api org.eclipse.jetty @@ -54,9 +87,5 @@ org.glassfish.jersey.containers jersey-container-servlet-core - - org.glassfish.jersey.inject - jersey-hk2 - diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/api/impl/PingApiServiceImpl.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/api/impl/PingApiServiceImpl.java index 0fc582467df8..45d62ee5458b 100644 --- a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/api/impl/PingApiServiceImpl.java +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/api/impl/PingApiServiceImpl.java @@ -17,9 +17,8 @@ package org.apache.iotdb.collector.api.impl; -import org.apache.iotdb.collector.api.NotFoundException; -import org.apache.iotdb.collector.api.PingApiService; -import org.apache.iotdb.collector.api.v1.model.ExecutionStatus; +import org.apache.iotdb.application.protocol.rest.PingApiService; +import org.apache.iotdb.application.protocol.rest.v1.model.ExecutionStatus; import org.apache.iotdb.rpc.TSStatusCode; import javax.ws.rs.core.Response; @@ -28,7 +27,7 @@ public class PingApiServiceImpl extends PingApiService { @Override - public Response tryPing(final SecurityContext securityContext) throws NotFoundException { + public Response tryPing(final SecurityContext securityContext) { return Response.ok() .entity( new ExecutionStatus() diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/api/v1/handler/RequestValidationHandler.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/api/v1/handler/RequestValidationHandler.java index 9456c2f33954..b07452113f37 100644 --- a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/api/v1/handler/RequestValidationHandler.java +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/api/v1/handler/RequestValidationHandler.java @@ -19,8 +19,8 @@ package org.apache.iotdb.collector.api.v1.handler; -import org.apache.iotdb.collector.api.v1.model.CreatePipeRequest; -import org.apache.iotdb.collector.api.v1.model.StopPipeRequest; +import org.apache.iotdb.application.protocol.rest.v1.model.CreatePipeRequest; +import org.apache.iotdb.application.protocol.rest.v1.model.StopPipeRequest; import java.util.Objects; diff --git a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/api/v1/impl/AdminApiServiceImpl.java b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/api/v1/impl/AdminApiServiceImpl.java index 07d1f3e7878b..a7a25dfdd6f3 100644 --- a/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/api/v1/impl/AdminApiServiceImpl.java +++ b/iotdb-collector/collector-core/src/main/java/org/apache/iotdb/collector/api/v1/impl/AdminApiServiceImpl.java @@ -19,15 +19,15 @@ package org.apache.iotdb.collector.api.v1.impl; +import org.apache.iotdb.application.protocol.rest.v1.AdminApiService; +import org.apache.iotdb.application.protocol.rest.v1.NotFoundException; +import org.apache.iotdb.application.protocol.rest.v1.model.AlterPipeRequest; +import org.apache.iotdb.application.protocol.rest.v1.model.CreatePipeRequest; +import org.apache.iotdb.application.protocol.rest.v1.model.DropPipeRequest; +import org.apache.iotdb.application.protocol.rest.v1.model.StartPipeRequest; +import org.apache.iotdb.application.protocol.rest.v1.model.StopPipeRequest; import org.apache.iotdb.collector.agent.CollectorAgent; -import org.apache.iotdb.collector.api.v1.AdminApiService; -import org.apache.iotdb.collector.api.v1.NotFoundException; import org.apache.iotdb.collector.api.v1.handler.RequestValidationHandler; -import org.apache.iotdb.collector.api.v1.model.AlterPipeRequest; -import org.apache.iotdb.collector.api.v1.model.CreatePipeRequest; -import org.apache.iotdb.collector.api.v1.model.DropPipeRequest; -import org.apache.iotdb.collector.api.v1.model.StartPipeRequest; -import org.apache.iotdb.collector.api.v1.model.StopPipeRequest; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/iotdb-collector/collector-openapi/pom.xml b/iotdb-collector/collector-openapi/pom.xml index 3ba2259f6636..4cf09b5192c3 100644 --- a/iotdb-collector/collector-openapi/pom.xml +++ b/iotdb-collector/collector-openapi/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-collector - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT collector-openapi IoTDB: Collector: OpenAPI diff --git a/iotdb-collector/pom.xml b/iotdb-collector/pom.xml index d23d331dc4d4..47d4e5b4a52e 100644 --- a/iotdb-collector/pom.xml +++ b/iotdb-collector/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-parent - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT iotdb-collector pom @@ -33,11 +33,4 @@ collector-core collector-openapi - - - org.apache.iotdb - node-commons - 2.0.0-SNAPSHOT - - diff --git a/iotdb-core/ainode/pom.xml b/iotdb-core/ainode/pom.xml index 34f3291b79fd..ad6679c77010 100644 --- a/iotdb-core/ainode/pom.xml +++ b/iotdb-core/ainode/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-core - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT iotdb-ainode IoTDB: Core: AINode @@ -33,25 +33,25 @@ org.apache.iotdb iotdb-thrift-commons - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT provided org.apache.iotdb iotdb-thrift - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT provided org.apache.iotdb iotdb-thrift-confignode - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT provided org.apache.iotdb iotdb-thrift-ainode - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT provided diff --git a/iotdb-core/ainode/pyproject.toml b/iotdb-core/ainode/pyproject.toml index c7f773b05908..efbbdb1edaf7 100644 --- a/iotdb-core/ainode/pyproject.toml +++ b/iotdb-core/ainode/pyproject.toml @@ -34,12 +34,12 @@ classifiers = [ "Topic :: Software Development :: Libraries :: Python Modules", ] include = [ - "iotdb/thrift/*", - "iotdb/thrift/common/*", - "iotdb/thrift/confignode/*", - "iotdb/thrift/datanode/*", - "iotdb/thrift/ainode/*", - "iotdb/conf/*", + {path = "iotdb/thrift/*", format = "wheel"}, + {path = "iotdb/thrift/common/*", format = "wheel"}, + {path = "iotdb/thrift/confignode/*", format = "wheel"}, + {path = "iotdb/thrift/datanode/*", format = "wheel"}, + {path = "iotdb/thrift/ainode/*", format = "wheel"}, + {path = "iotdb/conf/*", format = "wheel"}, ] packages = [ { include = "iotdb" } @@ -50,7 +50,7 @@ python = ">=3.8, <3.13" numpy = "^1.21.4" pandas = "^1.3.5" -torch = "2.2.0" +torch = ">=2.2.0" pylru = "^1.2.1" thrift = "^0.13.0" diff --git a/iotdb-core/ainode/resources/conf/ainode-env.sh b/iotdb-core/ainode/resources/conf/ainode-env.sh index b2737ebe1505..1ec434ad2d98 100644 --- a/iotdb-core/ainode/resources/conf/ainode-env.sh +++ b/iotdb-core/ainode/resources/conf/ainode-env.sh @@ -110,7 +110,7 @@ fi echo "Installing AINode..." cd "$SCRIPT_DIR/../lib/" shopt -s nullglob -for i in *.whl *.tar.gz; do +for i in *.whl; do if [[ $i =~ "ainode" ]]; then echo Installing AINode body: $i if [ -z "$p_pypi_mirror" ]; then diff --git a/iotdb-core/antlr/pom.xml b/iotdb-core/antlr/pom.xml index 1012420ff7ac..8b183bcb1111 100644 --- a/iotdb-core/antlr/pom.xml +++ b/iotdb-core/antlr/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-core - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT iotdb-antlr IoTDB: Core: Antlr-Parser diff --git a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IdentifierParser.g4 b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IdentifierParser.g4 index 63ff0c7afa49..c0f382d0b530 100644 --- a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IdentifierParser.g4 +++ b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IdentifierParser.g4 @@ -73,6 +73,8 @@ keyWords | CQ | CQS | CREATE + | CURRENT_SQL_DIALECT + | CURRENT_USER | DATA | DATA_REPLICATION_FACTOR | DATA_REGION_GROUP_NUM @@ -214,6 +216,7 @@ keyWords | SOFFSET | SOURCE | SPACE + | SQL_DIALECT | STORAGE | START | STARTTIME @@ -224,6 +227,7 @@ keyWords | SUBSCRIPTIONS | SUBSTRING | SYSTEM + | TABLE | TAGS | TAIL | TASK @@ -242,6 +246,7 @@ keyWords | TOPIC | TOPICS | TRACING + | TREE | TRIGGER | TRIGGERS | TRUE diff --git a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 index 5e7b05378256..de0b05a95e30 100644 --- a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 +++ b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 @@ -88,7 +88,7 @@ utilityStatement | setSystemStatus | showVersion | showFlushInfo | showLockInfo | showQueryResource | showQueries | showCurrentTimestamp | killQuery | grantWatermarkEmbedding | revokeWatermarkEmbedding | loadConfiguration | loadTimeseries | loadFile - | removeFile | unloadFile + | removeFile | unloadFile | setSqlDialect | showCurrentSqlDialect | showCurrentUser ; /** @@ -1183,6 +1183,18 @@ unloadFile : UNLOAD srcFileName=STRING_LITERAL dstFileDir=STRING_LITERAL ; +setSqlDialect + : SET SQL_DIALECT OPERATOR_SEQ (TABLE | TREE) + ; + +showCurrentSqlDialect + : SHOW CURRENT_SQL_DIALECT + ; + +showCurrentUser + : SHOW CURRENT_USER + ; + // attribute clauses syncAttributeClauses : attributePair (COMMA? attributePair)* diff --git a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 index 0a3453cfc541..8d4ba400adce 100644 --- a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 +++ b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 @@ -205,6 +205,14 @@ CREATE : C R E A T E ; +CURRENT_SQL_DIALECT + : C U R R E N T '_' S Q L '_' D I A L E C T + ; + +CURRENT_USER + : C U R R E N T '_' U S E R + ; + DATA : D A T A ; @@ -762,6 +770,10 @@ SPACE : S P A C E ; +SQL_DIALECT + : S Q L '_' D I A L E C T + ; + STORAGE : S T O R A G E ; @@ -802,6 +814,10 @@ SYSTEM : S Y S T E M ; +TABLE + : T A B L E + ; + TAGS : T A G S ; @@ -874,6 +890,10 @@ TRACING : T R A C I N G ; +TREE + : T R E E + ; + TRIGGER : T R I G G E R ; diff --git a/iotdb-core/confignode/pom.xml b/iotdb-core/confignode/pom.xml index 3b7998f091e8..2118228b5000 100644 --- a/iotdb-core/confignode/pom.xml +++ b/iotdb-core/confignode/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-core - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT iotdb-confignode IoTDB: Core: ConfigNode @@ -42,62 +42,62 @@ org.apache.iotdb service-rpc - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.apache.iotdb iotdb-consensus - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.apache.iotdb iotdb-server - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.apache.iotdb pipe-api - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.apache.iotdb trigger-api - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.apache.iotdb metrics-interface - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.apache.iotdb iotdb-thrift - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.apache.iotdb iotdb-thrift-commons - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.apache.iotdb iotdb-thrift-confignode - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.apache.iotdb iotdb-thrift-ainode - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.apache.iotdb node-commons - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.apache.iotdb udf-api - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.apache.tsfile diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanVisitor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanVisitor.java index d2a46baf5858..b5c1923eccab 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanVisitor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanVisitor.java @@ -19,7 +19,8 @@ package org.apache.iotdb.confignode.consensus.request; -import org.apache.iotdb.confignode.consensus.request.write.auth.AuthorPlan; +import org.apache.iotdb.confignode.consensus.request.write.auth.AuthorRelationalPlan; +import org.apache.iotdb.confignode.consensus.request.write.auth.AuthorTreePlan; import org.apache.iotdb.confignode.consensus.request.write.database.DatabaseSchemaPlan; import org.apache.iotdb.confignode.consensus.request.write.database.DeleteDatabasePlan; import org.apache.iotdb.confignode.consensus.request.write.database.SetTTLPlan; @@ -65,29 +66,83 @@ public R process(final ConfigPhysicalPlan plan, final C context) { case PipeDeactivateTemplate: return visitPipeDeactivateTemplate((PipeDeactivateTemplatePlan) plan, context); case CreateRole: - return visitCreateRole((AuthorPlan) plan, context); + return visitCreateRole((AuthorTreePlan) plan, context); case DropRole: - return visitDropRole((AuthorPlan) plan, context); + return visitDropRole((AuthorTreePlan) plan, context); case GrantRole: - return visitGrantRole((AuthorPlan) plan, context); + return visitGrantRole((AuthorTreePlan) plan, context); case RevokeRole: - return visitRevokeRole((AuthorPlan) plan, context); + return visitRevokeRole((AuthorTreePlan) plan, context); case CreateUser: - return visitCreateUser((AuthorPlan) plan, context); + return visitCreateUser((AuthorTreePlan) plan, context); case CreateUserWithRawPassword: - return visitCreateRawUser((AuthorPlan) plan, context); + return visitCreateRawUser((AuthorTreePlan) plan, context); case UpdateUser: - return visitUpdateUser((AuthorPlan) plan, context); + return visitUpdateUser((AuthorTreePlan) plan, context); case DropUser: - return visitDropUser((AuthorPlan) plan, context); + return visitDropUser((AuthorTreePlan) plan, context); case GrantUser: - return visitGrantUser((AuthorPlan) plan, context); + return visitGrantUser((AuthorTreePlan) plan, context); case RevokeUser: - return visitRevokeUser((AuthorPlan) plan, context); + return visitRevokeUser((AuthorTreePlan) plan, context); case GrantRoleToUser: - return visitGrantRoleToUser((AuthorPlan) plan, context); + return visitGrantRoleToUser((AuthorTreePlan) plan, context); case RevokeRoleFromUser: - return visitRevokeRoleFromUser((AuthorPlan) plan, context); + return visitRevokeRoleFromUser((AuthorTreePlan) plan, context); + case RCreateUser: + return visitRCreateUser((AuthorRelationalPlan) plan, context); + case RCreateRole: + return visitRCreateRole((AuthorRelationalPlan) plan, context); + case RUpdateUser: + return visitRUpdateUser((AuthorRelationalPlan) plan, context); + case RDropUser: + return visitRDropUserPlan((AuthorRelationalPlan) plan, context); + case RDropRole: + return visitRDropRolePlan((AuthorRelationalPlan) plan, context); + case RGrantUserRole: + return visitRGrantUserRole((AuthorRelationalPlan) plan, context); + case RRevokeUserRole: + return visitRRevokeUserRole((AuthorRelationalPlan) plan, context); + case RGrantUserAny: + return visitRGrantUserAny((AuthorRelationalPlan) plan, context); + case RGrantRoleAny: + return visitRGrantRoleAny((AuthorRelationalPlan) plan, context); + case RGrantUserAll: + return visitRGrantUserAll((AuthorRelationalPlan) plan, context); + case RGrantRoleAll: + return visitRGrantRoleAll((AuthorRelationalPlan) plan, context); + case RGrantUserDBPriv: + return visitRGrantUserDB((AuthorRelationalPlan) plan, context); + case RGrantUserTBPriv: + return visitRGrantUserTB((AuthorRelationalPlan) plan, context); + case RGrantRoleDBPriv: + return visitRGrantRoleDB((AuthorRelationalPlan) plan, context); + case RGrantRoleTBPriv: + return visitRGrantRoleTB((AuthorRelationalPlan) plan, context); + case RRevokeUserAny: + return visitRRevokeUserAny((AuthorRelationalPlan) plan, context); + case RRevokeRoleAny: + return visitRRevokeRoleAny((AuthorRelationalPlan) plan, context); + case RRevokeUserAll: + return visitRRevokeUserAll((AuthorRelationalPlan) plan, context); + case RRevokeRoleAll: + return visitRRevokeRoleAll((AuthorRelationalPlan) plan, context); + case RRevokeUserDBPriv: + return visitRRevokeUserDBPrivilege((AuthorRelationalPlan) plan, context); + case RRevokeUserTBPriv: + return visitRRevokeUserTBPrivilege((AuthorRelationalPlan) plan, context); + case RRevokeRoleDBPriv: + return visitRRevokeRoleDBPrivilege((AuthorRelationalPlan) plan, context); + case RRevokeRoleTBPriv: + return visitRRevokeRoleTBPrivilege((AuthorRelationalPlan) plan, context); + case RGrantUserSysPri: + return visitRGrantUserSysPrivilege((AuthorRelationalPlan) plan, context); + case RGrantRoleSysPri: + return visitRGrantRoleSysPrivilege((AuthorRelationalPlan) plan, context); + case RRevokeUserSysPri: + return visitRRevokeUserSysPrivilege((AuthorRelationalPlan) plan, context); + case RRevokeRoleSysPri: + return visitRRevokeRoleSysPrivilege((AuthorRelationalPlan) plan, context); case SetTTL: return visitTTL((SetTTLPlan) plan, context); case PipeCreateTable: @@ -164,54 +219,170 @@ public R visitPipeDeactivateTemplate( return visitPlan(pipeDeactivateTemplatePlan, context); } - public R visitCreateUser(final AuthorPlan createUserPlan, final C context) { + public R visitCreateUser(final AuthorTreePlan createUserPlan, final C context) { return visitPlan(createUserPlan, context); } - public R visitCreateRawUser(final AuthorPlan createRawUserPlan, final C context) { + public R visitCreateRawUser(final AuthorTreePlan createRawUserPlan, final C context) { return visitPlan(createRawUserPlan, context); } - public R visitUpdateUser(final AuthorPlan updateUserPlan, final C context) { + public R visitUpdateUser(final AuthorTreePlan updateUserPlan, final C context) { return visitPlan(updateUserPlan, context); } - public R visitDropUser(final AuthorPlan dropUserPlan, final C context) { + public R visitDropUser(final AuthorTreePlan dropUserPlan, final C context) { return visitPlan(dropUserPlan, context); } - public R visitGrantUser(final AuthorPlan grantUserPlan, final C context) { + public R visitGrantUser(final AuthorTreePlan grantUserPlan, final C context) { return visitPlan(grantUserPlan, context); } - public R visitRevokeUser(final AuthorPlan revokeUserPlan, final C context) { + public R visitRevokeUser(final AuthorTreePlan revokeUserPlan, final C context) { return visitPlan(revokeUserPlan, context); } - public R visitCreateRole(final AuthorPlan createRolePlan, final C context) { + public R visitCreateRole(final AuthorTreePlan createRolePlan, final C context) { return visitPlan(createRolePlan, context); } - public R visitDropRole(final AuthorPlan dropRolePlan, final C context) { + public R visitDropRole(final AuthorTreePlan dropRolePlan, final C context) { return visitPlan(dropRolePlan, context); } - public R visitGrantRole(final AuthorPlan grantRolePlan, final C context) { + public R visitGrantRole(final AuthorTreePlan grantRolePlan, final C context) { return visitPlan(grantRolePlan, context); } - public R visitRevokeRole(final AuthorPlan revokeRolePlan, final C context) { + public R visitRevokeRole(final AuthorTreePlan revokeRolePlan, final C context) { return visitPlan(revokeRolePlan, context); } - public R visitGrantRoleToUser(final AuthorPlan grantRoleToUserPlan, final C context) { + public R visitGrantRoleToUser(final AuthorTreePlan grantRoleToUserPlan, final C context) { return visitPlan(grantRoleToUserPlan, context); } - public R visitRevokeRoleFromUser(final AuthorPlan revokeRoleFromUserPlan, final C context) { + public R visitRevokeRoleFromUser(final AuthorTreePlan revokeRoleFromUserPlan, final C context) { return visitPlan(revokeRoleFromUserPlan, context); } + public R visitRCreateUser(final AuthorRelationalPlan rCreateUserPlan, final C context) { + return visitPlan(rCreateUserPlan, context); + } + + public R visitRCreateRole(final AuthorRelationalPlan rCreateRolePlan, final C context) { + return visitPlan(rCreateRolePlan, context); + } + + public R visitRUpdateUser(final AuthorRelationalPlan rUpdateUserPlan, final C context) { + return visitPlan(rUpdateUserPlan, context); + } + + public R visitRDropUserPlan(final AuthorRelationalPlan rDropUserPlan, final C context) { + return visitPlan(rDropUserPlan, context); + } + + public R visitRDropRolePlan(final AuthorRelationalPlan rDropRolePlan, final C context) { + return visitPlan(rDropRolePlan, context); + } + + public R visitRGrantUserRole(final AuthorRelationalPlan rGrantUserRolePlan, final C context) { + return visitPlan(rGrantUserRolePlan, context); + } + + public R visitRRevokeUserRole(final AuthorRelationalPlan rRevokeUserRolePlan, final C context) { + return visitPlan(rRevokeUserRolePlan, context); + } + + public R visitRGrantUserAny(final AuthorRelationalPlan rGrantUserAnyPlan, final C context) { + return visitPlan(rGrantUserAnyPlan, context); + } + + public R visitRGrantRoleAny(final AuthorRelationalPlan rGrantRoleAnyPlan, final C context) { + return visitPlan(rGrantRoleAnyPlan, context); + } + + public R visitRGrantUserAll(final AuthorRelationalPlan rGrantUserAllPlan, final C context) { + return visitPlan(rGrantUserAllPlan, context); + } + + public R visitRGrantRoleAll(final AuthorRelationalPlan rGrantRoleAllPlan, final C context) { + return visitPlan(rGrantRoleAllPlan, context); + } + + public R visitRGrantUserDB(final AuthorRelationalPlan rGrantUserDBPlan, final C context) { + return visitPlan(rGrantUserDBPlan, context); + } + + public R visitRGrantUserTB(final AuthorRelationalPlan rGrantUserTBPlan, final C context) { + return visitPlan(rGrantUserTBPlan, context); + } + + public R visitRGrantRoleDB(final AuthorRelationalPlan rGrantRoleDBPlan, final C context) { + return visitPlan(rGrantRoleDBPlan, context); + } + + public R visitRGrantRoleTB(final AuthorRelationalPlan rGrantRoleTBPlan, final C context) { + return visitPlan(rGrantRoleTBPlan, context); + } + + public R visitRRevokeUserAny(final AuthorRelationalPlan rRevokeUserAnyPlan, final C context) { + return visitPlan(rRevokeUserAnyPlan, context); + } + + public R visitRRevokeRoleAny(final AuthorRelationalPlan rRevokeRoleAnyPlan, final C context) { + return visitPlan(rRevokeRoleAnyPlan, context); + } + + public R visitRRevokeUserAll(final AuthorRelationalPlan rRevokeUserAllPlan, final C context) { + return visitPlan(rRevokeUserAllPlan, context); + } + + public R visitRRevokeRoleAll(final AuthorRelationalPlan rRevokeRoleAllPlan, final C context) { + return visitPlan(rRevokeRoleAllPlan, context); + } + + public R visitRRevokeUserDBPrivilege( + final AuthorRelationalPlan rRevokeUserDBPrivilegePlan, final C context) { + return visitPlan(rRevokeUserDBPrivilegePlan, context); + } + + public R visitRRevokeUserTBPrivilege( + final AuthorRelationalPlan rRevokeUserTBPrivilegePlan, final C context) { + return visitPlan(rRevokeUserTBPrivilegePlan, context); + } + + public R visitRRevokeRoleDBPrivilege( + final AuthorRelationalPlan rRevokeRoleTBPrivilegePlan, final C context) { + return visitPlan(rRevokeRoleTBPrivilegePlan, context); + } + + public R visitRRevokeRoleTBPrivilege( + final AuthorRelationalPlan rRevokeRoleTBPrivilegePlan, final C context) { + return visitPlan(rRevokeRoleTBPrivilegePlan, context); + } + + public R visitRGrantUserSysPrivilege( + final AuthorRelationalPlan rGrantUserSysPrivilegePlan, final C context) { + return visitPlan(rGrantUserSysPrivilegePlan, context); + } + + public R visitRGrantRoleSysPrivilege( + final AuthorRelationalPlan rGrantRoleSysPrivilegePlan, final C context) { + return visitPlan(rGrantRoleSysPrivilegePlan, context); + } + + public R visitRRevokeUserSysPrivilege( + final AuthorRelationalPlan rRevokeUserSysPrivilegePlan, final C context) { + return visitPlan(rRevokeUserSysPrivilegePlan, context); + } + + public R visitRRevokeRoleSysPrivilege( + final AuthorRelationalPlan rRevokeRoleSysPrivilegePlan, final C context) { + return visitPlan(rRevokeRoleSysPrivilegePlan, context); + } + public R visitTTL(final SetTTLPlan setTTLPlan, final C context) { return visitPlan(setTTLPlan, context); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/PermissionManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/PermissionManager.java index 4d31ae8961e8..bab7b08c25e4 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/PermissionManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/PermissionManager.java @@ -25,6 +25,7 @@ import org.apache.iotdb.commons.auth.entity.PrivilegeUnion; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; import org.apache.iotdb.confignode.consensus.request.write.auth.AuthorPlan; +import org.apache.iotdb.confignode.consensus.request.write.pipe.payload.PipeEnrichedPlan; import org.apache.iotdb.confignode.consensus.response.auth.PermissionInfoResp; import org.apache.iotdb.confignode.manager.consensus.ConsensusManager; import org.apache.iotdb.confignode.persistence.AuthorInfo; @@ -66,7 +67,9 @@ public TSStatus operatePermission(AuthorPlan authorPlan, boolean isGeneratedByPi if (authorPlan.getAuthorType() == ConfigPhysicalPlanType.CreateUser || authorPlan.getAuthorType() == ConfigPhysicalPlanType.CreateRole || authorPlan.getAuthorType() == ConfigPhysicalPlanType.CreateUserWithRawPassword) { - tsStatus = getConsensusManager().write(authorPlan); + tsStatus = + getConsensusManager() + .write(isGeneratedByPipe ? new PipeEnrichedPlan(authorPlan) : authorPlan); } else { List allDataNodes = configManager.getNodeManager().getRegisteredDataNodes(); @@ -76,7 +79,7 @@ public TSStatus operatePermission(AuthorPlan authorPlan, boolean isGeneratedByPi .operateAuthPlan(authorPlan, allDataNodes, isGeneratedByPipe); } return tsStatus; - } catch (ConsensusException e) { + } catch (final ConsensusException e) { LOGGER.warn("Failed in the write API executing the consensus layer due to: ", e); TSStatus res = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); res.setMessage(e.getMessage()); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java index 2b2acd14019b..002fcf859ffc 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java @@ -19,6 +19,7 @@ package org.apache.iotdb.confignode.manager; +import org.apache.iotdb.common.rpc.thrift.Model; import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId; import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType; import org.apache.iotdb.common.rpc.thrift.TDataNodeConfiguration; @@ -140,6 +141,7 @@ import org.apache.iotdb.consensus.ConsensusFactory; import org.apache.iotdb.db.exception.BatchProcessException; import org.apache.iotdb.db.schemaengine.template.Template; +import org.apache.iotdb.db.utils.constant.SqlConstant; import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.rpc.TSStatusCode; @@ -714,7 +716,8 @@ private TSStatus checkMigrateRegion( Arrays.asList( new Pair<>("Original DataNode", originalDataNode), new Pair<>("Destination DataNode", destDataNode), - new Pair<>("Coordinator for add peer", coordinatorForAddPeer))); + new Pair<>("Coordinator for add peer", coordinatorForAddPeer)), + migrateRegionReq.getModel()); if (configManager .getPartitionManager() .getAllReplicaSets(originalDataNode.getDataNodeId()) @@ -755,7 +758,8 @@ private TSStatus checkReconstructRegion( targetDataNode, Arrays.asList( new Pair<>("Target DataNode", targetDataNode), - new Pair<>("Coordinator", coordinator))); + new Pair<>("Coordinator", coordinator)), + req.getModel()); ConfigNodeConfig conf = ConfigNodeDescriptor.getInstance().getConf(); if (configManager @@ -796,7 +800,8 @@ private TSStatus checkExtendRegion( targetDataNode, Arrays.asList( new Pair<>("Target DataNode", targetDataNode), - new Pair<>("Coordinator", coordinator))); + new Pair<>("Coordinator", coordinator)), + req.getModel()); if (configManager .getPartitionManager() .getAllReplicaSets(targetDataNode.getDataNodeId()) @@ -828,9 +833,9 @@ private TSStatus checkRemoveRegion( targetDataNode, Arrays.asList( new Pair<>("Target DataNode", targetDataNode), - new Pair<>("Coordinator", coordinator))); + new Pair<>("Coordinator", coordinator)), + req.getModel()); - ConfigNodeConfig conf = ConfigNodeDescriptor.getInstance().getConf(); if (configManager .getPartitionManager() .getAllReplicaSetsMap(regionId.getType()) @@ -869,10 +874,11 @@ private TSStatus checkRemoveRegion( private String regionOperationCommonCheck( TConsensusGroupId regionId, TDataNodeLocation targetDataNode, - List> relatedDataNodes) { + List> relatedDataNodes, + Model model) { String failMessage; - ConfigNodeConfig conf = ConfigNodeDescriptor.getInstance().getConf(); + if (TConsensusGroupType.DataRegion == regionId.getType() && ConsensusFactory.SIMPLE_CONSENSUS.equals(conf.getDataRegionConsensusProtocolClass())) { failMessage = "SimpleConsensus not supports region operation."; @@ -899,6 +905,8 @@ private String regionOperationCommonCheck( } else if ((failMessage = checkRegionOperationWithRemoveDataNode(regionId, targetDataNode)) != null) { // need to do nothing more + } else if ((failMessage = checkRegionOperationModelCorrectness(regionId, model)) != null) { + // need to do nothing more } return failMessage; @@ -963,6 +971,20 @@ private String checkRegionOperationDuplication(TConsensusGroupId regionId) { return null; } + private String checkRegionOperationModelCorrectness(TConsensusGroupId regionId, Model model) { + String databaseName = configManager.getPartitionManager().getRegionDatabase(regionId); + boolean isTreeModelDatabase = databaseName.startsWith(SqlConstant.TREE_MODEL_DATABASE_PREFIX); + if (Model.TREE == model && isTreeModelDatabase + || Model.TABLE == model && !isTreeModelDatabase) { + return null; + } + return String.format( + "The region's database %s is belong to %s model, but the model you are operating is %s", + databaseName, + isTreeModelDatabase ? Model.TREE.toString() : Model.TABLE.toString(), + model.toString()); + } + // end region public TSStatus migrateRegion(TMigrateRegionReq migrateRegionReq) { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/event/PipeConfigRegionSnapshotEvent.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/event/PipeConfigRegionSnapshotEvent.java index b37bfde0baa0..eda703f6e512 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/event/PipeConfigRegionSnapshotEvent.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/event/PipeConfigRegionSnapshotEvent.java @@ -67,14 +67,24 @@ public class PipeConfigRegionSnapshotEvent extends PipeSnapshotEvent new HashSet<>( Arrays.asList( ConfigPhysicalPlanType.CreateRole.getPlanType(), - ConfigPhysicalPlanType.GrantRole.getPlanType())))); + ConfigPhysicalPlanType.GrantRole.getPlanType(), + ConfigPhysicalPlanType.RGrantRoleDBPriv.getPlanType(), + ConfigPhysicalPlanType.RGrantRoleTBPriv.getPlanType(), + ConfigPhysicalPlanType.RGrantRoleAny.getPlanType(), + ConfigPhysicalPlanType.RGrantRoleSysPri.getPlanType(), + ConfigPhysicalPlanType.RGrantRoleAll.getPlanType())))); SNAPSHOT_FILE_TYPE_2_CONFIG_PHYSICAL_PLAN_TYPE_MAP.put( CNSnapshotFileType.USER, Collections.unmodifiableSet( new HashSet<>( Arrays.asList( ConfigPhysicalPlanType.CreateUserWithRawPassword.getPlanType(), - ConfigPhysicalPlanType.GrantUser.getPlanType())))); + ConfigPhysicalPlanType.GrantUser.getPlanType(), + ConfigPhysicalPlanType.RGrantUserDBPriv.getPlanType(), + ConfigPhysicalPlanType.RGrantUserTBPriv.getPlanType(), + ConfigPhysicalPlanType.RGrantUserAny.getPlanType(), + ConfigPhysicalPlanType.RGrantUserSysPri.getPlanType(), + ConfigPhysicalPlanType.RGrantUserAll.getPlanType())))); SNAPSHOT_FILE_TYPE_2_CONFIG_PHYSICAL_PLAN_TYPE_MAP.put( CNSnapshotFileType.USER_ROLE, Collections.singleton(ConfigPhysicalPlanType.GrantRoleToUser.getPlanType())); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/extractor/ConfigRegionListeningFilter.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/extractor/ConfigRegionListeningFilter.java index 0c87b8d60e74..5f0e2889d9fc 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/extractor/ConfigRegionListeningFilter.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/extractor/ConfigRegionListeningFilter.java @@ -41,8 +41,8 @@ import static org.apache.iotdb.commons.pipe.datastructure.options.PipeInclusionOptions.getExclusionString; import static org.apache.iotdb.commons.pipe.datastructure.options.PipeInclusionOptions.getInclusionString; import static org.apache.iotdb.commons.pipe.datastructure.options.PipeInclusionOptions.parseOptions; -import static org.apache.iotdb.commons.pipe.datastructure.options.PipeInclusionOptions.tableOnlySyncPrefix; -import static org.apache.iotdb.commons.pipe.datastructure.options.PipeInclusionOptions.treeOnlySyncPrefix; +import static org.apache.iotdb.commons.pipe.datastructure.options.PipeInclusionOptions.tableOnlySyncPrefixes; +import static org.apache.iotdb.commons.pipe.datastructure.options.PipeInclusionOptions.treeOnlySyncPrefixes; /** * {@link ConfigRegionListeningFilter} is to classify the {@link ConfigPhysicalPlan}s to help {@link @@ -122,16 +122,45 @@ public class ConfigRegionListeningFilter { OPTION_PLAN_MAP.put( new PartialPath("auth.role.create"), - Collections.singletonList(ConfigPhysicalPlanType.CreateRole)); + Collections.unmodifiableList( + Arrays.asList( + ConfigPhysicalPlanType.CreateRole, ConfigPhysicalPlanType.RCreateRole))); OPTION_PLAN_MAP.put( new PartialPath("auth.role.drop"), - Collections.singletonList(ConfigPhysicalPlanType.DropRole)); + Collections.unmodifiableList( + Arrays.asList(ConfigPhysicalPlanType.DropRole, ConfigPhysicalPlanType.RDropRole))); + + // Both OPTION_PLAN_MAP.put( new PartialPath("auth.role.grant"), - Collections.singletonList(ConfigPhysicalPlanType.GrantRole)); + Collections.unmodifiableList( + Arrays.asList( + ConfigPhysicalPlanType.GrantRole, + ConfigPhysicalPlanType.RGrantRoleAll, + ConfigPhysicalPlanType.RGrantRoleSysPri))); OPTION_PLAN_MAP.put( new PartialPath("auth.role.revoke"), - Collections.singletonList(ConfigPhysicalPlanType.RevokeRole)); + Collections.unmodifiableList( + Arrays.asList( + ConfigPhysicalPlanType.RevokeRole, + ConfigPhysicalPlanType.RRevokeRoleAll, + ConfigPhysicalPlanType.RRevokeRoleSysPri))); + + // Table + OPTION_PLAN_MAP.put( + new PartialPath("auth.role.grant.table"), + Collections.unmodifiableList( + Arrays.asList( + ConfigPhysicalPlanType.RGrantRoleAny, + ConfigPhysicalPlanType.RGrantRoleDBPriv, + ConfigPhysicalPlanType.RGrantRoleTBPriv))); + OPTION_PLAN_MAP.put( + new PartialPath("auth.role.revoke.table"), + Collections.unmodifiableList( + Arrays.asList( + ConfigPhysicalPlanType.RRevokeRoleAny, + ConfigPhysicalPlanType.RRevokeRoleDBPriv, + ConfigPhysicalPlanType.RRevokeRoleTBPriv))); OPTION_PLAN_MAP.put( new PartialPath("auth.user.create"), @@ -141,20 +170,49 @@ public class ConfigRegionListeningFilter { ConfigPhysicalPlanType.CreateUserWithRawPassword))); OPTION_PLAN_MAP.put( new PartialPath("auth.user.alter"), - Collections.singletonList(ConfigPhysicalPlanType.UpdateUser)); + Collections.unmodifiableList( + Arrays.asList( + ConfigPhysicalPlanType.UpdateUser, ConfigPhysicalPlanType.RUpdateUser))); OPTION_PLAN_MAP.put( new PartialPath("auth.user.drop"), - Collections.singletonList(ConfigPhysicalPlanType.DropUser)); + Collections.unmodifiableList( + Arrays.asList(ConfigPhysicalPlanType.DropUser, ConfigPhysicalPlanType.RUpdateUser))); + + // Both OPTION_PLAN_MAP.put( new PartialPath("auth.user.grant"), Collections.unmodifiableList( Arrays.asList( - ConfigPhysicalPlanType.GrantUser, ConfigPhysicalPlanType.GrantRoleToUser))); + ConfigPhysicalPlanType.GrantUser, + ConfigPhysicalPlanType.GrantRoleToUser, + ConfigPhysicalPlanType.RGrantUserRole, + ConfigPhysicalPlanType.RGrantUserAll, + ConfigPhysicalPlanType.RGrantUserSysPri))); OPTION_PLAN_MAP.put( new PartialPath("auth.user.revoke"), Collections.unmodifiableList( Arrays.asList( - ConfigPhysicalPlanType.RevokeUser, ConfigPhysicalPlanType.RevokeRoleFromUser))); + ConfigPhysicalPlanType.RevokeUser, + ConfigPhysicalPlanType.RevokeRoleFromUser, + ConfigPhysicalPlanType.RRevokeUserRole, + ConfigPhysicalPlanType.RGrantUserAll, + ConfigPhysicalPlanType.RGrantUserSysPri))); + + // Table + OPTION_PLAN_MAP.put( + new PartialPath("auth.user.grant.table"), + Collections.unmodifiableList( + Arrays.asList( + ConfigPhysicalPlanType.RGrantUserAny, + ConfigPhysicalPlanType.RGrantUserDBPriv, + ConfigPhysicalPlanType.RGrantUserTBPriv))); + OPTION_PLAN_MAP.put( + new PartialPath("auth.user.revoke.table"), + Collections.unmodifiableList( + Arrays.asList( + ConfigPhysicalPlanType.RRevokeUserAny, + ConfigPhysicalPlanType.RRevokeUserDBPriv, + ConfigPhysicalPlanType.RRevokeUserTBPriv))); } catch (final IllegalPathException ignore) { // There won't be any exceptions here } @@ -185,11 +243,11 @@ public static Set parseListeningPlanTypeSet( exclusionOptions.forEach(exclusion -> planTypes.removeAll(getOptionsByPrefix(exclusion))); if (!TreePattern.isTreeModelDataAllowToBeCaptured(parameters)) { - planTypes.removeAll(getOptionsByPrefix(treeOnlySyncPrefix)); + treeOnlySyncPrefixes.forEach(prefix -> planTypes.removeAll(getOptionsByPrefix(prefix))); } if (!TablePattern.isTableModelDataAllowToBeCaptured(parameters)) { - planTypes.removeAll(getOptionsByPrefix(tableOnlySyncPrefix)); + tableOnlySyncPrefixes.forEach(prefix -> planTypes.removeAll(getOptionsByPrefix(prefix))); } return planTypes; @@ -197,7 +255,7 @@ public static Set parseListeningPlanTypeSet( private static Set getOptionsByPrefix(final PartialPath prefix) { return OPTION_PLAN_MAP.keySet().stream() - .filter(path -> path.overlapWithFullPathPrefix(prefix)) + .filter(path -> path.matchPrefixPath(prefix)) .map(OPTION_PLAN_MAP::get) .flatMap(Collection::stream) .collect(Collectors.toSet()); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/extractor/IoTDBConfigRegionExtractor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/extractor/IoTDBConfigRegionExtractor.java index 9387d67bebef..2795879dcce8 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/extractor/IoTDBConfigRegionExtractor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/extractor/IoTDBConfigRegionExtractor.java @@ -62,6 +62,10 @@ public class IoTDBConfigRegionExtractor extends IoTDBNonDataRegionExtractor { new PipeConfigPhysicalPlanTreePatternParseVisitor(); public static final PipeConfigPhysicalPlanTablePatternParseVisitor TABLE_PATTERN_PARSE_VISITOR = new PipeConfigPhysicalPlanTablePatternParseVisitor(); + public static final PipeConfigPhysicalPlanTreeScopeParseVisitor TREE_SCOPE_PARSE_VISITOR = + new PipeConfigPhysicalPlanTreeScopeParseVisitor(); + public static final PipeConfigPhysicalPlanTableScopeParseVisitor TABLE_SCOPE_PARSE_VISITOR = + new PipeConfigPhysicalPlanTableScopeParseVisitor(); private Set listenedTypeSet = new HashSet<>(); @@ -149,8 +153,23 @@ public static Optional parseConfigPlan( return result; } } - if (!Boolean.FALSE.equals(isTableDatabasePlan)) { - result = TABLE_PATTERN_PARSE_VISITOR.process(plan, tablePattern); + if (!Boolean.FALSE.equals(isTableDatabasePlan) && result.isPresent()) { + result = TABLE_PATTERN_PARSE_VISITOR.process(result.get(), tablePattern); + if (!result.isPresent()) { + return result; + } + } + if (!treePattern.isTreeModelDataAllowedToBeCaptured() && result.isPresent()) { + result = TREE_SCOPE_PARSE_VISITOR.process(result.get(), null); + if (!result.isPresent()) { + return result; + } + } + if (!tablePattern.isTableModelDataAllowedToBeCaptured() && result.isPresent()) { + result = TABLE_SCOPE_PARSE_VISITOR.process(result.get(), null); + if (!result.isPresent()) { + return result; + } } return result; } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/extractor/PipeConfigPhysicalPlanTablePatternParseVisitor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/extractor/PipeConfigPhysicalPlanTablePatternParseVisitor.java index 91be75a3bb5b..679614ac458c 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/extractor/PipeConfigPhysicalPlanTablePatternParseVisitor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/extractor/PipeConfigPhysicalPlanTablePatternParseVisitor.java @@ -23,6 +23,7 @@ import org.apache.iotdb.commons.utils.PathUtils; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanVisitor; +import org.apache.iotdb.confignode.consensus.request.write.auth.AuthorRelationalPlan; import org.apache.iotdb.confignode.consensus.request.write.database.DatabaseSchemaPlan; import org.apache.iotdb.confignode.consensus.request.write.database.DeleteDatabasePlan; import org.apache.iotdb.confignode.consensus.request.write.pipe.payload.PipeCreateTablePlan; @@ -132,4 +133,65 @@ private boolean matchDatabaseAndTableName( return pattern.matchesDatabase(PathUtils.unQualifyDatabaseName(database)) && pattern.matchesTable(tableName); } + + @Override + public Optional visitRGrantUserDB( + final AuthorRelationalPlan plan, final TablePattern pattern) { + return visitAuthorDBPlan(plan, pattern); + } + + @Override + public Optional visitRGrantRoleDB( + final AuthorRelationalPlan plan, final TablePattern pattern) { + return visitAuthorDBPlan(plan, pattern); + } + + @Override + public Optional visitRRevokeUserDBPrivilege( + final AuthorRelationalPlan plan, final TablePattern pattern) { + return visitAuthorDBPlan(plan, pattern); + } + + @Override + public Optional visitRRevokeRoleDBPrivilege( + final AuthorRelationalPlan plan, final TablePattern pattern) { + return visitAuthorDBPlan(plan, pattern); + } + + private Optional visitAuthorDBPlan( + final AuthorRelationalPlan plan, final TablePattern pattern) { + return pattern.matchesDatabase(plan.getDatabaseName()) ? Optional.of(plan) : Optional.empty(); + } + + @Override + public Optional visitRGrantUserTB( + final AuthorRelationalPlan plan, final TablePattern pattern) { + return visitAuthorTBPlan(plan, pattern); + } + + @Override + public Optional visitRGrantRoleTB( + final AuthorRelationalPlan plan, final TablePattern pattern) { + return visitAuthorTBPlan(plan, pattern); + } + + @Override + public Optional visitRRevokeUserTBPrivilege( + final AuthorRelationalPlan plan, final TablePattern pattern) { + return visitAuthorTBPlan(plan, pattern); + } + + @Override + public Optional visitRRevokeRoleTBPrivilege( + final AuthorRelationalPlan plan, final TablePattern pattern) { + return visitAuthorTBPlan(plan, pattern); + } + + private Optional visitAuthorTBPlan( + final AuthorRelationalPlan plan, final TablePattern pattern) { + return pattern.matchesDatabase(plan.getDatabaseName()) + && pattern.matchesTable(plan.getTableName()) + ? Optional.of(plan) + : Optional.empty(); + } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/extractor/PipeConfigPhysicalPlanTableScopeParseVisitor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/extractor/PipeConfigPhysicalPlanTableScopeParseVisitor.java new file mode 100644 index 000000000000..e5673c0dafd4 --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/extractor/PipeConfigPhysicalPlanTableScopeParseVisitor.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.manager.pipe.extractor; + +import org.apache.iotdb.commons.auth.entity.PrivilegeType; +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanVisitor; +import org.apache.iotdb.confignode.consensus.request.write.auth.AuthorRelationalPlan; +import org.apache.iotdb.confignode.consensus.request.write.auth.AuthorTreePlan; + +import java.util.Arrays; +import java.util.Collections; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +public class PipeConfigPhysicalPlanTableScopeParseVisitor + extends ConfigPhysicalPlanVisitor, Void> { + @Override + public Optional visitPlan(final ConfigPhysicalPlan plan, final Void context) { + return Optional.of(plan); + } + + @Override + public Optional visitRGrantUserAll( + final AuthorRelationalPlan plan, final Void context) { + return visitTableAuthorPlan(plan, ConfigPhysicalPlanType.GrantUser); + } + + @Override + public Optional visitRGrantRoleAll( + final AuthorRelationalPlan plan, final Void context) { + return visitTableAuthorPlan(plan, ConfigPhysicalPlanType.GrantRole); + } + + @Override + public Optional visitRRevokeUserAll( + final AuthorRelationalPlan plan, final Void context) { + return visitTableAuthorPlan(plan, ConfigPhysicalPlanType.RevokeUser); + } + + @Override + public Optional visitRRevokeRoleAll( + final AuthorRelationalPlan plan, final Void context) { + return visitTableAuthorPlan(plan, ConfigPhysicalPlanType.RevokeRole); + } + + private Optional visitTableAuthorPlan( + final AuthorRelationalPlan authorRelationalPlan, final ConfigPhysicalPlanType type) { + final Set permissions = + Arrays.stream(PrivilegeType.values()) + .filter(PrivilegeType::forRelationalSys) + .map(Enum::ordinal) + .collect(Collectors.toSet()); + return !permissions.isEmpty() + ? Optional.of( + new AuthorTreePlan( + type, + authorRelationalPlan.getUserName(), + authorRelationalPlan.getRoleName(), + authorRelationalPlan.getPassword(), + authorRelationalPlan.getNewPassword(), + permissions, + authorRelationalPlan.getGrantOpt(), + Collections.emptyList())) + : Optional.empty(); + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/extractor/PipeConfigPhysicalPlanTreePatternParseVisitor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/extractor/PipeConfigPhysicalPlanTreePatternParseVisitor.java index f5e5888ed7d5..344a61c2304f 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/extractor/PipeConfigPhysicalPlanTreePatternParseVisitor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/extractor/PipeConfigPhysicalPlanTreePatternParseVisitor.java @@ -25,7 +25,6 @@ import org.apache.iotdb.commons.pipe.datastructure.pattern.IoTDBTreePattern; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanVisitor; -import org.apache.iotdb.confignode.consensus.request.write.auth.AuthorPlan; import org.apache.iotdb.confignode.consensus.request.write.auth.AuthorTreePlan; import org.apache.iotdb.confignode.consensus.request.write.database.DatabaseSchemaPlan; import org.apache.iotdb.confignode.consensus.request.write.database.DeleteDatabasePlan; @@ -166,52 +165,51 @@ public Optional visitExtendSchemaTemplate( @Override public Optional visitGrantUser( - final AuthorPlan grantUserPlan, final IoTDBTreePattern pattern) { + final AuthorTreePlan grantUserPlan, final IoTDBTreePattern pattern) { return visitTreeAuthorPlan(grantUserPlan, pattern); } @Override public Optional visitRevokeUser( - final AuthorPlan revokeUserPlan, final IoTDBTreePattern pattern) { + final AuthorTreePlan revokeUserPlan, final IoTDBTreePattern pattern) { return visitTreeAuthorPlan(revokeUserPlan, pattern); } @Override public Optional visitGrantRole( - final AuthorPlan revokeUserPlan, final IoTDBTreePattern pattern) { + final AuthorTreePlan revokeUserPlan, final IoTDBTreePattern pattern) { return visitTreeAuthorPlan(revokeUserPlan, pattern); } @Override public Optional visitRevokeRole( - final AuthorPlan revokeUserPlan, final IoTDBTreePattern pattern) { + final AuthorTreePlan revokeUserPlan, final IoTDBTreePattern pattern) { return visitTreeAuthorPlan(revokeUserPlan, pattern); } private Optional visitTreeAuthorPlan( - final AuthorPlan pathRelatedAuthorPlan, final IoTDBTreePattern pattern) { - AuthorTreePlan plan = (AuthorTreePlan) pathRelatedAuthorPlan; + final AuthorTreePlan pathRelatedAuthorTreePlan, final IoTDBTreePattern pattern) { final List intersectedPaths = - plan.getNodeNameList().stream() + pathRelatedAuthorTreePlan.getNodeNameList().stream() .map(pattern::getIntersection) .flatMap(Collection::stream) .collect(Collectors.toList()); final Set permissions = !intersectedPaths.isEmpty() - ? plan.getPermissions() - : plan.getPermissions().stream() + ? pathRelatedAuthorTreePlan.getPermissions() + : pathRelatedAuthorTreePlan.getPermissions().stream() .filter(permission -> !PrivilegeType.values()[permission].isPathPrivilege()) .collect(Collectors.toSet()); return !permissions.isEmpty() ? Optional.of( new AuthorTreePlan( - plan.getAuthorType(), - plan.getUserName(), - plan.getRoleName(), - plan.getPassword(), - plan.getNewPassword(), + pathRelatedAuthorTreePlan.getAuthorType(), + pathRelatedAuthorTreePlan.getUserName(), + pathRelatedAuthorTreePlan.getRoleName(), + pathRelatedAuthorTreePlan.getPassword(), + pathRelatedAuthorTreePlan.getNewPassword(), permissions, - plan.getGrantOpt(), + pathRelatedAuthorTreePlan.getGrantOpt(), intersectedPaths)) : Optional.empty(); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/extractor/PipeConfigPhysicalPlanTreeScopeParseVisitor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/extractor/PipeConfigPhysicalPlanTreeScopeParseVisitor.java new file mode 100644 index 000000000000..175b836532f7 --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/extractor/PipeConfigPhysicalPlanTreeScopeParseVisitor.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.manager.pipe.extractor; + +import org.apache.iotdb.commons.auth.entity.PrivilegeType; +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanVisitor; +import org.apache.iotdb.confignode.consensus.request.write.auth.AuthorTreePlan; + +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +public class PipeConfigPhysicalPlanTreeScopeParseVisitor + extends ConfigPhysicalPlanVisitor, Void> { + @Override + public Optional visitPlan(final ConfigPhysicalPlan plan, final Void context) { + return Optional.of(plan); + } + + @Override + public Optional visitGrantRole( + final AuthorTreePlan grantRolePlan, final Void context) { + return visitTreeAuthorPlan(grantRolePlan); + } + + @Override + public Optional visitGrantUser( + final AuthorTreePlan grantUserPlan, final Void context) { + return visitTreeAuthorPlan(grantUserPlan); + } + + @Override + public Optional visitRevokeUser( + final AuthorTreePlan revokeUserPlan, final Void context) { + return visitTreeAuthorPlan(revokeUserPlan); + } + + @Override + public Optional visitRevokeRole( + final AuthorTreePlan revokeRolePlan, final Void context) { + return visitTreeAuthorPlan(revokeRolePlan); + } + + private Optional visitTreeAuthorPlan(final AuthorTreePlan authorTreePlan) { + final Set permissions = + authorTreePlan.getPermissions().stream() + .filter(permission -> PrivilegeType.values()[permission].forRelationalSys()) + .collect(Collectors.toSet()); + return !permissions.isEmpty() + ? Optional.of( + new AuthorTreePlan( + authorTreePlan.getAuthorType(), + authorTreePlan.getUserName(), + authorTreePlan.getRoleName(), + authorTreePlan.getPassword(), + authorTreePlan.getNewPassword(), + permissions, + authorTreePlan.getGrantOpt(), + authorTreePlan.getNodeNameList())) + : Optional.empty(); + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/receiver/protocol/IoTDBConfigNodeReceiver.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/receiver/protocol/IoTDBConfigNodeReceiver.java index 48d9a6f2f167..15da365f57ee 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/receiver/protocol/IoTDBConfigNodeReceiver.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/receiver/protocol/IoTDBConfigNodeReceiver.java @@ -41,6 +41,7 @@ import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; import org.apache.iotdb.confignode.consensus.request.write.auth.AuthorPlan; +import org.apache.iotdb.confignode.consensus.request.write.auth.AuthorRelationalPlan; import org.apache.iotdb.confignode.consensus.request.write.auth.AuthorTreePlan; import org.apache.iotdb.confignode.consensus.request.write.database.DatabaseSchemaPlan; import org.apache.iotdb.confignode.consensus.request.write.database.DeleteDatabasePlan; @@ -317,6 +318,35 @@ private TSStatus checkPermission(final ConfigPhysicalPlan plan) { return configManager .checkUserPrivileges(username, new PrivilegeUnion(PrivilegeType.USE_TRIGGER)) .getStatus(); + case PipeCreateTable: + return configManager + .checkUserPrivileges( + username, + new PrivilegeUnion( + ((PipeCreateTablePlan) plan).getDatabase(), + ((PipeCreateTablePlan) plan).getTable().getTableName(), + PrivilegeType.CREATE)) + .getStatus(); + case AddTableColumn: + case SetTableProperties: + case CommitDeleteColumn: + return configManager + .checkUserPrivileges( + username, + new PrivilegeUnion( + ((PipeCreateTablePlan) plan).getDatabase(), + ((PipeCreateTablePlan) plan).getTable().getTableName(), + PrivilegeType.ALTER)) + .getStatus(); + case CommitDeleteTable: + return configManager + .checkUserPrivileges( + username, + new PrivilegeUnion( + ((PipeCreateTablePlan) plan).getDatabase(), + ((PipeCreateTablePlan) plan).getTable().getTableName(), + PrivilegeType.DROP)) + .getStatus(); case GrantRole: case GrantUser: case RevokeUser: @@ -338,22 +368,122 @@ private TSStatus checkPermission(final ConfigPhysicalPlan plan) { } } return StatusUtils.OK; + case RGrantUserAny: + case RGrantRoleAny: + case RRevokeUserAny: + case RRevokeRoleAny: + for (final int permission : ((AuthorRelationalPlan) plan).getPermissions()) { + final TSStatus status = + configManager + .checkUserPrivileges( + username, new PrivilegeUnion(PrivilegeType.values()[permission], true, true)) + .getStatus(); + if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + return status; + } + } + return StatusUtils.OK; + case RGrantUserAll: + case RGrantRoleAll: + case RRevokeUserAll: + case RRevokeRoleAll: + for (PrivilegeType privilegeType : PrivilegeType.values()) { + final TSStatus status; + if (privilegeType.isRelationalPrivilege()) { + status = + configManager + .checkUserPrivileges(username, new PrivilegeUnion(privilegeType, true, true)) + .getStatus(); + } else if (privilegeType.forRelationalSys()) { + status = + configManager + .checkUserPrivileges(username, new PrivilegeUnion(privilegeType, true)) + .getStatus(); + } else { + continue; + } + if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + return status; + } + } + return StatusUtils.OK; + case RGrantUserDBPriv: + case RGrantRoleDBPriv: + case RRevokeUserDBPriv: + case RRevokeRoleDBPriv: + for (final int permission : ((AuthorRelationalPlan) plan).getPermissions()) { + final TSStatus status = + configManager + .checkUserPrivileges( + username, + new PrivilegeUnion( + ((AuthorRelationalPlan) plan).getDatabaseName(), + PrivilegeType.values()[permission], + true)) + .getStatus(); + if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + return status; + } + } + return StatusUtils.OK; + case RGrantUserTBPriv: + case RGrantRoleTBPriv: + case RRevokeUserTBPriv: + case RRevokeRoleTBPriv: + for (final int permission : ((AuthorRelationalPlan) plan).getPermissions()) { + final TSStatus status = + configManager + .checkUserPrivileges( + username, + new PrivilegeUnion( + ((AuthorRelationalPlan) plan).getDatabaseName(), + ((AuthorRelationalPlan) plan).getTableName(), + PrivilegeType.values()[permission], + true)) + .getStatus(); + if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + return status; + } + } + return StatusUtils.OK; + case RGrantUserSysPri: + case RGrantRoleSysPri: + case RRevokeUserSysPri: + case RRevokeRoleSysPri: + for (final int permission : ((AuthorRelationalPlan) plan).getPermissions()) { + final TSStatus status = + configManager + .checkUserPrivileges( + username, new PrivilegeUnion(PrivilegeType.values()[permission], true)) + .getStatus(); + if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + return status; + } + } + return StatusUtils.OK; case UpdateUser: + case RUpdateUser: return ((AuthorPlan) plan).getUserName().equals(username) ? StatusUtils.OK : configManager .checkUserPrivileges(username, new PrivilegeUnion(PrivilegeType.MANAGE_USER)) .getStatus(); case CreateUser: + case RCreateUser: case CreateUserWithRawPassword: case DropUser: + case RDropUser: return configManager .checkUserPrivileges(username, new PrivilegeUnion(PrivilegeType.MANAGE_USER)) .getStatus(); case CreateRole: + case RCreateRole: case DropRole: + case RDropRole: case GrantRoleToUser: + case RGrantUserRole: case RevokeRoleFromUser: + case RRevokeUserRole: return configManager .checkUserPrivileges(username, new PrivilegeUnion(PrivilegeType.MANAGE_ROLE)) .getStatus(); @@ -552,6 +682,9 @@ private TSStatus executePlan(final ConfigPhysicalPlan plan) throws ConsensusExce ByteBuffer.wrap(((PipeDeleteDevicesPlan) plan).getModBytes())), true) .getStatus(); + case CreateUser: + case CreateUserWithRawPassword: + case CreateRole: case DropUser: case DropRole: case GrantRole: @@ -561,11 +694,34 @@ private TSStatus executePlan(final ConfigPhysicalPlan plan) throws ConsensusExce case RevokeRole: case RevokeRoleFromUser: case UpdateUser: + case RCreateUser: + case RCreateRole: + case RDropUser: + case RDropRole: + case RGrantRoleAll: + case RGrantUserAll: + case RRevokeRoleAll: + case RRevokeUserAll: + case RGrantRoleAny: + case RGrantUserAny: + case RRevokeUserAny: + case RRevokeRoleAny: + case RGrantRoleDBPriv: + case RGrantUserDBPriv: + case RRevokeRoleDBPriv: + case RRevokeUserDBPriv: + case RGrantRoleTBPriv: + case RGrantUserTBPriv: + case RRevokeRoleTBPriv: + case RRevokeUserTBPriv: + case RGrantRoleSysPri: + case RGrantUserSysPri: + case RRevokeRoleSysPri: + case RRevokeUserSysPri: + case RGrantUserRole: + case RRevokeUserRole: return configManager.getPermissionManager().operatePermission((AuthorPlan) plan, true); case CreateSchemaTemplate: - case CreateUser: - case CreateRole: - case CreateUserWithRawPassword: default: return configManager.getConsensusManager().write(new PipeEnrichedPlan(plan)); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/receiver/visitor/PipeConfigPhysicalPlanTSStatusVisitor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/receiver/visitor/PipeConfigPhysicalPlanTSStatusVisitor.java index 0583e32e2801..653ef4339ea9 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/receiver/visitor/PipeConfigPhysicalPlanTSStatusVisitor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/receiver/visitor/PipeConfigPhysicalPlanTSStatusVisitor.java @@ -22,7 +22,8 @@ import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanVisitor; -import org.apache.iotdb.confignode.consensus.request.write.auth.AuthorPlan; +import org.apache.iotdb.confignode.consensus.request.write.auth.AuthorRelationalPlan; +import org.apache.iotdb.confignode.consensus.request.write.auth.AuthorTreePlan; import org.apache.iotdb.confignode.consensus.request.write.database.DatabaseSchemaPlan; import org.apache.iotdb.confignode.consensus.request.write.database.DeleteDatabasePlan; import org.apache.iotdb.confignode.consensus.request.write.database.SetTTLPlan; @@ -191,7 +192,7 @@ public TSStatus visitPipeDeactivateTemplate( } @Override - public TSStatus visitCreateUser(final AuthorPlan plan, final TSStatus context) { + public TSStatus visitCreateUser(final AuthorTreePlan plan, final TSStatus context) { if (context.getCode() == TSStatusCode.USER_ALREADY_EXIST.getStatusCode()) { return new TSStatus(TSStatusCode.PIPE_RECEIVER_IDEMPOTENT_CONFLICT_EXCEPTION.getStatusCode()) .setMessage(context.getMessage()); @@ -200,7 +201,7 @@ public TSStatus visitCreateUser(final AuthorPlan plan, final TSStatus context) { } @Override - public TSStatus visitCreateRawUser(final AuthorPlan plan, final TSStatus context) { + public TSStatus visitCreateRawUser(final AuthorTreePlan plan, final TSStatus context) { if (context.getCode() == TSStatusCode.USER_ALREADY_EXIST.getStatusCode()) { return new TSStatus(TSStatusCode.PIPE_RECEIVER_IDEMPOTENT_CONFLICT_EXCEPTION.getStatusCode()) .setMessage(context.getMessage()); @@ -209,7 +210,7 @@ public TSStatus visitCreateRawUser(final AuthorPlan plan, final TSStatus context } @Override - public TSStatus visitUpdateUser(final AuthorPlan plan, final TSStatus context) { + public TSStatus visitUpdateUser(final AuthorTreePlan plan, final TSStatus context) { if (context.getCode() == TSStatusCode.USER_NOT_EXIST.getStatusCode()) { return new TSStatus(TSStatusCode.PIPE_RECEIVER_USER_CONFLICT_EXCEPTION.getStatusCode()) .setMessage(context.getMessage()); @@ -218,7 +219,7 @@ public TSStatus visitUpdateUser(final AuthorPlan plan, final TSStatus context) { } @Override - public TSStatus visitDropUser(final AuthorPlan plan, final TSStatus context) { + public TSStatus visitDropUser(final AuthorTreePlan plan, final TSStatus context) { if (context.getCode() == TSStatusCode.USER_NOT_EXIST.getStatusCode()) { return new TSStatus(TSStatusCode.PIPE_RECEIVER_IDEMPOTENT_CONFLICT_EXCEPTION.getStatusCode()) .setMessage(context.getMessage()); @@ -227,7 +228,7 @@ public TSStatus visitDropUser(final AuthorPlan plan, final TSStatus context) { } @Override - public TSStatus visitGrantUser(final AuthorPlan plan, final TSStatus context) { + public TSStatus visitGrantUser(final AuthorTreePlan plan, final TSStatus context) { if (context.getCode() == TSStatusCode.NO_PERMISSION.getStatusCode()) { // Admin user return new TSStatus(TSStatusCode.PIPE_RECEIVER_IDEMPOTENT_CONFLICT_EXCEPTION.getStatusCode()) @@ -241,7 +242,7 @@ public TSStatus visitGrantUser(final AuthorPlan plan, final TSStatus context) { } @Override - public TSStatus visitRevokeUser(final AuthorPlan plan, final TSStatus context) { + public TSStatus visitRevokeUser(final AuthorTreePlan plan, final TSStatus context) { if (context.getCode() == TSStatusCode.NOT_HAS_PRIVILEGE.getStatusCode()) { return new TSStatus(TSStatusCode.PIPE_RECEIVER_IDEMPOTENT_CONFLICT_EXCEPTION.getStatusCode()) .setMessage(context.getMessage()); @@ -255,7 +256,7 @@ public TSStatus visitRevokeUser(final AuthorPlan plan, final TSStatus context) { } @Override - public TSStatus visitCreateRole(final AuthorPlan plan, final TSStatus context) { + public TSStatus visitCreateRole(final AuthorTreePlan plan, final TSStatus context) { if (context.getCode() == TSStatusCode.ROLE_ALREADY_EXIST.getStatusCode()) { return new TSStatus(TSStatusCode.PIPE_RECEIVER_IDEMPOTENT_CONFLICT_EXCEPTION.getStatusCode()) .setMessage(context.getMessage()); @@ -264,7 +265,7 @@ public TSStatus visitCreateRole(final AuthorPlan plan, final TSStatus context) { } @Override - public TSStatus visitDropRole(final AuthorPlan plan, final TSStatus context) { + public TSStatus visitDropRole(final AuthorTreePlan plan, final TSStatus context) { if (context.getCode() == TSStatusCode.ROLE_NOT_EXIST.getStatusCode()) { return new TSStatus(TSStatusCode.PIPE_RECEIVER_IDEMPOTENT_CONFLICT_EXCEPTION.getStatusCode()) .setMessage(context.getMessage()); @@ -273,7 +274,7 @@ public TSStatus visitDropRole(final AuthorPlan plan, final TSStatus context) { } @Override - public TSStatus visitGrantRole(final AuthorPlan plan, final TSStatus context) { + public TSStatus visitGrantRole(final AuthorTreePlan plan, final TSStatus context) { if (context.getCode() == TSStatusCode.ROLE_NOT_EXIST.getStatusCode()) { return new TSStatus(TSStatusCode.PIPE_RECEIVER_USER_CONFLICT_EXCEPTION.getStatusCode()) .setMessage(context.getMessage()); @@ -282,7 +283,7 @@ public TSStatus visitGrantRole(final AuthorPlan plan, final TSStatus context) { } @Override - public TSStatus visitRevokeRole(final AuthorPlan plan, final TSStatus context) { + public TSStatus visitRevokeRole(final AuthorTreePlan plan, final TSStatus context) { if (context.getCode() == TSStatusCode.NOT_HAS_PRIVILEGE.getStatusCode()) { return new TSStatus(TSStatusCode.PIPE_RECEIVER_IDEMPOTENT_CONFLICT_EXCEPTION.getStatusCode()) .setMessage(context.getMessage()); @@ -294,7 +295,7 @@ public TSStatus visitRevokeRole(final AuthorPlan plan, final TSStatus context) { } @Override - public TSStatus visitGrantRoleToUser(final AuthorPlan plan, final TSStatus context) { + public TSStatus visitGrantRoleToUser(final AuthorTreePlan plan, final TSStatus context) { if (context.getCode() == TSStatusCode.USER_ALREADY_HAS_ROLE.getStatusCode()) { return new TSStatus(TSStatusCode.PIPE_RECEIVER_IDEMPOTENT_CONFLICT_EXCEPTION.getStatusCode()) .setMessage(context.getMessage()); @@ -303,7 +304,8 @@ public TSStatus visitGrantRoleToUser(final AuthorPlan plan, final TSStatus conte } @Override - public TSStatus visitRevokeRoleFromUser(final AuthorPlan plan, final TSStatus context) { + public TSStatus visitRevokeRoleFromUser( + final AuthorTreePlan revokeRoleFromUserPlan, final TSStatus context) { if (context.getCode() == TSStatusCode.USER_NOT_HAS_ROLE.getStatusCode()) { return new TSStatus(TSStatusCode.PIPE_RECEIVER_IDEMPOTENT_CONFLICT_EXCEPTION.getStatusCode()) .setMessage(context.getMessage()); @@ -311,7 +313,184 @@ public TSStatus visitRevokeRoleFromUser(final AuthorPlan plan, final TSStatus co return new TSStatus(TSStatusCode.PIPE_RECEIVER_USER_CONFLICT_EXCEPTION.getStatusCode()) .setMessage(context.getMessage()); } - return super.visitRevokeRoleFromUser(plan, context); + return super.visitRevokeRoleFromUser(revokeRoleFromUserPlan, context); + } + + @Override + public TSStatus visitRCreateUser( + final AuthorRelationalPlan rCreateUserPlan, final TSStatus context) { + return visitAuthorRelationalPlan(rCreateUserPlan, context); + } + + @Override + public TSStatus visitRCreateRole( + final AuthorRelationalPlan rCreateRolePlan, final TSStatus context) { + return visitAuthorRelationalPlan(rCreateRolePlan, context); + } + + @Override + public TSStatus visitRUpdateUser( + final AuthorRelationalPlan rUpdateUserPlan, final TSStatus context) { + return visitAuthorRelationalPlan(rUpdateUserPlan, context); + } + + @Override + public TSStatus visitRDropUserPlan( + final AuthorRelationalPlan rDropUserPlan, final TSStatus context) { + return visitAuthorRelationalPlan(rDropUserPlan, context); + } + + @Override + public TSStatus visitRDropRolePlan( + final AuthorRelationalPlan rDropRolePlan, final TSStatus context) { + return visitAuthorRelationalPlan(rDropRolePlan, context); + } + + @Override + public TSStatus visitRGrantUserRole( + final AuthorRelationalPlan rGrantUserRolePlan, final TSStatus context) { + return visitAuthorRelationalPlan(rGrantUserRolePlan, context); + } + + @Override + public TSStatus visitRRevokeUserRole( + final AuthorRelationalPlan rRevokeUserRolePlan, final TSStatus context) { + return visitAuthorRelationalPlan(rRevokeUserRolePlan, context); + } + + @Override + public TSStatus visitRGrantUserAny( + final AuthorRelationalPlan rGrantUserAnyPlan, final TSStatus context) { + return visitAuthorRelationalPlan(rGrantUserAnyPlan, context); + } + + @Override + public TSStatus visitRGrantRoleAny( + final AuthorRelationalPlan rGrantRoleAnyPlan, final TSStatus context) { + return visitAuthorRelationalPlan(rGrantRoleAnyPlan, context); + } + + @Override + public TSStatus visitRGrantUserAll( + final AuthorRelationalPlan rGrantUserAllPlan, final TSStatus context) { + return visitAuthorRelationalPlan(rGrantUserAllPlan, context); + } + + @Override + public TSStatus visitRGrantRoleAll( + final AuthorRelationalPlan rGrantRoleAllPlan, final TSStatus context) { + return visitAuthorRelationalPlan(rGrantRoleAllPlan, context); + } + + @Override + public TSStatus visitRGrantUserDB( + final AuthorRelationalPlan rGrantUserDBPlan, final TSStatus context) { + return visitAuthorRelationalPlan(rGrantUserDBPlan, context); + } + + @Override + public TSStatus visitRGrantUserTB( + final AuthorRelationalPlan rGrantUserTBPlan, final TSStatus context) { + return visitAuthorRelationalPlan(rGrantUserTBPlan, context); + } + + @Override + public TSStatus visitRGrantRoleDB( + final AuthorRelationalPlan rGrantRoleDBPlan, final TSStatus context) { + return visitAuthorRelationalPlan(rGrantRoleDBPlan, context); + } + + @Override + public TSStatus visitRGrantRoleTB( + final AuthorRelationalPlan rGrantRoleTBPlan, final TSStatus context) { + return visitAuthorRelationalPlan(rGrantRoleTBPlan, context); + } + + @Override + public TSStatus visitRRevokeUserAny( + final AuthorRelationalPlan rRevokeUserAnyPlan, final TSStatus context) { + return visitAuthorRelationalPlan(rRevokeUserAnyPlan, context); + } + + @Override + public TSStatus visitRRevokeRoleAny( + final AuthorRelationalPlan rRevokeRoleAnyPlan, final TSStatus context) { + return visitAuthorRelationalPlan(rRevokeRoleAnyPlan, context); + } + + @Override + public TSStatus visitRRevokeUserAll( + final AuthorRelationalPlan rRevokeUserAllPlan, final TSStatus context) { + return visitAuthorRelationalPlan(rRevokeUserAllPlan, context); + } + + @Override + public TSStatus visitRRevokeRoleAll( + final AuthorRelationalPlan rRevokeRoleAllPlan, final TSStatus context) { + return visitAuthorRelationalPlan(rRevokeRoleAllPlan, context); + } + + @Override + public TSStatus visitRRevokeUserDBPrivilege( + final AuthorRelationalPlan rRevokeUserDBPrivilegePlan, final TSStatus context) { + return visitAuthorRelationalPlan(rRevokeUserDBPrivilegePlan, context); + } + + @Override + public TSStatus visitRRevokeUserTBPrivilege( + final AuthorRelationalPlan rRevokeUserTBPrivilegePlan, final TSStatus context) { + return visitAuthorRelationalPlan(rRevokeUserTBPrivilegePlan, context); + } + + @Override + public TSStatus visitRRevokeRoleDBPrivilege( + final AuthorRelationalPlan rRevokeRoleTBPrivilegePlan, final TSStatus context) { + return visitAuthorRelationalPlan(rRevokeRoleTBPrivilegePlan, context); + } + + @Override + public TSStatus visitRRevokeRoleTBPrivilege( + final AuthorRelationalPlan rRevokeRoleTBPrivilegePlan, final TSStatus context) { + return visitAuthorRelationalPlan(rRevokeRoleTBPrivilegePlan, context); + } + + @Override + public TSStatus visitRGrantUserSysPrivilege( + final AuthorRelationalPlan rGrantUserSysPrivilegePlan, final TSStatus context) { + return visitAuthorRelationalPlan(rGrantUserSysPrivilegePlan, context); + } + + @Override + public TSStatus visitRGrantRoleSysPrivilege( + final AuthorRelationalPlan rGrantRoleSysPrivilegePlan, final TSStatus context) { + return visitAuthorRelationalPlan(rGrantRoleSysPrivilegePlan, context); + } + + @Override + public TSStatus visitRRevokeUserSysPrivilege( + final AuthorRelationalPlan rRevokeUserSysPrivilegePlan, final TSStatus context) { + return visitAuthorRelationalPlan(rRevokeUserSysPrivilegePlan, context); + } + + @Override + public TSStatus visitRRevokeRoleSysPrivilege( + final AuthorRelationalPlan rRevokeRoleSysPrivilegePlan, final TSStatus context) { + return visitAuthorRelationalPlan(rRevokeRoleSysPrivilegePlan, context); + } + + private TSStatus visitAuthorRelationalPlan( + final AuthorRelationalPlan plan, final TSStatus context) { + if (context.getCode() == TSStatusCode.USER_NOT_EXIST.getStatusCode() + || context.getCode() == TSStatusCode.USER_ALREADY_EXIST.getStatusCode() + || context.getCode() == TSStatusCode.ROLE_NOT_EXIST.getStatusCode() + || context.getCode() == TSStatusCode.ROLE_ALREADY_EXIST.getStatusCode() + || context.getCode() == TSStatusCode.USER_ALREADY_HAS_ROLE.getStatusCode() + || context.getCode() == TSStatusCode.USER_NOT_HAS_ROLE.getStatusCode() + || context.getCode() == TSStatusCode.NO_PERMISSION.getStatusCode()) { + return new TSStatus(TSStatusCode.PIPE_RECEIVER_IDEMPOTENT_CONFLICT_EXCEPTION.getStatusCode()) + .setMessage(context.getMessage()); + } + return visitPlan(plan, context); } @Override diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/AuthorInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/AuthorInfo.java index 62abe8af8198..88d173da0464 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/AuthorInfo.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/AuthorInfo.java @@ -320,6 +320,21 @@ public TSStatus authorNonQuery(AuthorRelationalPlan authorPlan) { } break; case RGrantUserAll: + // database scope and table scope all + if (!database.isEmpty()) { + for (PrivilegeType privilege : privileges) { + if (privilege.isRelationalPrivilege()) { + if (table.isEmpty()) { + authorizer.grantPrivilegeToUser( + userName, new PrivilegeUnion(database, privilege, grantOpt)); + } else { + authorizer.grantPrivilegeToUser( + userName, new PrivilegeUnion(database, table, privilege, grantOpt)); + } + } + } + break; + } for (PrivilegeType privilege : PrivilegeType.values()) { if (privilege.forRelationalSys()) { authorizer.grantPrivilegeToUser(userName, new PrivilegeUnion(privilege, grantOpt)); @@ -331,6 +346,21 @@ public TSStatus authorNonQuery(AuthorRelationalPlan authorPlan) { } break; case RGrantRoleAll: + // database scope and table scope all + if (!database.isEmpty()) { + for (PrivilegeType privilege : privileges) { + if (privilege.isRelationalPrivilege()) { + if (table.isEmpty()) { + authorizer.grantPrivilegeToRole( + roleName, new PrivilegeUnion(database, privilege, grantOpt)); + } else { + authorizer.grantPrivilegeToRole( + roleName, new PrivilegeUnion(database, table, privilege, grantOpt)); + } + } + } + break; + } for (PrivilegeType privilege : PrivilegeType.values()) { if (privilege.forRelationalSys()) { authorizer.grantPrivilegeToRole(roleName, new PrivilegeUnion(privilege, grantOpt)); @@ -378,26 +408,38 @@ public TSStatus authorNonQuery(AuthorRelationalPlan authorPlan) { } break; case RRevokeUserAll: - for (PrivilegeType privilege : PrivilegeType.values()) { - if (privilege.forRelationalSys()) { - authorizer.revokePrivilegeFromUser(userName, new PrivilegeUnion(privilege, grantOpt)); - } - if (privilege.isRelationalPrivilege()) { - authorizer.revokePrivilegeFromUser( - userName, new PrivilegeUnion(privilege, grantOpt, true)); + if (!database.isEmpty()) { + for (PrivilegeType privilege : PrivilegeType.values()) { + if (privilege.isRelationalPrivilege()) { + if (table.isEmpty()) { + authorizer.revokePrivilegeFromUser( + userName, new PrivilegeUnion(database, privilege, grantOpt)); + } else { + authorizer.revokePrivilegeFromUser( + userName, new PrivilegeUnion(database, table, privilege, grantOpt)); + } + } } + break; } + authorizer.revokeAllPrivilegeFromUser(userName); break; case RRevokeRoleAll: - for (PrivilegeType privilege : PrivilegeType.values()) { - if (privilege.forRelationalSys()) { - authorizer.revokePrivilegeFromRole(roleName, new PrivilegeUnion(privilege, grantOpt)); - } - if (privilege.isRelationalPrivilege()) { - authorizer.revokePrivilegeFromRole( - roleName, new PrivilegeUnion(privilege, grantOpt, true)); + if (!database.isEmpty()) { + for (PrivilegeType privilege : PrivilegeType.values()) { + if (privilege.isRelationalPrivilege()) { + if (table.isEmpty()) { + authorizer.revokePrivilegeFromRole( + roleName, new PrivilegeUnion(database, privilege, grantOpt)); + } else { + authorizer.revokePrivilegeFromRole( + roleName, new PrivilegeUnion(database, table, privilege, grantOpt)); + } + } } + break; } + authorizer.revokeAllPrivilegeFromRole(roleName); break; case RRevokeUserDBPriv: for (PrivilegeType privilege : privileges) { diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/extractor/PipeConfigPhysicalPlanScopeParseVisitorTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/extractor/PipeConfigPhysicalPlanScopeParseVisitorTest.java new file mode 100644 index 000000000000..6e5e9870fbfa --- /dev/null +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/extractor/PipeConfigPhysicalPlanScopeParseVisitorTest.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.manager.pipe.extractor; + +import org.apache.iotdb.commons.auth.entity.PrivilegeType; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; +import org.apache.iotdb.confignode.consensus.request.write.auth.AuthorRelationalPlan; +import org.apache.iotdb.confignode.consensus.request.write.auth.AuthorTreePlan; + +import org.junit.Assert; +import org.junit.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.stream.Collectors; + +public class PipeConfigPhysicalPlanScopeParseVisitorTest { + @Test + public void testTreeScopeParsing() { + testTreeScopeParsing(ConfigPhysicalPlanType.GrantRole, false); + testTreeScopeParsing(ConfigPhysicalPlanType.RevokeRole, false); + testTreeScopeParsing(ConfigPhysicalPlanType.GrantUser, true); + testTreeScopeParsing(ConfigPhysicalPlanType.RevokeUser, true); + } + + private void testTreeScopeParsing(final ConfigPhysicalPlanType type, final boolean isUser) { + Assert.assertEquals( + new AuthorTreePlan( + type, + isUser ? "user" : "", + isUser ? "" : "role", + "", + "", + new HashSet<>( + Arrays.stream(PrivilegeType.values()) + .filter(PrivilegeType::forRelationalSys) + .map(Enum::ordinal) + .collect(Collectors.toList())), + false, + Collections.singletonList(new PartialPath(new String[] {"root", "**"}))), + IoTDBConfigRegionExtractor.TREE_SCOPE_PARSE_VISITOR + .process( + new AuthorTreePlan( + type, + isUser ? "user" : "", + isUser ? "" : "role", + "", + "", + new HashSet<>( + Arrays.stream(PrivilegeType.values()) + .filter(privilegeType -> !privilegeType.isRelationalPrivilege()) + .map(Enum::ordinal) + .collect(Collectors.toList())), + false, + Collections.singletonList(new PartialPath(new String[] {"root", "**"}))), + null) + .orElseThrow(AssertionError::new)); + } + + @Test + public void testTableScopeParsing() { + testTableScopeParsing( + ConfigPhysicalPlanType.GrantRole, ConfigPhysicalPlanType.RGrantRoleAll, false); + testTableScopeParsing( + ConfigPhysicalPlanType.RevokeRole, ConfigPhysicalPlanType.RRevokeRoleAll, false); + testTableScopeParsing( + ConfigPhysicalPlanType.GrantUser, ConfigPhysicalPlanType.RGrantUserAll, true); + testTableScopeParsing( + ConfigPhysicalPlanType.RevokeUser, ConfigPhysicalPlanType.RRevokeUserAll, true); + } + + private void testTableScopeParsing( + final ConfigPhysicalPlanType outputType, + final ConfigPhysicalPlanType inputType, + final boolean isUser) { + Assert.assertEquals( + new AuthorTreePlan( + outputType, + isUser ? "user" : "", + isUser ? "" : "role", + "", + "", + new HashSet<>( + Arrays.stream(PrivilegeType.values()) + .filter(PrivilegeType::forRelationalSys) + .map(Enum::ordinal) + .collect(Collectors.toList())), + true, + Collections.emptyList()), + IoTDBConfigRegionExtractor.TABLE_SCOPE_PARSE_VISITOR + .process( + new AuthorRelationalPlan( + inputType, isUser ? "user" : "", isUser ? "" : "role", "", "", -1, true), + null) + .orElseThrow(AssertionError::new)); + } +} diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/extractor/PipeConfigPhysicalPlanTablePatternParseVisitorTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/extractor/PipeConfigPhysicalPlanTablePatternParseVisitorTest.java index 0681dc5aaca6..64ec14be5d84 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/extractor/PipeConfigPhysicalPlanTablePatternParseVisitorTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/extractor/PipeConfigPhysicalPlanTablePatternParseVisitorTest.java @@ -19,10 +19,12 @@ package org.apache.iotdb.confignode.manager.pipe.extractor; +import org.apache.iotdb.commons.auth.entity.PrivilegeType; import org.apache.iotdb.commons.pipe.datastructure.pattern.TablePattern; import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; +import org.apache.iotdb.confignode.consensus.request.write.auth.AuthorRelationalPlan; import org.apache.iotdb.confignode.consensus.request.write.database.DatabaseSchemaPlan; import org.apache.iotdb.confignode.consensus.request.write.database.DeleteDatabasePlan; import org.apache.iotdb.confignode.consensus.request.write.pipe.payload.PipeCreateTablePlan; @@ -112,6 +114,29 @@ public void testCommitDeleteTable() { new CommitDeleteTablePlan("da", "ac")); } + @Test + public void testAuth() { + testInput( + new AuthorRelationalPlan( + ConfigPhysicalPlanType.RGrantRoleAll, "", "role", "", "", -1, false), + new AuthorRelationalPlan( + ConfigPhysicalPlanType.RGrantUserDBPriv, + "user", + "", + "da", + "", + PrivilegeType.SELECT.ordinal(), + false), + new AuthorRelationalPlan( + ConfigPhysicalPlanType.RGrantUserTBPriv, + "user", + "", + "db1", + "ac", + PrivilegeType.DROP.ordinal(), + false)); + } + private void testInput( final ConfigPhysicalPlan trueInput, final ConfigPhysicalPlan falseInput1, diff --git a/iotdb-core/consensus/pom.xml b/iotdb-core/consensus/pom.xml index bbf4e2e34592..00295dcbfd27 100644 --- a/iotdb-core/consensus/pom.xml +++ b/iotdb-core/consensus/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-core - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT iotdb-consensus IoTDB: Core: Consensus @@ -39,32 +39,32 @@ org.apache.iotdb node-commons - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.apache.iotdb metrics-interface - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.apache.iotdb iotdb-thrift-commons - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.apache.iotdb iotdb-thrift-consensus - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.apache.iotdb service-rpc - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.apache.iotdb pipe-api - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.apache.ratis diff --git a/iotdb-core/datanode/pom.xml b/iotdb-core/datanode/pom.xml index 8e8e5f6e2325..ad5c6aaced52 100644 --- a/iotdb-core/datanode/pom.xml +++ b/iotdb-core/datanode/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-core - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT iotdb-server IoTDB: Core: Data-Node (Server) @@ -37,12 +37,12 @@ org.apache.iotdb service-rpc - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.apache.iotdb iotdb-consensus - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.apache.tsfile @@ -57,82 +57,82 @@ org.apache.iotdb external-api - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.apache.iotdb openapi - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.apache.iotdb node-commons - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.apache.iotdb isession - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.apache.iotdb iotdb-antlr - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.apache.iotdb iotdb-relational-grammar - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.apache.iotdb iotdb-thrift-commons - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.apache.iotdb iotdb-thrift-consensus - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.apache.iotdb udf-api - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.apache.iotdb trigger-api - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.apache.iotdb metrics-interface - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.apache.iotdb iotdb-thrift - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.apache.iotdb iotdb-thrift-confignode - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.apache.iotdb iotdb-thrift-ainode - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.apache.iotdb pipe-api - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.apache.iotdb iotdb-session - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.apache.commons @@ -314,7 +314,7 @@ org.apache.iotdb metrics-core - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.mockito diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/auth/AuthorityChecker.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/auth/AuthorityChecker.java index 0d6d2727250c..d084e17123fb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/auth/AuthorityChecker.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/auth/AuthorityChecker.java @@ -66,6 +66,9 @@ public class AuthorityChecker { public static final TSStatus SUCCEED = new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); + public static final String ONLY_ADMIN_ALLOWED = + "No permissions for this operation, only root user is allowed"; + private static final String NO_PERMISSION_PROMOTION = "No permissions for this operation, please add privilege "; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/connector/PipeRealtimePriorityBlockingQueue.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/connector/PipeRealtimePriorityBlockingQueue.java index 185b60c29e98..3edbcef539c5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/connector/PipeRealtimePriorityBlockingQueue.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/connector/PipeRealtimePriorityBlockingQueue.java @@ -51,6 +51,8 @@ public PipeRealtimePriorityBlockingQueue() { @Override public boolean directOffer(final Event event) { + checkBeforeOffer(event); + if (event instanceof TsFileInsertionEvent) { tsfileInsertEventDeque.add((TsFileInsertionEvent) event); return true; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/processor/PipeProcessorSubtask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/processor/PipeProcessorSubtask.java index 7f29b3c55f99..b4bd024bfec4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/processor/PipeProcessorSubtask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/processor/PipeProcessorSubtask.java @@ -228,10 +228,9 @@ public void close() { PipeProcessorMetrics.getInstance().deregister(taskID); try { isClosed.set(true); - - // pipeProcessor closes first, then no more events will be added into outputEventCollector. - // only after that, outputEventCollector can be closed. pipeProcessor.close(); + // It is important to note that even if the subtask and its corresponding processor are + // closed, the execution thread may still deliver events downstream. } catch (final Exception e) { LOGGER.info( "Exception occurred when closing pipe processor subtask {}, root cause: {}", diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/processor/PipeProcessorSubtaskWorker.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/processor/PipeProcessorSubtaskWorker.java index 44fd578c84db..813d07317212 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/processor/PipeProcessorSubtaskWorker.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/processor/PipeProcessorSubtaskWorker.java @@ -33,9 +33,6 @@ public class PipeProcessorSubtaskWorker extends WrappedRunnable { private static final Logger LOGGER = LoggerFactory.getLogger(PipeProcessorSubtaskWorker.class); - private static final long CLOSED_SUBTASK_CLEANUP_ROUND_INTERVAL = 1000; - private long closedSubtaskCleanupRoundCounter = 0; - private static final int SLEEP_INTERVAL_ADJUSTMENT_ROUND_INTERVAL = 100; private int totalRoundInAdjustmentInterval = 0; private int workingRoundInAdjustmentInterval = 0; @@ -56,12 +53,10 @@ public void runMayThrow() { } private void cleanupClosedSubtasksIfNecessary() { - if (++closedSubtaskCleanupRoundCounter % CLOSED_SUBTASK_CLEANUP_ROUND_INTERVAL == 0) { - subtasks.stream() - .filter(PipeProcessorSubtask::isClosed) - .collect(Collectors.toList()) - .forEach(subtasks::remove); - } + subtasks.stream() + .filter(PipeProcessorSubtask::isClosed) + .collect(Collectors.toList()) + .forEach(subtasks::remove); } private boolean runSubtasks() { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeTsFileInsertionEvent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeTsFileInsertionEvent.java index 78b67fc701e6..7b8e9a69b73c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeTsFileInsertionEvent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeTsFileInsertionEvent.java @@ -56,6 +56,7 @@ import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; import static org.apache.tsfile.common.constant.TsFileConstant.PATH_ROOT; import static org.apache.tsfile.common.constant.TsFileConstant.PATH_SEPARATOR; @@ -81,7 +82,7 @@ public class PipeTsFileInsertionEvent extends PipeInsertionEvent private final boolean isGeneratedByHistoricalExtractor; private final AtomicBoolean isClosed; - private TsFileInsertionEventParser eventParser; + private final AtomicReference eventParser; // The point count of the TsFile. Used for metrics on PipeConsensus' receiver side. // May be updated after it is flushed. Should be negative if not set. @@ -188,6 +189,8 @@ public PipeTsFileInsertionEvent( // If the status is "closed", then the resource status is "closed", the tsFile won't be altered // and can be sent. isClosed.set(resource.isClosed()); + + this.eventParser = new AtomicReference<>(null); } /** @@ -570,13 +573,12 @@ public boolean isGeneratedByHistoricalExtractor() { private TsFileInsertionEventParser initEventParser() { try { - if (eventParser == null) { - eventParser = - new TsFileInsertionEventParserProvider( - tsFile, treePattern, tablePattern, startTime, endTime, pipeTaskMeta, this) - .provide(); - } - return eventParser; + eventParser.compareAndSet( + null, + new TsFileInsertionEventParserProvider( + tsFile, treePattern, tablePattern, startTime, endTime, pipeTaskMeta, this) + .provide()); + return eventParser.get(); } catch (final IOException e) { close(); @@ -613,10 +615,13 @@ public long count(final boolean skipReportOnCommit) throws IOException { /** Release the resource of {@link TsFileInsertionEventParser}. */ @Override public void close() { - if (eventParser != null) { - eventParser.close(); - eventParser = null; - } + eventParser.getAndUpdate( + parser -> { + if (Objects.nonNull(parser)) { + parser.close(); + } + return null; + }); } /////////////////////////// Object /////////////////////////// @@ -654,7 +659,8 @@ public PipeEventResource eventResourceBuilder() { this.tsFile, this.isWithMod, this.modFile, - this.sharedModFile); + this.sharedModFile, + this.eventParser); } private static class PipeTsFileInsertionEventResource extends PipeEventResource { @@ -662,7 +668,8 @@ private static class PipeTsFileInsertionEventResource extends PipeEventResource private final File tsFile; private final boolean isWithMod; private final File modFile; - private final File sharedModFile; + private final File sharedModFile; // unused now + private final AtomicReference eventParser; private PipeTsFileInsertionEventResource( final AtomicBoolean isReleased, @@ -670,21 +677,33 @@ private PipeTsFileInsertionEventResource( final File tsFile, final boolean isWithMod, final File modFile, - File sharedModFile) { + final File sharedModFile, + final AtomicReference eventParser) { super(isReleased, referenceCount); this.tsFile = tsFile; this.isWithMod = isWithMod; this.modFile = modFile; this.sharedModFile = sharedModFile; + this.eventParser = eventParser; } @Override protected void finalizeResource() { try { + // decrease reference count PipeDataNodeResourceManager.tsfile().decreaseFileReference(tsFile); if (isWithMod) { PipeDataNodeResourceManager.tsfile().decreaseFileReference(modFile); } + + // close event parser + eventParser.getAndUpdate( + parser -> { + if (Objects.nonNull(parser)) { + parser.close(); + } + return null; + }); } catch (final Exception e) { LOGGER.warn( String.format("Decrease reference count for TsFile %s error.", tsFile.getPath()), e); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/schemaregion/SchemaRegionListeningFilter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/schemaregion/SchemaRegionListeningFilter.java index cb64d208b382..86e985f330c2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/schemaregion/SchemaRegionListeningFilter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/schemaregion/SchemaRegionListeningFilter.java @@ -41,8 +41,8 @@ import static org.apache.iotdb.commons.pipe.datastructure.options.PipeInclusionOptions.getExclusionString; import static org.apache.iotdb.commons.pipe.datastructure.options.PipeInclusionOptions.getInclusionString; import static org.apache.iotdb.commons.pipe.datastructure.options.PipeInclusionOptions.parseOptions; -import static org.apache.iotdb.commons.pipe.datastructure.options.PipeInclusionOptions.tableOnlySyncPrefix; -import static org.apache.iotdb.commons.pipe.datastructure.options.PipeInclusionOptions.treeOnlySyncPrefix; +import static org.apache.iotdb.commons.pipe.datastructure.options.PipeInclusionOptions.tableOnlySyncPrefixes; +import static org.apache.iotdb.commons.pipe.datastructure.options.PipeInclusionOptions.treeOnlySyncPrefixes; /** * {@link SchemaRegionListeningFilter} is to classify the {@link PlanNode}s to help {@link @@ -118,11 +118,11 @@ public static Set parseListeningPlanTypeSet(final PipeParameters p exclusionOptions.forEach(exclusion -> planTypes.removeAll(getOptionsByPrefix(exclusion))); if (!TreePattern.isTreeModelDataAllowToBeCaptured(parameters)) { - planTypes.removeAll(getOptionsByPrefix(treeOnlySyncPrefix)); + treeOnlySyncPrefixes.forEach(prefix -> planTypes.removeAll(getOptionsByPrefix(prefix))); } if (!TablePattern.isTableModelDataAllowToBeCaptured(parameters)) { - planTypes.removeAll(getOptionsByPrefix(tableOnlySyncPrefix)); + tableOnlySyncPrefixes.forEach(prefix -> planTypes.removeAll(getOptionsByPrefix(prefix))); } return planTypes; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java index c5db184030db..121f37bf2ea3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java @@ -90,6 +90,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.SeriesScanOptions; import org.apache.iotdb.db.queryengine.plan.relational.metadata.Metadata; import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache.TreeDeviceSchemaCacheManager; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SetSqlDialect; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Use; import org.apache.iotdb.db.queryengine.plan.relational.sql.parser.ParsingException; import org.apache.iotdb.db.queryengine.plan.relational.sql.parser.SqlParser; @@ -113,6 +114,7 @@ import org.apache.iotdb.db.queryengine.plan.statement.metadata.template.DropSchemaTemplateStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.template.SetSchemaTemplateStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.template.UnsetSchemaTemplateStatement; +import org.apache.iotdb.db.queryengine.plan.statement.sys.SetSqlDialectStatement; import org.apache.iotdb.db.schemaengine.template.TemplateQueryType; import org.apache.iotdb.db.storageengine.StorageEngine; import org.apache.iotdb.db.storageengine.dataregion.DataRegion; @@ -312,12 +314,17 @@ private TSExecuteStatementResp executeStatementInternal( StatementType statementType = null; Throwable t = null; boolean useDatabase = false; + boolean setSqlDialect = false; try { // create and cache dataset ExecutionResult result; if (clientSession.getSqlDialect() == IClientSession.SqlDialect.TREE) { Statement s = StatementGenerator.createStatement(statement, clientSession.getZoneId()); + if (s instanceof SetSqlDialectStatement) { + setSqlDialect = true; + } + if (s == null) { return RpcUtils.getTSExecuteStatementResp( RpcUtils.getStatus( @@ -357,6 +364,10 @@ private TSExecuteStatementResp executeStatementInternal( useDatabase = true; } + if (s instanceof SetSqlDialect) { + setSqlDialect = true; + } + if (s == null) { return RpcUtils.getTSExecuteStatementResp( RpcUtils.getStatus( @@ -404,6 +415,12 @@ private TSExecuteStatementResp executeStatementInternal( if (useDatabase) { resp.setDatabase(clientSession.getDatabaseName()); } + + if (setSqlDialect) { + resp.setTableModel( + SESSION_MANAGER.getCurrSessionAndUpdateIdleTime().getSqlDialect() + == IClientSession.SqlDialect.TABLE); + } } return resp; } @@ -2230,10 +2247,12 @@ public TSStatus insertTablet(TSInsertTabletReq req) { return RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS); } - // permission check - TSStatus status = AuthorityChecker.checkAuthority(statement, clientSession); - if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - return status; + // permission check for tree model, table model does this in the analysis stage + if (!req.isWriteToTable()) { + TSStatus status = AuthorityChecker.checkAuthority(statement, clientSession); + if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + return status; + } } quota = diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/SharedTsBlockQueue.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/SharedTsBlockQueue.java index b3b94f66282f..f38104263980 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/SharedTsBlockQueue.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/SharedTsBlockQueue.java @@ -196,8 +196,8 @@ public TsBlock remove() { localFragmentInstanceId.getQueryId(), fullFragmentInstanceId, localPlanNodeId, - tsBlock.getRetainedSizeInBytes()); - bufferRetainedSizeInBytes -= tsBlock.getRetainedSizeInBytes(); + tsBlock.getSizeInBytes()); + bufferRetainedSizeInBytes -= tsBlock.getSizeInBytes(); // Every time LocalSourceHandle consumes a TsBlock, it needs to send the event to // corresponding LocalSinkChannel. if (sinkChannel != null) { @@ -236,10 +236,10 @@ public ListenableFuture add(TsBlock tsBlock) { localFragmentInstanceId.getQueryId(), fullFragmentInstanceId, localPlanNodeId, - tsBlock.getRetainedSizeInBytes(), + tsBlock.getSizeInBytes(), maxBytesCanReserve); blockedOnMemory = pair.left; - bufferRetainedSizeInBytes += tsBlock.getRetainedSizeInBytes(); + bufferRetainedSizeInBytes += tsBlock.getSizeInBytes(); // reserve memory failed, we should wait until there is enough memory if (!Boolean.TRUE.equals(pair.right)) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/sink/SinkChannel.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/sink/SinkChannel.java index 9f29a8a753fc..8915938eb03c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/sink/SinkChannel.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/sink/SinkChannel.java @@ -201,7 +201,7 @@ public synchronized void send(TsBlock tsBlock) { if (noMoreTsBlocks) { return; } - long retainedSizeInBytes = tsBlock.getRetainedSizeInBytes(); + long sizeInBytes = tsBlock.getSizeInBytes(); int startSequenceId; startSequenceId = nextSequenceId; blocked = @@ -211,17 +211,16 @@ public synchronized void send(TsBlock tsBlock) { localFragmentInstanceId.getQueryId(), fullFragmentInstanceId, localPlanNodeId, - retainedSizeInBytes, + sizeInBytes, maxBytesCanReserve) .left; - bufferRetainedSizeInBytes += retainedSizeInBytes; + bufferRetainedSizeInBytes += sizeInBytes; sequenceIdToTsBlock.put(nextSequenceId, new Pair<>(tsBlock, currentTsBlockSize)); nextSequenceId += 1; - currentTsBlockSize = retainedSizeInBytes; + currentTsBlockSize = sizeInBytes; - // TODO: consider merge multiple NewDataBlockEvent for less network traffic. - submitSendNewDataBlockEventTask(startSequenceId, ImmutableList.of(retainedSizeInBytes)); + submitSendNewDataBlockEventTask(startSequenceId, ImmutableList.of(sizeInBytes)); } finally { DATA_EXCHANGE_COST_METRIC_SET.recordDataExchangeCost( SINK_HANDLE_SEND_TSBLOCK_REMOTE, System.nanoTime() - startTime); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/source/LocalSourceHandle.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/source/LocalSourceHandle.java index 4da89f72d07a..63feb571cb09 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/source/LocalSourceHandle.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/source/LocalSourceHandle.java @@ -124,9 +124,7 @@ public TsBlock receive() { if (tsBlock != null) { if (LOGGER.isDebugEnabled()) { LOGGER.debug( - "[GetTsBlockFromQueue] TsBlock:{} size:{}", - currSequenceId, - tsBlock.getRetainedSizeInBytes()); + "[GetTsBlockFromQueue] TsBlock:{} size:{}", currSequenceId, tsBlock.getSizeInBytes()); } currSequenceId++; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/executor/RegionWriteExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/executor/RegionWriteExecutor.java index 079238309c75..e9fac5c1d32a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/executor/RegionWriteExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/executor/RegionWriteExecutor.java @@ -85,6 +85,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.concurrent.locks.ReentrantReadWriteLock; @@ -485,11 +486,11 @@ private RegionExecutionResult executeCreateMultiTimeSeries( measurementGroupMap.remove(emptyDevice); } - final RegionExecutionResult failingResult = + final RegionExecutionResult executionResult = registerTimeSeries(measurementGroupMap, node, context, failingStatus); - if (failingResult != null) { - return failingResult; + if (executionResult != null) { + return executionResult; } final TSStatus status = RpcUtils.getStatus(failingStatus); @@ -616,7 +617,12 @@ private RegionExecutionResult executeInternalCreateTimeSeries( measurementGroup.removeMeasurements(failingMeasurementMap.keySet()); return processExecutionResultOfInternalCreateSchema( - super.visitInternalCreateTimeSeries(node, context), + !measurementGroup.isEmpty() + ? super.visitInternalCreateTimeSeries(node, context) + : RegionExecutionResult.create( + true, + "Execute successfully", + RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS, "Execute successfully")), failingStatus, alreadyExistingStatus); } finally { @@ -661,8 +667,13 @@ private RegionExecutionResult executeInternalCreateMultiTimeSeries( MeasurementGroup measurementGroup; Map failingMeasurementMap; MetadataException metadataException; - for (final Map.Entry> deviceEntry : - node.getDeviceMap().entrySet()) { + + final Iterator>> iterator = + node.getDeviceMap().entrySet().iterator(); + + while (iterator.hasNext()) { + final Map.Entry> deviceEntry = + iterator.next(); measurementGroup = deviceEntry.getValue().right; failingMeasurementMap = schemaRegion.checkMeasurementExistence( @@ -690,10 +701,18 @@ private RegionExecutionResult executeInternalCreateMultiTimeSeries( } } measurementGroup.removeMeasurements(failingMeasurementMap.keySet()); + if (measurementGroup.isEmpty()) { + iterator.remove(); + } } return processExecutionResultOfInternalCreateSchema( - super.visitInternalCreateMultiTimeSeries(node, context), + !node.getDeviceMap().isEmpty() + ? super.visitInternalCreateMultiTimeSeries(node, context) + : RegionExecutionResult.create( + true, + "Execute successfully", + RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS, "Execute successfully")), failingStatus, alreadyExistingStatus); } finally { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java index 6d035e1dc3c6..14629fa37fa1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java @@ -29,6 +29,7 @@ import org.apache.iotdb.db.queryengine.common.FragmentInstanceId; import org.apache.iotdb.db.queryengine.common.QueryId; import org.apache.iotdb.db.queryengine.common.SessionInfo; +import org.apache.iotdb.db.queryengine.metric.DriverSchedulerMetricSet; import org.apache.iotdb.db.queryengine.metric.QueryRelatedResourceMetricSet; import org.apache.iotdb.db.queryengine.metric.SeriesScanCostMetricSet; import org.apache.iotdb.db.queryengine.plan.planner.memory.MemoryReservationManager; @@ -57,10 +58,14 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; +import static org.apache.iotdb.db.queryengine.metric.DriverSchedulerMetricSet.BLOCK_QUEUED_TIME; +import static org.apache.iotdb.db.queryengine.metric.DriverSchedulerMetricSet.READY_QUEUED_TIME; + public class FragmentInstanceContext extends QueryContext { private static final Logger LOGGER = LoggerFactory.getLogger(FragmentInstanceContext.class); @@ -371,19 +376,32 @@ public void setDataNodeQueryContext(DataNodeQueryContext dataNodeQueryContext) { } public FragmentInstanceInfo getInstanceInfo() { - return getErrorCode() - .map( - s -> - new FragmentInstanceInfo( - stateMachine.getState(), - getEndTime(), - getFailedCause(), - getFailureInfoList(), - s)) - .orElseGet( - () -> - new FragmentInstanceInfo( - stateMachine.getState(), getEndTime(), getFailedCause(), getFailureInfoList())); + FragmentInstanceState state = stateMachine.getState(); + long endTime = getEndTime(); + LinkedBlockingQueue failures = stateMachine.getFailureCauses(); + String failureCause = ""; + List failureInfoList = new ArrayList<>(); + TSStatus status = null; + + for (Throwable failure : failures) { + if (failureCause.isEmpty()) { + failureCause = failure.getMessage(); + } + failureInfoList.add(FragmentInstanceFailureInfo.toFragmentInstanceFailureInfo(failure)); + if (failure instanceof IoTDBException) { + status = new TSStatus(((IoTDBException) failure).getErrorCode()); + status.setMessage(failure.getMessage()); + } else if (failure instanceof IoTDBRuntimeException) { + status = new TSStatus(((IoTDBRuntimeException) failure).getErrorCode()); + status.setMessage(failure.getMessage()); + } + } + + if (status == null) { + return new FragmentInstanceInfo(state, endTime, failureCause, failureInfoList); + } else { + return new FragmentInstanceInfo(state, endTime, failureCause, failureInfoList, status); + } } public FragmentInstanceStateMachine getStateMachine() { @@ -675,6 +693,11 @@ public synchronized void releaseResource() { // record fragment instance execution time and metadata get time to metrics long durationTime = System.currentTimeMillis() - executionStartTime.get(); + DriverSchedulerMetricSet.getInstance() + .recordTaskQueueTime(BLOCK_QUEUED_TIME, blockQueueTime.get()); + DriverSchedulerMetricSet.getInstance() + .recordTaskQueueTime(READY_QUEUED_TIME, readyQueueTime.get()); + QueryRelatedResourceMetricSet.getInstance().updateFragmentInstanceTime(durationTime); SeriesScanCostMetricSet.getInstance() diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/AbstractOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/AbstractOperator.java index 1278dc15d8dc..5c6d731f9a04 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/AbstractOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/AbstractOperator.java @@ -44,7 +44,7 @@ public void initializeMaxTsBlockLength(TsBlock tsBlock) { long oneTupleSize = Math.max( 1, - (tsBlock.getRetainedSizeInBytes() - tsBlock.getTotalInstanceSize()) + (tsBlock.getSizeInBytes() - tsBlock.getTotalInstanceSize()) / tsBlock.getPositionCount()); if (oneTupleSize > maxReturnSize) { // make sure at least one-tuple-at-a-time diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/AbstractSortOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/AbstractSortOperator.java index c017cded6b74..0bfa1d3e7673 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/AbstractSortOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/AbstractSortOperator.java @@ -162,7 +162,7 @@ private void prepareSortReaders() throws IoTDBException { } protected void cacheTsBlock(TsBlock tsBlock) throws IoTDBException { - long bytesSize = tsBlock.getRetainedSizeInBytes(); + long bytesSize = tsBlock.getSizeInBytes(); if (bytesSize + cachedBytes < sortBufferManager.getSortBufferSize()) { cachedBytes += bytesSize; for (int i = 0; i < tsBlock.getPositionCount(); i++) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/SortOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/SortOperator.java index eb556e1cfac0..31029a30da64 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/SortOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/SortOperator.java @@ -62,7 +62,7 @@ public TsBlock next() throws Exception { if (tsBlock == null) { return null; } - dataSize += tsBlock.getRetainedSizeInBytes(); + dataSize += tsBlock.getSizeInBytes(); cacheTsBlock(tsBlock); } catch (IoTDBException e) { clear(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/TableStreamSortOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/TableStreamSortOperator.java index 71e938f4f309..90565a858759 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/TableStreamSortOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/TableStreamSortOperator.java @@ -119,7 +119,7 @@ public TsBlock next() throws Exception { return null; } // record total sorted data size - dataSize += currentTsBlock.getRetainedSizeInBytes(); + dataSize += currentTsBlock.getSizeInBytes(); // if currentTsBlock line count + remainingCount is still less than minLinesToOutput, just // cache it diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java index cd00aa5e4196..213e4990a5f4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java @@ -41,7 +41,7 @@ import org.apache.tsfile.file.metadata.IChunkMetadata; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.ITimeSeriesMetadata; -import org.apache.tsfile.file.metadata.TableDeviceMetadata; +import org.apache.tsfile.file.metadata.TableDeviceTimeSeriesMetadata; import org.apache.tsfile.file.metadata.TimeseriesMetadata; import org.apache.tsfile.read.controller.IChunkLoader; import org.apache.tsfile.read.filter.basic.Filter; @@ -395,7 +395,7 @@ private static AbstractAlignedTimeSeriesMetadata setModifications( AbstractAlignedTimeSeriesMetadata alignedTimeSeriesMetadata = ignoreAllNullRows ? new AlignedTimeSeriesMetadata(timeColumnMetadata, valueColumnMetadataList) - : new TableDeviceMetadata(timeColumnMetadata, valueColumnMetadataList); + : new TableDeviceTimeSeriesMetadata(timeColumnMetadata, valueColumnMetadataList); alignedTimeSeriesMetadata.setChunkMetadataLoader( new DiskAlignedChunkMetadataLoader( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/InformationSchemaContentSupplierFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/InformationSchemaContentSupplierFactory.java index a798a1879c15..d199fb4c9b84 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/InformationSchemaContentSupplierFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/InformationSchemaContentSupplierFactory.java @@ -19,8 +19,10 @@ package org.apache.iotdb.db.queryengine.execution.operator.source.relational; +import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType; import org.apache.iotdb.commons.conf.IoTDBConstant; -import org.apache.iotdb.commons.exception.IoTDBException; +import org.apache.iotdb.commons.pipe.agent.plugin.builtin.BuiltinPipePlugin; +import org.apache.iotdb.commons.pipe.agent.plugin.meta.PipePluginMeta; import org.apache.iotdb.commons.schema.table.InformationSchema; import org.apache.iotdb.commons.schema.table.TableNodeStatus; import org.apache.iotdb.commons.schema.table.TsTable; @@ -29,8 +31,16 @@ import org.apache.iotdb.confignode.rpc.thrift.TDatabaseInfo; import org.apache.iotdb.confignode.rpc.thrift.TDescTable4InformationSchemaResp; import org.apache.iotdb.confignode.rpc.thrift.TGetDatabaseReq; -import org.apache.iotdb.confignode.rpc.thrift.TShowDatabaseResp; +import org.apache.iotdb.confignode.rpc.thrift.TRegionInfo; +import org.apache.iotdb.confignode.rpc.thrift.TShowPipeInfo; +import org.apache.iotdb.confignode.rpc.thrift.TShowPipeReq; +import org.apache.iotdb.confignode.rpc.thrift.TShowRegionReq; +import org.apache.iotdb.confignode.rpc.thrift.TShowSubscriptionInfo; +import org.apache.iotdb.confignode.rpc.thrift.TShowSubscriptionReq; +import org.apache.iotdb.confignode.rpc.thrift.TShowTopicInfo; +import org.apache.iotdb.confignode.rpc.thrift.TShowTopicReq; import org.apache.iotdb.confignode.rpc.thrift.TTableInfo; +import org.apache.iotdb.db.pipe.metric.PipeDataNodeRemainingEventAndTimeMetrics; import org.apache.iotdb.db.protocol.client.ConfigNodeClient; import org.apache.iotdb.db.protocol.client.ConfigNodeClientManager; import org.apache.iotdb.db.protocol.client.ConfigNodeInfo; @@ -38,6 +48,7 @@ import org.apache.iotdb.db.queryengine.plan.Coordinator; import org.apache.iotdb.db.queryengine.plan.execution.IQueryExecution; import org.apache.iotdb.db.schemaengine.table.InformationSchemaUtils; +import org.apache.iotdb.db.utils.TimestampPrecisionUtils; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.common.conf.TSFileConfig; @@ -57,27 +68,40 @@ import java.util.NoSuchElementException; import java.util.Objects; import java.util.Set; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import static org.apache.iotdb.commons.conf.IoTDBConstant.TTL_INFINITE; import static org.apache.iotdb.commons.schema.SchemaConstant.ALL_MATCH_SCOPE; import static org.apache.iotdb.commons.schema.SchemaConstant.ALL_RESULT_NODES; import static org.apache.iotdb.commons.schema.table.TsTable.TTL_PROPERTY; +import static org.apache.iotdb.db.queryengine.plan.execution.config.metadata.ShowPipePluginsTask.PIPE_PLUGIN_TYPE_BUILTIN; +import static org.apache.iotdb.db.queryengine.plan.execution.config.metadata.ShowPipePluginsTask.PIPE_PLUGIN_TYPE_EXTERNAL; public class InformationSchemaContentSupplierFactory { private InformationSchemaContentSupplierFactory() {} public static Iterator getSupplier( - final String tableName, final List dataTypes, final String userName) { + final String tableName, final List dataTypes) { switch (tableName) { case InformationSchema.QUERIES: return new QueriesSupplier(dataTypes); case InformationSchema.DATABASES: - return new DatabaseSupplier(dataTypes, userName); + return new DatabaseSupplier(dataTypes); case InformationSchema.TABLES: - return new TableSupplier(dataTypes, userName); + return new TableSupplier(dataTypes); case InformationSchema.COLUMNS: - return new ColumnSupplier(dataTypes, userName); + return new ColumnSupplier(dataTypes); + case InformationSchema.REGIONS: + return new RegionSupplier(dataTypes); + case InformationSchema.PIPES: + return new PipeSupplier(dataTypes); + case InformationSchema.PIPE_PLUGINS: + return new PipePluginSupplier(dataTypes); + case InformationSchema.TOPICS: + return new TopicSupplier(dataTypes); + case InformationSchema.SUBSCRIPTIONS: + return new SubscriptionSupplier(dataTypes); default: throw new UnsupportedOperationException("Unknown table: " + tableName); } @@ -86,31 +110,32 @@ public static Iterator getSupplier( private static class QueriesSupplier extends TsBlockSupplier { private final long currTime = System.currentTimeMillis(); // We initialize it later for the convenience of data preparation - protected int totalSize; protected int nextConsumedIndex; - private final List queryExecutions = - Coordinator.getInstance().getAllQueryExecutions(); + private final List queryExecutions; private QueriesSupplier(final List dataTypes) { super(dataTypes); - this.totalSize = queryExecutions.size(); + queryExecutions = Coordinator.getInstance().getAllQueryExecutions(); } @Override protected void constructLine() { - IQueryExecution queryExecution = queryExecutions.get(nextConsumedIndex); + final IQueryExecution queryExecution = queryExecutions.get(nextConsumedIndex); if (queryExecution.getSQLDialect().equals(IClientSession.SqlDialect.TABLE)) { - String[] splits = queryExecution.getQueryId().split("_"); - int dataNodeId = Integer.parseInt(splits[splits.length - 1]); + final String[] splits = queryExecution.getQueryId().split("_"); + final int dataNodeId = Integer.parseInt(splits[splits.length - 1]); columnBuilders[0].writeBinary(BytesUtils.valueOf(queryExecution.getQueryId())); - columnBuilders[1].writeLong(queryExecution.getStartExecutionTime()); + columnBuilders[1].writeLong( + TimestampPrecisionUtils.convertToCurrPrecision( + queryExecution.getStartExecutionTime(), TimeUnit.MILLISECONDS)); columnBuilders[2].writeInt(dataNodeId); columnBuilders[3].writeFloat( (float) (currTime - queryExecution.getStartExecutionTime()) / 1000); columnBuilders[4].writeBinary( BytesUtils.valueOf(queryExecution.getExecuteSQL().orElse("UNKNOWN"))); + columnBuilders[5].writeBinary(BytesUtils.valueOf(queryExecution.getUser())); resultBuilder.declarePosition(); } nextConsumedIndex++; @@ -118,26 +143,28 @@ protected void constructLine() { @Override public boolean hasNext() { - return nextConsumedIndex < totalSize; + return nextConsumedIndex < queryExecutions.size(); } } private static class DatabaseSupplier extends TsBlockSupplier { private Iterator> iterator; private TDatabaseInfo currentDatabase; - private final String userName; private boolean hasShownInformationSchema; - private DatabaseSupplier(final List dataTypes, final String userName) { + private DatabaseSupplier(final List dataTypes) { super(dataTypes); - this.userName = userName; try (final ConfigNodeClient client = ConfigNodeClientManager.getInstance().borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { - final TShowDatabaseResp resp = - client.showDatabase( - new TGetDatabaseReq(Arrays.asList(ALL_RESULT_NODES), ALL_MATCH_SCOPE.serialize()) - .setIsTableModel(true)); - iterator = resp.getDatabaseInfoMap().entrySet().iterator(); + iterator = + client + .showDatabase( + new TGetDatabaseReq( + Arrays.asList(ALL_RESULT_NODES), ALL_MATCH_SCOPE.serialize()) + .setIsTableModel(true)) + .getDatabaseInfoMap() + .entrySet() + .iterator(); } catch (final Exception e) { lastException = e; } @@ -172,19 +199,10 @@ protected void constructLine() { @Override public boolean hasNext() { if (!hasShownInformationSchema) { - if (!canShowDB(userName, InformationSchema.INFORMATION_DATABASE)) { - hasShownInformationSchema = true; - } else { - return true; - } + return true; } - while (iterator.hasNext()) { - final Map.Entry result = iterator.next(); - if (!canShowDB(userName, result.getKey())) { - continue; - } - currentDatabase = result.getValue(); - break; + if (iterator.hasNext()) { + currentDatabase = iterator.next().getValue(); } return Objects.nonNull(currentDatabase); } @@ -193,12 +211,11 @@ public boolean hasNext() { private static class TableSupplier extends TsBlockSupplier { private Iterator>> dbIterator; private Iterator tableInfoIterator = null; + private TTableInfo currentTable; private String dbName; - private final String userName; - private TableSupplier(final List dataTypes, final String userName) { + private TableSupplier(final List dataTypes) { super(dataTypes); - this.userName = userName; try (final ConfigNodeClient client = ConfigNodeClientManager.getInstance().borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { final Map> databaseTableInfoMap = @@ -224,28 +241,31 @@ private TableSupplier(final List dataTypes, final String userName) { @Override protected void constructLine() { - final TTableInfo info = tableInfoIterator.next(); columnBuilders[0].writeBinary(new Binary(dbName, TSFileConfig.STRING_CHARSET)); - columnBuilders[1].writeBinary(new Binary(info.getTableName(), TSFileConfig.STRING_CHARSET)); - columnBuilders[2].writeBinary(new Binary(info.getTTL(), TSFileConfig.STRING_CHARSET)); + columnBuilders[1].writeBinary( + new Binary(currentTable.getTableName(), TSFileConfig.STRING_CHARSET)); + columnBuilders[2].writeBinary(new Binary(currentTable.getTTL(), TSFileConfig.STRING_CHARSET)); columnBuilders[3].writeBinary( new Binary( - TableNodeStatus.values()[info.getState()].toString(), TSFileConfig.STRING_CHARSET)); + TableNodeStatus.values()[currentTable.getState()].toString(), + TSFileConfig.STRING_CHARSET)); resultBuilder.declarePosition(); + currentTable = null; } @Override public boolean hasNext() { // Get next table info iterator - while (Objects.isNull(tableInfoIterator) || !tableInfoIterator.hasNext()) { + while (Objects.isNull(currentTable)) { + if (Objects.nonNull(tableInfoIterator) && tableInfoIterator.hasNext()) { + currentTable = tableInfoIterator.next(); + return true; + } if (!dbIterator.hasNext()) { return false; } final Map.Entry> entry = dbIterator.next(); dbName = entry.getKey(); - if (!canShowDB(userName, dbName)) { - continue; - } tableInfoIterator = entry.getValue().iterator(); } return true; @@ -259,11 +279,9 @@ private static class ColumnSupplier extends TsBlockSupplier { private String dbName; private String tableName; private Set preDeletedColumns; - private final String userName; - private ColumnSupplier(final List dataTypes, final String userName) { + private ColumnSupplier(final List dataTypes) { super(dataTypes); - this.userName = userName; try (final ConfigNodeClient client = ConfigNodeClientManager.getInstance().borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { final TDescTable4InformationSchemaResp resp = client.descTables4InformationSchema(); @@ -323,31 +341,232 @@ public boolean hasNext() { final Map.Entry>>> entry = dbIterator.next(); dbName = entry.getKey(); - if (!canShowDB(userName, dbName)) { - continue; - } tableInfoIterator = entry.getValue().entrySet().iterator(); } - final Map.Entry>> tableEntry = tableInfoIterator.next(); + Map.Entry>> tableEntry = tableInfoIterator.next(); tableName = tableEntry.getKey(); preDeletedColumns = tableEntry.getValue().getRight(); columnSchemaIterator = tableEntry.getValue().getLeft().getColumnList().iterator(); + break; } return true; } } - private static boolean canShowDB(final String userName, final String dbName) { - try { - Coordinator.getInstance().getAccessControl().checkCanShowOrUseDatabase(userName, dbName); - } catch (final RuntimeException e) { - if (e.getCause() instanceof IoTDBException) { - return false; + private static class RegionSupplier extends TsBlockSupplier { + private Iterator iterator; + + private RegionSupplier(final List dataTypes) { + super(dataTypes); + try (final ConfigNodeClient client = + ConfigNodeClientManager.getInstance().borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { + iterator = + client + .showRegion(new TShowRegionReq().setIsTableModel(true).setDatabases(null)) + .getRegionInfoListIterator(); + } catch (final Exception e) { + lastException = e; + } + } + + @Override + protected void constructLine() { + final TRegionInfo regionInfo = iterator.next(); + columnBuilders[0].writeInt(regionInfo.getConsensusGroupId().getId()); + columnBuilders[1].writeInt(regionInfo.getDataNodeId()); + if (regionInfo.getConsensusGroupId().getType().ordinal() + == TConsensusGroupType.SchemaRegion.ordinal()) { + columnBuilders[2].writeBinary( + BytesUtils.valueOf(String.valueOf(TConsensusGroupType.SchemaRegion))); + } else if (regionInfo.getConsensusGroupId().getType().ordinal() + == TConsensusGroupType.DataRegion.ordinal()) { + columnBuilders[2].writeBinary( + BytesUtils.valueOf(String.valueOf(TConsensusGroupType.DataRegion))); + } + columnBuilders[3].writeBinary( + BytesUtils.valueOf(regionInfo.getStatus() == null ? "" : regionInfo.getStatus())); + columnBuilders[4].writeBinary(BytesUtils.valueOf(regionInfo.getDatabase())); + columnBuilders[5].writeInt(regionInfo.getSeriesSlots()); + columnBuilders[6].writeLong(regionInfo.getTimeSlots()); + columnBuilders[7].writeBinary(BytesUtils.valueOf(regionInfo.getClientRpcIp())); + columnBuilders[8].writeInt(regionInfo.getClientRpcPort()); + columnBuilders[9].writeBinary(BytesUtils.valueOf(regionInfo.getInternalAddress())); + columnBuilders[10].writeBinary(BytesUtils.valueOf(regionInfo.getRoleType())); + columnBuilders[11].writeLong(regionInfo.getCreateTime()); + if (regionInfo.getConsensusGroupId().getType().ordinal() + == TConsensusGroupType.DataRegion.ordinal()) { + columnBuilders[12].writeLong(regionInfo.getTsFileSize()); + } else { + columnBuilders[12].appendNull(); + } + resultBuilder.declarePosition(); + } + + @Override + public boolean hasNext() { + return iterator.hasNext(); + } + } + + private static class PipeSupplier extends TsBlockSupplier { + private Iterator iterator; + + private PipeSupplier(final List dataTypes) { + super(dataTypes); + try (final ConfigNodeClient client = + ConfigNodeClientManager.getInstance().borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { + iterator = + client.showPipe(new TShowPipeReq().setIsTableModel(true)).getPipeInfoListIterator(); + } catch (final Exception e) { + lastException = e; + } + } + + @Override + protected void constructLine() { + final TShowPipeInfo tPipeInfo = iterator.next(); + columnBuilders[0].writeBinary(new Binary(tPipeInfo.getId(), TSFileConfig.STRING_CHARSET)); + columnBuilders[1].writeLong( + TimestampPrecisionUtils.convertToCurrPrecision( + tPipeInfo.getCreationTime(), TimeUnit.MILLISECONDS)); + columnBuilders[2].writeBinary(new Binary(tPipeInfo.getState(), TSFileConfig.STRING_CHARSET)); + columnBuilders[3].writeBinary( + new Binary(tPipeInfo.getPipeExtractor(), TSFileConfig.STRING_CHARSET)); + columnBuilders[4].writeBinary( + new Binary(tPipeInfo.getPipeProcessor(), TSFileConfig.STRING_CHARSET)); + columnBuilders[5].writeBinary( + new Binary(tPipeInfo.getPipeConnector(), TSFileConfig.STRING_CHARSET)); + columnBuilders[6].writeBinary( + new Binary(tPipeInfo.getExceptionMessage(), TSFileConfig.STRING_CHARSET)); + + // Optional, default 0/0.0 + long remainingEventCount = tPipeInfo.getRemainingEventCount(); + double remainingTime = tPipeInfo.getEstimatedRemainingTime(); + + if (remainingEventCount == -1 && remainingTime == -1) { + final Pair remainingEventAndTime = + PipeDataNodeRemainingEventAndTimeMetrics.getInstance() + .getRemainingEventAndTime(tPipeInfo.getId(), tPipeInfo.getCreationTime()); + remainingEventCount = remainingEventAndTime.getLeft(); + remainingTime = remainingEventAndTime.getRight(); + } + + columnBuilders[7].writeLong(tPipeInfo.isSetRemainingEventCount() ? remainingEventCount : -1); + columnBuilders[8].writeDouble(tPipeInfo.isSetEstimatedRemainingTime() ? remainingTime : -1); + + resultBuilder.declarePosition(); + } + + @Override + public boolean hasNext() { + return iterator.hasNext(); + } + } + + private static class PipePluginSupplier extends TsBlockSupplier { + private Iterator iterator; + + private PipePluginSupplier(final List dataTypes) { + super(dataTypes); + try (final ConfigNodeClient client = + ConfigNodeClientManager.getInstance().borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { + iterator = + client.getPipePluginTable().getAllPipePluginMeta().stream() + .map(PipePluginMeta::deserialize) + .filter( + pipePluginMeta -> + !BuiltinPipePlugin.SHOW_PIPE_PLUGINS_BLACKLIST.contains( + pipePluginMeta.getPluginName())) + .iterator(); + } catch (final Exception e) { + lastException = e; + } + } + + @Override + protected void constructLine() { + final PipePluginMeta pipePluginMeta = iterator.next(); + columnBuilders[0].writeBinary(BytesUtils.valueOf(pipePluginMeta.getPluginName())); + columnBuilders[1].writeBinary( + pipePluginMeta.isBuiltin() ? PIPE_PLUGIN_TYPE_BUILTIN : PIPE_PLUGIN_TYPE_EXTERNAL); + columnBuilders[2].writeBinary(BytesUtils.valueOf(pipePluginMeta.getClassName())); + if (Objects.nonNull(pipePluginMeta.getJarName())) { + columnBuilders[3].writeBinary(BytesUtils.valueOf(pipePluginMeta.getJarName())); + } else { + columnBuilders[3].appendNull(); + } + resultBuilder.declarePosition(); + } + + @Override + public boolean hasNext() { + return iterator.hasNext(); + } + } + + private static class TopicSupplier extends TsBlockSupplier { + private Iterator iterator; + + private TopicSupplier(final List dataTypes) { + super(dataTypes); + try (final ConfigNodeClient client = + ConfigNodeClientManager.getInstance().borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { + iterator = client.showTopic(new TShowTopicReq()).getTopicInfoList().iterator(); + } catch (final Exception e) { + lastException = e; + } + } + + @Override + protected void constructLine() { + final TShowTopicInfo topicInfo = iterator.next(); + columnBuilders[0].writeBinary( + new Binary(topicInfo.getTopicName(), TSFileConfig.STRING_CHARSET)); + columnBuilders[1].writeBinary( + new Binary(topicInfo.getTopicAttributes(), TSFileConfig.STRING_CHARSET)); + resultBuilder.declarePosition(); + } + + @Override + public boolean hasNext() { + return iterator.hasNext(); + } + } + + private static class SubscriptionSupplier extends TsBlockSupplier { + private Iterator iterator; + + private SubscriptionSupplier(final List dataTypes) { + super(dataTypes); + try (final ConfigNodeClient client = + ConfigNodeClientManager.getInstance().borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { + iterator = + client + .showSubscription(new TShowSubscriptionReq()) + .getSubscriptionInfoList() + .iterator(); + } catch (final Exception e) { + lastException = e; } - throw e; } - return true; + + @Override + protected void constructLine() { + final TShowSubscriptionInfo tSubscriptionInfo = iterator.next(); + columnBuilders[0].writeBinary( + new Binary(tSubscriptionInfo.getTopicName(), TSFileConfig.STRING_CHARSET)); + columnBuilders[1].writeBinary( + new Binary(tSubscriptionInfo.getConsumerGroupId(), TSFileConfig.STRING_CHARSET)); + columnBuilders[2].writeBinary( + new Binary(tSubscriptionInfo.getConsumerIds().toString(), TSFileConfig.STRING_CHARSET)); + resultBuilder.declarePosition(); + } + + @Override + public boolean hasNext() { + return iterator.hasNext(); + } } private abstract static class TsBlockSupplier implements Iterator { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/HashAggregationOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/HashAggregationOperator.java index 31ddcbe293ba..ce7e91f84485 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/HashAggregationOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/HashAggregationOperator.java @@ -45,8 +45,6 @@ public class HashAggregationOperator extends AbstractOperator { private static final long INSTANCE_SIZE = RamUsageEstimator.shallowSizeOfInstance(HashAggregationOperator.class); - private final OperatorContext operatorContext; - private final Operator child; private final List groupByTypes; @@ -81,7 +79,7 @@ public HashAggregationOperator( long maxPartialMemory, boolean spillEnabled, long unspillMemoryLimit) { - this.operatorContext = operatorContext; + super.operatorContext = operatorContext; this.child = child; this.groupByTypes = ImmutableList.copyOf(groupByTypes); this.groupByChannels = ImmutableList.copyOf(groupByChannels); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/StreamingAggregationOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/StreamingAggregationOperator.java index 7b0d37c365f3..3ed2bbb1bded 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/StreamingAggregationOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/StreamingAggregationOperator.java @@ -23,7 +23,7 @@ import org.apache.iotdb.db.queryengine.execution.operator.AbstractOperator; import org.apache.iotdb.db.queryengine.execution.operator.Operator; import org.apache.iotdb.db.queryengine.execution.operator.OperatorContext; -import org.apache.iotdb.db.queryengine.execution.operator.source.relational.TableScanOperator; +import org.apache.iotdb.db.queryengine.execution.operator.source.relational.AbstractTableScanOperator; import org.apache.iotdb.db.queryengine.execution.operator.source.relational.aggregation.TableAggregator; import org.apache.iotdb.db.queryengine.plan.relational.type.InternalTypeManager; import org.apache.iotdb.db.utils.datastructure.SortKey; @@ -52,8 +52,6 @@ public class StreamingAggregationOperator extends AbstractOperator { private static final long INSTANCE_SIZE = RamUsageEstimator.shallowSizeOfInstance(StreamingAggregationOperator.class); - private final OperatorContext operatorContext; - private final Operator child; private final List aggregators; @@ -85,7 +83,7 @@ public StreamingAggregationOperator( long maxPartialMemory, boolean spillEnabled, long unSpillMemoryLimit) { - this.operatorContext = operatorContext; + super.operatorContext = operatorContext; this.child = child; this.groupByChannels = Ints.toArray(groupByChannels); this.groupKeyComparator = groupKeyComparator; @@ -208,7 +206,8 @@ private void evaluateAndFlushGroup(TsBlock page, int position, boolean lastCalcu outputs.add( resultBuilder.build( new RunLengthEncodedColumn( - TableScanOperator.TIME_COLUMN_TEMPLATE, resultBuilder.getPositionCount()))); + AbstractTableScanOperator.TIME_COLUMN_TEMPLATE, + resultBuilder.getPositionCount()))); resultBuilder.reset(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/StreamingHashAggregationOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/StreamingHashAggregationOperator.java index 374a6964de22..8862a8137186 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/StreamingHashAggregationOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/StreamingHashAggregationOperator.java @@ -53,8 +53,6 @@ public class StreamingHashAggregationOperator extends AbstractOperator { private static final long INSTANCE_SIZE = RamUsageEstimator.shallowSizeOfInstance(StreamingHashAggregationOperator.class); - private final OperatorContext operatorContext; - private final Operator child; private final int[] preGroupedChannels; @@ -103,7 +101,7 @@ public StreamingHashAggregationOperator( long maxPartialMemory, boolean spillEnabled, long unSpillMemoryLimit) { - this.operatorContext = operatorContext; + super.operatorContext = operatorContext; this.child = child; this.preGroupedChannels = Ints.toArray(preGroupedChannels); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/relational/ColumnTransformerBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/relational/ColumnTransformerBuilder.java index ce69b23c3bcf..badd487b5fe2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/relational/ColumnTransformerBuilder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/relational/ColumnTransformerBuilder.java @@ -86,6 +86,7 @@ import org.apache.iotdb.db.queryengine.transformation.dag.column.leaf.LeafColumnTransformer; import org.apache.iotdb.db.queryengine.transformation.dag.column.leaf.NullColumnTransformer; import org.apache.iotdb.db.queryengine.transformation.dag.column.leaf.TimeColumnTransformer; +import org.apache.iotdb.db.queryengine.transformation.dag.column.multi.AbstractGreatestLeastColumnTransformer; import org.apache.iotdb.db.queryengine.transformation.dag.column.multi.CoalesceColumnTransformer; import org.apache.iotdb.db.queryengine.transformation.dag.column.multi.InBinaryMultiColumnTransformer; import org.apache.iotdb.db.queryengine.transformation.dag.column.multi.InBooleanMultiColumnTransformer; @@ -977,6 +978,20 @@ private ColumnTransformer getFunctionColumnTransformer( } return new FormatColumnTransformer( STRING, columnTransformers, context.sessionInfo.getZoneId()); + } else if (TableBuiltinScalarFunction.GREATEST + .getFunctionName() + .equalsIgnoreCase(functionName)) { + List columnTransformers = + children.stream().map(child -> this.process(child, context)).collect(Collectors.toList()); + Type returnType = columnTransformers.get(0).getType(); + return AbstractGreatestLeastColumnTransformer.getGreatestColumnTransformer( + returnType, columnTransformers); + } else if (TableBuiltinScalarFunction.LEAST.getFunctionName().equalsIgnoreCase(functionName)) { + List columnTransformers = + children.stream().map(child -> this.process(child, context)).collect(Collectors.toList()); + Type returnType = columnTransformers.get(0).getType(); + return AbstractGreatestLeastColumnTransformer.getLeastColumnTransformer( + returnType, columnTransformers); } else { // user defined function if (TableUDFUtils.isScalarFunction(functionName)) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/schedule/DriverScheduler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/schedule/DriverScheduler.java index dfe4fc9a742e..80dec27b4cf5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/schedule/DriverScheduler.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/schedule/DriverScheduler.java @@ -43,7 +43,6 @@ import org.apache.iotdb.db.queryengine.execution.schedule.queue.multilevelqueue.MultilevelPriorityQueue; import org.apache.iotdb.db.queryengine.execution.schedule.task.DriverTask; import org.apache.iotdb.db.queryengine.execution.schedule.task.DriverTaskStatus; -import org.apache.iotdb.db.queryengine.metric.DriverSchedulerMetricSet; import org.apache.iotdb.db.storageengine.rescon.quotas.DataNodeThrottleQuotaManager; import org.apache.iotdb.db.utils.SetThreadName; import org.apache.iotdb.mpp.rpc.thrift.TFragmentInstanceId; @@ -66,15 +65,10 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; -import static org.apache.iotdb.db.queryengine.metric.DriverSchedulerMetricSet.BLOCK_QUEUED_TIME; -import static org.apache.iotdb.db.queryengine.metric.DriverSchedulerMetricSet.READY_QUEUED_TIME; - /** The manager of fragment instances scheduling. */ public class DriverScheduler implements IDriverScheduler, IService { private static final Logger logger = LoggerFactory.getLogger(DriverScheduler.class); - private static final DriverSchedulerMetricSet DRIVER_SCHEDULER_METRIC_SET = - DriverSchedulerMetricSet.getInstance(); private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); private static final double LEVEL_TIME_MULTIPLIER = 2; @@ -485,7 +479,6 @@ public void blockedToReady(DriverTask task) { .getDriverContext() .getFragmentInstanceContext() .addBlockQueuedTime(blockQueuedTime); - DRIVER_SCHEDULER_METRIC_SET.recordTaskQueueTime(BLOCK_QUEUED_TIME, blockQueuedTime); task.setLastEnterReadyQueueTime(currentTime); task.resetLevelScheduledTime(); readyQueue.repush(task); @@ -509,7 +502,6 @@ public boolean readyToRunning(DriverTask task) { .getDriverContext() .getFragmentInstanceContext() .addReadyQueuedTime(readyQueuedTime); - DRIVER_SCHEDULER_METRIC_SET.recordTaskQueueTime(READY_QUEUED_TIME, readyQueuedTime); } finally { task.unlock(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/Coordinator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/Coordinator.java index a0602bec83dc..dd56b601f7e3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/Coordinator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/Coordinator.java @@ -70,13 +70,20 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropDB; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropFunction; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropTable; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ExtendRegion; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Flush; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.KillQuery; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LoadConfiguration; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.MigrateRegion; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.PipeStatement; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ReconstructRegion; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.RelationalAuthorStatement; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.RemoveDataNode; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.RemoveRegion; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SetConfiguration; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SetProperties; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SetSqlDialect; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SetSystemStatus; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowAINodes; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowCluster; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowClusterId; @@ -407,12 +414,15 @@ private IQueryExecution createQueryExecutionForTableModel( || statement instanceof Flush || statement instanceof ClearCache || statement instanceof SetConfiguration + || statement instanceof LoadConfiguration + || statement instanceof SetSystemStatus || statement instanceof StartRepairData || statement instanceof StopRepairData || statement instanceof PipeStatement || statement instanceof RemoveDataNode || statement instanceof SubscriptionStatement || statement instanceof ShowCurrentSqlDialect + || statement instanceof SetSqlDialect || statement instanceof ShowCurrentUser || statement instanceof ShowCurrentDatabase || statement instanceof ShowVersion @@ -423,7 +433,11 @@ private IQueryExecution createQueryExecutionForTableModel( || statement instanceof CreateFunction || statement instanceof DropFunction || statement instanceof ShowFunctions - || statement instanceof RelationalAuthorStatement) { + || statement instanceof RelationalAuthorStatement + || statement instanceof MigrateRegion + || statement instanceof ReconstructRegion + || statement instanceof ExtendRegion + || statement instanceof RemoveRegion) { return new ConfigExecution( queryContext, null, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeUtils.java index ffc6442b89f2..99c1dad2c536 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeUtils.java @@ -329,8 +329,7 @@ public static void analyzeDelete(final Delete node, final MPPQueryContext queryC } @SuppressWarnings("java:S3655") // optional is checked - private static void validateSchema(final Delete node, final MPPQueryContext queryContext) { - final String tableName = node.getTable().getName().getSuffix(); + public static String getDatabaseName(final Delete node, final MPPQueryContext queryContext) { final String databaseName; if (node.getTable().getName().getPrefix().isPresent()) { databaseName = node.getTable().getName().getPrefix().get().toString(); @@ -339,6 +338,12 @@ private static void validateSchema(final Delete node, final MPPQueryContext quer } else { throw new SemanticException(DATABASE_NOT_SPECIFIED); } + return databaseName; + } + + private static void validateSchema(final Delete node, final MPPQueryContext queryContext) { + final String tableName = node.getTable().getName().getSuffix(); + final String databaseName = getDatabaseName(node, queryContext); InformationSchemaUtils.checkDBNameInWrite(databaseName); node.setDatabaseName(databaseName); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java index f7d0f4c8f59f..27111794cc9c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java @@ -220,7 +220,7 @@ import static org.apache.iotdb.db.schemaengine.schemaregion.view.visitor.GetSourcePathsVisitor.getSourcePaths; import static org.apache.iotdb.db.storageengine.load.metrics.LoadTsFileCostMetricsSet.ANALYSIS; import static org.apache.iotdb.db.utils.constant.SqlConstant.COUNT_TIME_HEADER; -import static org.apache.iotdb.db.utils.constant.SqlConstant.ROOT_DOT; +import static org.apache.iotdb.db.utils.constant.SqlConstant.TREE_MODEL_DATABASE_PREFIX; /** This visitor is used to analyze each type of Statement and returns the {@link Analysis}. */ public class AnalyzeVisitor extends StatementVisitor { @@ -2430,7 +2430,7 @@ private void analyzeInto( if (viewPath != null) { try { // if it's really view path, it should start with root. - if (viewPath.startsWith(ROOT_DOT)) { + if (viewPath.startsWith(TREE_MODEL_DATABASE_PREFIX)) { sourcePath = new MeasurementPath(viewPath); } else { // otherwise it should just be an alias diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/IDualKeyCacheStats.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/IDualKeyCacheStats.java index 9ed842cc602d..9247e20a9c48 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/IDualKeyCacheStats.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/IDualKeyCacheStats.java @@ -39,4 +39,10 @@ public interface IDualKeyCacheStats { /** Return current memory usage of dual key cache. */ long memoryUsage(); + + /** Return capacity of dual key cache. */ + long capacity(); + + /** Return entries num of dual key cache */ + long entriesCount(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/CacheStats.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/CacheStats.java index 817092d3eb90..99c331962a20 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/CacheStats.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/CacheStats.java @@ -31,6 +31,7 @@ class CacheStats implements IDualKeyCacheStats { private final long memoryThreshold; private final AtomicLong memoryUsage = new AtomicLong(0); + private final AtomicLong entriesCount = new AtomicLong(0); private final AtomicLong requestCount = new AtomicLong(0); private final AtomicLong hitCount = new AtomicLong(0); @@ -68,6 +69,14 @@ void recordMiss(int num) { requestCount.getAndAdd(num); } + void increaseEntryCount() { + entriesCount.incrementAndGet(); + } + + void decreaseEntryCount() { + entriesCount.decrementAndGet(); + } + @Override public long requestCount() { return requestCount.get(); @@ -96,13 +105,24 @@ public long memoryUsage() { return memoryUsage.get(); } + @Override + public long capacity() { + return memoryThreshold; + } + + @Override + public long entriesCount() { + return entriesCount.get(); + } + void reset() { - resetMemoryUsage(); + resetMemoryUsageAndEntriesCount(); hitCount.set(0); requestCount.set(0); } - void resetMemoryUsage() { + void resetMemoryUsageAndEntriesCount() { memoryUsage.set(0); + entriesCount.set(0); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/DualKeyCacheImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/DualKeyCacheImpl.java index 126f5314495d..9ce72932733e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/DualKeyCacheImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/DualKeyCacheImpl.java @@ -160,6 +160,7 @@ public void put(final FK firstKey, final SK secondKey, final V value) { cacheEntry = cacheEntryManager.createCacheEntry(secondKey, value, finalCacheEntryGroup); cacheEntryManager.put(cacheEntry); + cacheStats.increaseEntryCount(); usedMemorySize.getAndAdd(sizeComputer.computeSecondKeySize(sk)); } else { final V existingValue = cacheEntry.getValue(); @@ -208,6 +209,7 @@ public void update( cacheEntryManager.createCacheEntry( secondKey, value, finalCacheEntryGroup); cacheEntryManager.put(entry); + cacheStats.increaseEntryCount(); usedMemorySize.getAndAdd( sizeComputer.computeSecondKeySize(sk) + sizeComputer.computeValueSize(entry.getValue())); @@ -266,21 +268,25 @@ public void update( if (!firstKeyChecker.test(firstKey)) { continue; } - final ICacheEntryGroup entryGroup = firstKeyMap.get(firstKey); - if (Objects.nonNull(entryGroup)) { - entryGroup - .getAllCacheEntries() - .forEachRemaining( - entry -> { - if (!secondKeyChecker.test(entry.getKey())) { - return; - } - final int result = updater.applyAsInt(entry.getValue().getValue()); - if (Objects.nonNull(entryGroup.getCacheEntry(entry.getKey()))) { - usedMemorySize.getAndAdd(result); - } - }); - } + firstKeyMap.compute( + firstKey, + (fk, entryGroup) -> { + if (Objects.nonNull(entryGroup)) { + entryGroup + .getAllCacheEntries() + .forEachRemaining( + entry -> { + if (!secondKeyChecker.test(entry.getKey())) { + return; + } + final int result = updater.applyAsInt(entry.getValue().getValue()); + if (Objects.nonNull(entryGroup.getCacheEntry(entry.getKey()))) { + usedMemorySize.getAndAdd(result); + } + }); + } + return entryGroup; + }); } increaseMemoryUsageAndMayEvict(usedMemorySize.get()); } @@ -299,30 +305,33 @@ private int evictOneCacheEntry() { } final AtomicInteger evictedSize = new AtomicInteger(0); - evictedSize.getAndAdd(sizeComputer.computeValueSize(evictCacheEntry.getValue())); final ICacheEntryGroup belongedGroup = evictCacheEntry.getBelongedGroup(); evictCacheEntry.setBelongedGroup(null); - belongedGroup.removeCacheEntry(evictCacheEntry.getSecondKey()); - evictedSize.getAndAdd(sizeComputer.computeSecondKeySize(evictCacheEntry.getSecondKey())); - if (belongedGroup.isEmpty()) { - firstKeyMap.compute( - belongedGroup.getFirstKey(), - (firstKey, cacheEntryGroup) -> { - if (cacheEntryGroup == null) { - // has been removed by other threads - return null; - } - if (cacheEntryGroup.isEmpty()) { - evictedSize.getAndAdd(sizeComputer.computeFirstKeySize(firstKey)); - return null; - } + firstKeyMap.compute( + belongedGroup.getFirstKey(), + (firstKey, cacheEntryGroup) -> { + belongedGroup.removeCacheEntry(evictCacheEntry.getSecondKey()); + cacheStats.decreaseEntryCount(); + evictedSize.getAndAdd( + sizeComputer.computeValueSize(evictCacheEntry.getValue()) + + sizeComputer.computeSecondKeySize(evictCacheEntry.getSecondKey())); + + if (cacheEntryGroup == null) { + // has been removed by other threads + return null; + } + + if (cacheEntryGroup.isEmpty()) { + evictedSize.getAndAdd(sizeComputer.computeFirstKeySize(firstKey)); + return null; + } + + // some other thread has put value to it + return cacheEntryGroup; + }); - // some other thread has put value to it - return cacheEntryGroup; - }); - } return evictedSize.get(); } @@ -334,7 +343,7 @@ public void invalidateAll() { private void executeInvalidateAll() { firstKeyMap.clear(); cacheEntryManager.cleanUp(); - cacheStats.resetMemoryUsage(); + cacheStats.resetMemoryUsageAndEntriesCount(); } @Override @@ -364,6 +373,7 @@ public void invalidate(final FK firstKey) { it.hasNext(); ) { final Map.Entry entry = it.next(); if (cacheEntryManager.invalidate(entry.getValue())) { + cacheStats.decreaseEntryCount(); estimateSize += sizeComputer.computeSecondKeySize(entry.getKey()) + sizeComputer.computeValueSize(entry.getValue().getValue()); @@ -387,6 +397,7 @@ public void invalidate(final FK firstKey, final SK secondKey) { final T entry = cacheEntryGroup.getCacheEntry(secondKey); if (Objects.nonNull(entry) && cacheEntryManager.invalidate(entry)) { + cacheStats.decreaseEntryCount(); usedMemorySize.getAndAdd( sizeComputer.computeSecondKeySize(entry.getSecondKey()) + sizeComputer.computeValueSize(entry.getValue())); @@ -418,6 +429,7 @@ public void invalidate(final FK firstKey, final Predicate secondKeyChecker) it.hasNext(); ) { final Map.Entry entry = it.next(); if (cacheEntryManager.invalidate(entry.getValue())) { + cacheStats.decreaseEntryCount(); cacheEntryGroup.removeCacheEntry(entry.getKey()); estimateSize.addAndGet( sizeComputer.computeSecondKeySize(entry.getKey()) @@ -444,28 +456,36 @@ public void invalidate( if (!firstKeyChecker.test(firstKey)) { continue; } - final ICacheEntryGroup entryGroup = firstKeyMap.get(firstKey); - for (final Iterator> it = entryGroup.getAllCacheEntries(); it.hasNext(); ) { - final Map.Entry entry = it.next(); - if (!secondKeyChecker.test(entry.getKey())) { - continue; - } - if (cacheEntryManager.invalidate(entry.getValue())) { - entryGroup.removeCacheEntry(entry.getKey()); - estimateSize.addAndGet( - sizeComputer.computeSecondKeySize(entry.getKey()) - + sizeComputer.computeValueSize(entry.getValue().getValue())); - } - } firstKeyMap.compute( firstKey, - (fk, sk) -> { - if (sk.isEmpty()) { + (fk, cacheEntryGroup) -> { + if (cacheEntryGroup == null) { + // has been removed by other threads + return null; + } + + for (final Iterator> it = cacheEntryGroup.getAllCacheEntries(); + it.hasNext(); ) { + final Map.Entry entry = it.next(); + if (!secondKeyChecker.test(entry.getKey())) { + continue; + } + + if (cacheEntryManager.invalidate(entry.getValue())) { + cacheStats.decreaseEntryCount(); + cacheEntryGroup.removeCacheEntry(entry.getKey()); + estimateSize.addAndGet( + sizeComputer.computeSecondKeySize(entry.getKey()) + + sizeComputer.computeValueSize(entry.getValue().getValue())); + } + } + + if (cacheEntryGroup.isEmpty()) { estimateSize.getAndAdd(sizeComputer.computeFirstKeySize(firstKey)); return null; } - return sk; + return cacheEntryGroup; }); } cacheStats.decreaseMemoryUsage(estimateSize.get()); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java index 92cd9584841c..010fdf84bf5a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java @@ -194,9 +194,16 @@ protected void executeTabletConversion(final IAnalysis analysis, final LoadAnaly : null; if (status == null) { + LOGGER.warn( + "Load: Failed to convert to tablets from statement {}. Status is null.", + isTableModelStatement ? loadTsFileTableStatement : loadTsFileTreeStatement); analysis.setFailStatus( new TSStatus(TSStatusCode.LOAD_FILE_ERROR.getStatusCode()).setMessage(e.getMessage())); } else if (!loadTsFileDataTypeConverter.isSuccessful(status)) { + LOGGER.warn( + "Load: Failed to convert to tablets from statement {}. Status: {}", + isTableModelStatement ? loadTsFileTableStatement : loadTsFileTreeStatement, + status); analysis.setFailStatus(status); } analysis.setFinishQueryAfterAnalyze(true); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/SchemaValidator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/SchemaValidator.java index aaf5d08f43c8..441e882242ad 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/SchemaValidator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/SchemaValidator.java @@ -25,6 +25,8 @@ import org.apache.iotdb.db.queryengine.common.MPPQueryContext; import org.apache.iotdb.db.queryengine.common.schematree.ISchemaTree; import org.apache.iotdb.db.queryengine.plan.relational.metadata.Metadata; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.QualifiedObjectName; +import org.apache.iotdb.db.queryengine.plan.relational.security.AccessControl; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.WrappedInsertStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertBaseStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertMultiTabletsStatement; @@ -39,6 +41,8 @@ import java.util.List; +import static org.apache.iotdb.commons.utils.PathUtils.unQualifyDatabaseName; + public class SchemaValidator { private static final Logger LOGGER = LoggerFactory.getLogger(SchemaValidator.class); @@ -64,10 +68,15 @@ public static void validate( public static void validate( final Metadata metadata, final WrappedInsertStatement insertStatement, - final MPPQueryContext context) { + final MPPQueryContext context, + AccessControl accessControl) { try { - insertStatement.toLowerCase(); insertStatement.validateTableSchema(metadata, context); + accessControl.checkCanInsertIntoTable( + context.getSession().getUserName(), + new QualifiedObjectName( + unQualifyDatabaseName(insertStatement.getDatabase()), + insertStatement.getTableName())); insertStatement.updateAfterSchemaValidation(context); insertStatement.validateDeviceSchema(metadata, context); } catch (final QueryProcessException e) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/IQueryExecution.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/IQueryExecution.java index 366af9fdfca8..d5a35ff8ca8c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/IQueryExecution.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/IQueryExecution.java @@ -72,4 +72,6 @@ public interface IQueryExecution { String getStatementType(); IClientSession.SqlDialect getSQLDialect(); + + String getUser(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/QueryExecution.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/QueryExecution.java index c387e9267b55..84a757893bc2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/QueryExecution.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/QueryExecution.java @@ -693,6 +693,11 @@ public IClientSession.SqlDialect getSQLDialect() { return context.getSession().getSqlDialect(); } + @Override + public String getUser() { + return context.getSession().getUserName(); + } + public MPPQueryContext getContext() { return context; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/ConfigExecution.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/ConfigExecution.java index 92a8a979d653..80d99236c39b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/ConfigExecution.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/ConfigExecution.java @@ -93,7 +93,9 @@ public class ConfigExecution implements IQueryExecution { TSStatusCode.ROLE_NOT_EXIST.getStatusCode(), TSStatusCode.USER_ALREADY_HAS_ROLE.getStatusCode(), TSStatusCode.USER_NOT_HAS_ROLE.getStatusCode(), - TSStatusCode.NOT_HAS_PRIVILEGE_GRANTOPT.getStatusCode()))); + TSStatusCode.NOT_HAS_PRIVILEGE_GRANTOPT.getStatusCode(), + TSStatusCode.SEMANTIC_ERROR.getStatusCode(), + TSStatusCode.NO_SUCH_QUERY.getStatusCode()))); private final MPPQueryContext context; private final ExecutorService executor; @@ -308,4 +310,9 @@ public String getStatementType() { public IClientSession.SqlDialect getSQLDialect() { return context.getSession().getSqlDialect(); } + + @Override + public String getUser() { + return context.getSession().getUserName(); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java index 18dd15c3ae2f..db1c40390104 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java @@ -44,6 +44,10 @@ import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.ShowPipePluginsTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.ShowRegionTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.ShowVariablesTask; +import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.region.ExtendRegionTask; +import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.region.MigrateRegionTask; +import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.region.ReconstructRegionTask; +import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.region.RemoveRegionTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.AlterDBTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.AlterTableAddColumnTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.AlterTableDropColumnTask; @@ -66,6 +70,7 @@ import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.ShowTablesDetailsTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.ShowTablesTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.UseDBTask; +import org.apache.iotdb.db.queryengine.plan.execution.config.session.SetSqlDialectTask; import org.apache.iotdb.db.queryengine.plan.execution.config.session.ShowCurrentDatabaseTask; import org.apache.iotdb.db.queryengine.plan.execution.config.session.ShowCurrentSqlDialectTask; import org.apache.iotdb.db.queryengine.plan.execution.config.session.ShowCurrentTimestampTask; @@ -73,7 +78,9 @@ import org.apache.iotdb.db.queryengine.plan.execution.config.session.ShowVersionTask; import org.apache.iotdb.db.queryengine.plan.execution.config.sys.FlushTask; import org.apache.iotdb.db.queryengine.plan.execution.config.sys.KillQueryTask; +import org.apache.iotdb.db.queryengine.plan.execution.config.sys.LoadConfigurationTask; import org.apache.iotdb.db.queryengine.plan.execution.config.sys.SetConfigurationTask; +import org.apache.iotdb.db.queryengine.plan.execution.config.sys.SetSystemStatusTask; import org.apache.iotdb.db.queryengine.plan.execution.config.sys.StartRepairDataTask; import org.apache.iotdb.db.queryengine.plan.execution.config.sys.StopRepairDataTask; import org.apache.iotdb.db.queryengine.plan.execution.config.sys.pipe.AlterPipeTask; @@ -116,19 +123,26 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropTable; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropTopic; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ExtendRegion; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Flush; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.KillQuery; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Literal; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LoadConfiguration; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LongLiteral; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.MigrateRegion; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Node; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Property; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.QualifiedName; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ReconstructRegion; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.RelationalAuthorStatement; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.RemoveDataNode; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.RemoveRegion; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.RenameColumn; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.RenameTable; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SetConfiguration; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SetProperties; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SetSqlDialect; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SetSystemStatus; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowAINodes; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowCluster; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowClusterId; @@ -160,7 +174,9 @@ import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowClusterStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowRegionStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.FlushStatement; +import org.apache.iotdb.db.queryengine.plan.statement.sys.LoadConfigurationStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.SetConfigurationStatement; +import org.apache.iotdb.db.queryengine.plan.statement.sys.SetSystemStatusStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.StartRepairDataStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.StopRepairDataStatement; import org.apache.iotdb.db.schemaengine.table.InformationSchemaUtils; @@ -730,6 +746,20 @@ protected IConfigTask visitStopRepairData(StopRepairData node, MPPQueryContext c return new StopRepairDataTask(((StopRepairDataStatement) node.getInnerTreeStatement())); } + @Override + protected IConfigTask visitLoadConfiguration(LoadConfiguration node, MPPQueryContext context) { + context.setQueryType(QueryType.WRITE); + accessControl.checkUserHasMaintainPrivilege(context.getSession().getUserName()); + return new LoadConfigurationTask(((LoadConfigurationStatement) node.getInnerTreeStatement())); + } + + @Override + protected IConfigTask visitSetSystemStatus(SetSystemStatus node, MPPQueryContext context) { + context.setQueryType(QueryType.WRITE); + accessControl.checkUserHasMaintainPrivilege(context.getSession().getUserName()); + return new SetSystemStatusTask(((SetSystemStatusStatement) node.getInnerTreeStatement())); + } + private Optional parseStringFromLiteralIfBinary(final Object value) { return value instanceof Literal && ((Literal) value).getTsValue() instanceof Binary ? Optional.of( @@ -920,6 +950,12 @@ protected IConfigTask visitShowCurrentSqlDialect( return new ShowCurrentSqlDialectTask(context.getSession().getSqlDialect().name()); } + @Override + protected IConfigTask visitSetSqlDialect(SetSqlDialect node, MPPQueryContext context) { + context.setQueryType(QueryType.WRITE); + return new SetSqlDialectTask(node.getSqlDialect()); + } + @Override protected IConfigTask visitShowCurrentDatabase( ShowCurrentDatabase node, MPPQueryContext context) { @@ -966,12 +1002,14 @@ protected IConfigTask visitRelationalAuthorPlan( @Override protected IConfigTask visitKillQuery(KillQuery node, MPPQueryContext context) { context.setQueryType(QueryType.WRITE); + accessControl.checkUserHasMaintainPrivilege(context.getSession().getUserName()); return new KillQueryTask(node); } @Override protected IConfigTask visitCreateFunction(CreateFunction node, MPPQueryContext context) { context.setQueryType(QueryType.WRITE); + accessControl.checkUserIsAdmin(context.getSession().getUserName()); if (node.getUriString().map(ExecutableManager::isUriTrusted).orElse(true)) { // 1. user specified uri and that uri is trusted // 2. user doesn't specify uri @@ -991,6 +1029,44 @@ protected IConfigTask visitShowFunctions(ShowFunctions node, MPPQueryContext con @Override protected IConfigTask visitDropFunction(DropFunction node, MPPQueryContext context) { context.setQueryType(QueryType.WRITE); + accessControl.checkUserIsAdmin(context.getSession().getUserName()); return new DropFunctionTask(Model.TABLE, node.getUdfName()); } + + @Override + protected IConfigTask visitMigrateRegion(MigrateRegion migrateRegion, MPPQueryContext context) { + context.setQueryType(QueryType.WRITE); + accessControl.checkUserHasMaintainPrivilege(context.getSession().getUserName()); + // As the implementation is identical, we'll simply translate to the + // corresponding tree-model variant and execute that. + return new MigrateRegionTask(migrateRegion); + } + + @Override + protected IConfigTask visitReconstructRegion( + ReconstructRegion reconstructRegion, MPPQueryContext context) { + context.setQueryType(QueryType.WRITE); + accessControl.checkUserHasMaintainPrivilege(context.getSession().getUserName()); + // As the implementation is identical, we'll simply translate to the + // corresponding tree-model variant and execute that. + return new ReconstructRegionTask(reconstructRegion); + } + + @Override + protected IConfigTask visitExtendRegion(ExtendRegion extendRegion, MPPQueryContext context) { + context.setQueryType(QueryType.WRITE); + accessControl.checkUserHasMaintainPrivilege(context.getSession().getUserName()); + // As the implementation is identical, we'll simply translate to the + // corresponding tree-model variant and execute that. + return new ExtendRegionTask(extendRegion); + } + + @Override + protected IConfigTask visitRemoveRegion(RemoveRegion removeRegion, MPPQueryContext context) { + context.setQueryType(QueryType.WRITE); + accessControl.checkUserHasMaintainPrivilege(context.getSession().getUserName()); + // As the implementation is identical, we'll simply translate to the + // corresponding tree-model variant and execute that. + return new RemoveRegionTask(removeRegion); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TreeConfigTaskVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TreeConfigTaskVisitor.java index 7ec00aacf58d..39fba9d7b318 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TreeConfigTaskVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TreeConfigTaskVisitor.java @@ -76,6 +76,9 @@ import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.view.AlterLogicalViewTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.view.DeleteLogicalViewTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.view.RenameLogicalViewTask; +import org.apache.iotdb.db.queryengine.plan.execution.config.session.SetSqlDialectTask; +import org.apache.iotdb.db.queryengine.plan.execution.config.session.ShowCurrentSqlDialectTask; +import org.apache.iotdb.db.queryengine.plan.execution.config.session.ShowCurrentUserTask; import org.apache.iotdb.db.queryengine.plan.execution.config.sys.AuthorizerTask; import org.apache.iotdb.db.queryengine.plan.execution.config.sys.ClearCacheTask; import org.apache.iotdb.db.queryengine.plan.execution.config.sys.FlushTask; @@ -172,7 +175,10 @@ import org.apache.iotdb.db.queryengine.plan.statement.sys.LoadConfigurationStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.MergeStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.SetConfigurationStatement; +import org.apache.iotdb.db.queryengine.plan.statement.sys.SetSqlDialectStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.SetSystemStatusStatement; +import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowCurrentSqlDialectStatement; +import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowCurrentUserStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.StartRepairDataStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.StopRepairDataStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.TestConnectionStatement; @@ -702,6 +708,12 @@ public IConfigTask visitShowThrottleQuota( return new ShowThrottleQuotaTask(showThrottleQuotaStatement); } + @Override + public IConfigTask visitSetSqlDialect( + SetSqlDialectStatement setSqlDialectStatement, MPPQueryContext context) { + return new SetSqlDialectTask(setSqlDialectStatement.getSqlDialect()); + } + /** AI Model Management */ @Override public IConfigTask visitCreateModel( @@ -727,4 +739,15 @@ public IConfigTask visitShowModels( ShowModelsStatement showModelsStatement, MPPQueryContext context) { return new ShowModelsTask(showModelsStatement.getModelName()); } + + @Override + public IConfigTask visitShowCurrentUser(ShowCurrentUserStatement node, MPPQueryContext context) { + return new ShowCurrentUserTask(context.getSession().getUserName()); + } + + @Override + public IConfigTask visitShowCurrentSqlDialect( + ShowCurrentSqlDialectStatement node, MPPQueryContext context) { + return new ShowCurrentSqlDialectTask(context.getSession().getSqlDialect().name()); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java index 4a70c5991c98..0eac1e050c98 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java @@ -150,13 +150,13 @@ import org.apache.iotdb.db.exception.StorageEngineException; import org.apache.iotdb.db.exception.metadata.PathNotExistException; import org.apache.iotdb.db.exception.metadata.SchemaQuotaExceededException; -import org.apache.iotdb.db.exception.sql.SemanticException; import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent; import org.apache.iotdb.db.protocol.client.ConfigNodeClient; import org.apache.iotdb.db.protocol.client.ConfigNodeClientManager; import org.apache.iotdb.db.protocol.client.ConfigNodeInfo; import org.apache.iotdb.db.protocol.client.DataNodeClientPoolFactory; import org.apache.iotdb.db.protocol.session.IClientSession; +import org.apache.iotdb.db.protocol.session.SessionManager; import org.apache.iotdb.db.queryengine.common.MPPQueryContext; import org.apache.iotdb.db.queryengine.common.SessionInfo; import org.apache.iotdb.db.queryengine.common.schematree.ISchemaTree; @@ -187,6 +187,10 @@ import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.ShowTriggersTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.ShowVariablesTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.model.ShowModelsTask; +import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.region.ExtendRegionTask; +import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.region.MigrateRegionTask; +import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.region.ReconstructRegionTask; +import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.region.RemoveRegionTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.DeleteDeviceTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.DescribeTableDetailsTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.DescribeTableTask; @@ -241,10 +245,6 @@ import org.apache.iotdb.db.queryengine.plan.statement.metadata.pipe.ShowPipesStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.pipe.StartPipeStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.pipe.StopPipeStatement; -import org.apache.iotdb.db.queryengine.plan.statement.metadata.region.ExtendRegionStatement; -import org.apache.iotdb.db.queryengine.plan.statement.metadata.region.MigrateRegionStatement; -import org.apache.iotdb.db.queryengine.plan.statement.metadata.region.ReconstructRegionStatement; -import org.apache.iotdb.db.queryengine.plan.statement.metadata.region.RemoveRegionStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.subscription.CreateTopicStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.subscription.DropTopicStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.subscription.ShowSubscriptionsStatement; @@ -1344,19 +1344,29 @@ public SettableFuture setSystemStatus(boolean onCluster, NodeS public SettableFuture killQuery(final KillQueryStatement killQueryStatement) { int dataNodeId = -1; String queryId = killQueryStatement.getQueryId(); + SettableFuture future = SettableFuture.create(); if (!killQueryStatement.isKillAll()) { String[] splits = queryId.split("_"); try { // We just judge the input queryId has three '_' and the DataNodeId from it is non-negative // here if (splits.length != 4 || ((dataNodeId = Integer.parseInt(splits[3])) < 0)) { - throw new SemanticException("Please ensure your input is correct"); + future.setException( + new IoTDBException( + "Please ensure your input is correct", + TSStatusCode.SEMANTIC_ERROR.getStatusCode(), + true)); + return future; } } catch (NumberFormatException e) { - throw new SemanticException("Please ensure your input is correct"); + future.setException( + new IoTDBException( + "Please ensure your input is correct", + TSStatusCode.SEMANTIC_ERROR.getStatusCode(), + true)); + return future; } } - SettableFuture future = SettableFuture.create(); try (ConfigNodeClient client = CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { final TSStatus executionStatus = client.killQuery(queryId, dataNodeId); @@ -1436,6 +1446,18 @@ public SettableFuture showCurrentSqlDialect(final String sqlDi return future; } + @Override + public SettableFuture setSqlDialect(IClientSession.SqlDialect sqlDialect) { + final SettableFuture future = SettableFuture.create(); + try { + SessionManager.getInstance().getCurrSession().setSqlDialect(sqlDialect); + future.set(new ConfigTaskResult(TSStatusCode.SUCCESS_STATUS)); + } catch (Exception e) { + future.setException(e); + } + return future; + } + @Override public SettableFuture showCurrentDatabase( @Nullable final String currentDatabase) { @@ -2780,16 +2802,16 @@ public SettableFuture countTimeSlotList( } @Override - public SettableFuture migrateRegion( - final MigrateRegionStatement migrateRegionStatement) { + public SettableFuture migrateRegion(final MigrateRegionTask migrateRegionTask) { final SettableFuture future = SettableFuture.create(); try (ConfigNodeClient configNodeClient = CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { final TMigrateRegionReq tMigrateRegionReq = new TMigrateRegionReq( - migrateRegionStatement.getRegionId(), - migrateRegionStatement.getFromId(), - migrateRegionStatement.getToId()); + migrateRegionTask.getStatement().getRegionId(), + migrateRegionTask.getStatement().getFromId(), + migrateRegionTask.getStatement().getToId(), + migrateRegionTask.getModel()); final TSStatus status = configNodeClient.migrateRegion(tMigrateRegionReq); if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { future.setException(new IoTDBException(status.message, status.code)); @@ -2869,14 +2891,15 @@ public SettableFuture removeDataNode( @Override public SettableFuture reconstructRegion( - ReconstructRegionStatement reconstructRegionStatement) { + ReconstructRegionTask reconstructRegionTask) { final SettableFuture future = SettableFuture.create(); try (ConfigNodeClient configNodeClient = CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { final TReconstructRegionReq req = new TReconstructRegionReq( - reconstructRegionStatement.getRegionIds(), - reconstructRegionStatement.getDataNodeId()); + reconstructRegionTask.getStatement().getRegionIds(), + reconstructRegionTask.getStatement().getDataNodeId(), + reconstructRegionTask.getModel()); final TSStatus status = configNodeClient.reconstructRegion(req); if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { future.setException(new IoTDBException(status.message, status.code)); @@ -2891,14 +2914,15 @@ public SettableFuture reconstructRegion( } @Override - public SettableFuture extendRegion( - ExtendRegionStatement extendRegionStatement) { + public SettableFuture extendRegion(ExtendRegionTask extendRegionTask) { final SettableFuture future = SettableFuture.create(); try (ConfigNodeClient configNodeClient = CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { final TExtendRegionReq req = new TExtendRegionReq( - extendRegionStatement.getRegionId(), extendRegionStatement.getDataNodeId()); + extendRegionTask.getStatement().getRegionId(), + extendRegionTask.getStatement().getDataNodeId(), + extendRegionTask.getModel()); final TSStatus status = configNodeClient.extendRegion(req); if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { future.setException(new IoTDBException(status.message, status.code)); @@ -2913,14 +2937,15 @@ public SettableFuture extendRegion( } @Override - public SettableFuture removeRegion( - RemoveRegionStatement removeRegionStatement) { + public SettableFuture removeRegion(RemoveRegionTask removeRegionTask) { final SettableFuture future = SettableFuture.create(); try (ConfigNodeClient configNodeClient = CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { final TRemoveRegionReq req = new TRemoveRegionReq( - removeRegionStatement.getRegionId(), removeRegionStatement.getDataNodeId()); + removeRegionTask.getStatement().getRegionId(), + removeRegionTask.getStatement().getDataNodeId(), + removeRegionTask.getModel()); final TSStatus status = configNodeClient.removeRegion(req); if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { future.setException(new IoTDBException(status.message, status.code)); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java index d34ae66ac16e..1372598493a1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java @@ -35,6 +35,10 @@ import org.apache.iotdb.db.queryengine.common.MPPQueryContext; import org.apache.iotdb.db.queryengine.common.SessionInfo; import org.apache.iotdb.db.queryengine.plan.execution.config.ConfigTaskResult; +import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.region.ExtendRegionTask; +import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.region.MigrateRegionTask; +import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.region.ReconstructRegionTask; +import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.region.RemoveRegionTask; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write.view.AlterLogicalViewNode; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DeleteDevice; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropDB; @@ -67,10 +71,6 @@ import org.apache.iotdb.db.queryengine.plan.statement.metadata.pipe.ShowPipesStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.pipe.StartPipeStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.pipe.StopPipeStatement; -import org.apache.iotdb.db.queryengine.plan.statement.metadata.region.ExtendRegionStatement; -import org.apache.iotdb.db.queryengine.plan.statement.metadata.region.MigrateRegionStatement; -import org.apache.iotdb.db.queryengine.plan.statement.metadata.region.ReconstructRegionStatement; -import org.apache.iotdb.db.queryengine.plan.statement.metadata.region.RemoveRegionStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.subscription.CreateTopicStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.subscription.DropTopicStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.subscription.ShowSubscriptionsStatement; @@ -253,14 +253,13 @@ SettableFuture getTimeSlotList( SettableFuture countTimeSlotList( CountTimeSlotListStatement countTimeSlotListStatement); - SettableFuture migrateRegion(MigrateRegionStatement migrateRegionStatement); + SettableFuture migrateRegion(MigrateRegionTask migrateRegionTask); - SettableFuture reconstructRegion( - ReconstructRegionStatement reconstructRegionStatement); + SettableFuture reconstructRegion(ReconstructRegionTask reconstructRegionTask); - SettableFuture extendRegion(ExtendRegionStatement extendRegionStatement); + SettableFuture extendRegion(ExtendRegionTask extendRegionTask); - SettableFuture removeRegion(RemoveRegionStatement removeRegionStatement); + SettableFuture removeRegion(RemoveRegionTask removeRegionTask); SettableFuture removeDataNode(RemoveDataNodeStatement removeDataNodeStatement); @@ -373,6 +372,8 @@ SettableFuture deleteDevice( SettableFuture showCurrentSqlDialect(String sqlDialect); + SettableFuture setSqlDialect(IClientSession.SqlDialect sqlDialect); + SettableFuture showCurrentUser(String currentUser); SettableFuture showCurrentDatabase(@Nullable String currentDatabase); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/ShowPipePluginsTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/ShowPipePluginsTask.java index 3292d838e206..f186702d595c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/ShowPipePluginsTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/ShowPipePluginsTask.java @@ -46,8 +46,8 @@ public class ShowPipePluginsTask implements IConfigTask { - private static final Binary PIPE_PLUGIN_TYPE_BUILTIN = BytesUtils.valueOf("Builtin"); - private static final Binary PIPE_PLUGIN_TYPE_EXTERNAL = BytesUtils.valueOf("External"); + public static final Binary PIPE_PLUGIN_TYPE_BUILTIN = BytesUtils.valueOf("Builtin"); + public static final Binary PIPE_PLUGIN_TYPE_EXTERNAL = BytesUtils.valueOf("External"); private static final Binary PIPE_JAR_NAME_EMPTY_FIELD = BytesUtils.valueOf(""); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/region/ExtendRegionTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/region/ExtendRegionTask.java index 00dafd9d1549..1ffce0109276 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/region/ExtendRegionTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/region/ExtendRegionTask.java @@ -19,9 +19,11 @@ package org.apache.iotdb.db.queryengine.plan.execution.config.metadata.region; +import org.apache.iotdb.common.rpc.thrift.Model; import org.apache.iotdb.db.queryengine.plan.execution.config.ConfigTaskResult; import org.apache.iotdb.db.queryengine.plan.execution.config.IConfigTask; import org.apache.iotdb.db.queryengine.plan.execution.config.executor.IConfigTaskExecutor; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ExtendRegion; import org.apache.iotdb.db.queryengine.plan.statement.metadata.region.ExtendRegionStatement; import com.google.common.util.concurrent.ListenableFuture; @@ -29,14 +31,30 @@ public class ExtendRegionTask implements IConfigTask { protected final ExtendRegionStatement statement; + private final Model model; public ExtendRegionTask(ExtendRegionStatement statement) { this.statement = statement; + this.model = Model.TREE; + } + + public ExtendRegionTask(ExtendRegion extendRegion) { + this.statement = + new ExtendRegionStatement(extendRegion.getRegionId(), extendRegion.getDataNodeId()); + this.model = Model.TABLE; } @Override public ListenableFuture execute(IConfigTaskExecutor configTaskExecutor) throws InterruptedException { - return configTaskExecutor.extendRegion(statement); + return configTaskExecutor.extendRegion(this); + } + + public ExtendRegionStatement getStatement() { + return statement; + } + + public Model getModel() { + return model; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/region/MigrateRegionTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/region/MigrateRegionTask.java index fd22585aa8cf..8c6f2bfe36ba 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/region/MigrateRegionTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/region/MigrateRegionTask.java @@ -19,9 +19,11 @@ package org.apache.iotdb.db.queryengine.plan.execution.config.metadata.region; +import org.apache.iotdb.common.rpc.thrift.Model; import org.apache.iotdb.db.queryengine.plan.execution.config.ConfigTaskResult; import org.apache.iotdb.db.queryengine.plan.execution.config.IConfigTask; import org.apache.iotdb.db.queryengine.plan.execution.config.executor.IConfigTaskExecutor; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.MigrateRegion; import org.apache.iotdb.db.queryengine.plan.statement.metadata.region.MigrateRegionStatement; import com.google.common.util.concurrent.ListenableFuture; @@ -29,15 +31,32 @@ public class MigrateRegionTask implements IConfigTask { protected final MigrateRegionStatement statement; + private final Model model; public MigrateRegionTask(MigrateRegionStatement migrateRegionStatement) { this.statement = migrateRegionStatement; + this.model = Model.TREE; + } + + public MigrateRegionTask(MigrateRegion migrateRegion) { + this.statement = + new MigrateRegionStatement( + migrateRegion.getRegionId(), migrateRegion.getFromId(), migrateRegion.getToId()); + this.model = Model.TABLE; } @Override public ListenableFuture execute(IConfigTaskExecutor configTaskExecutor) { // If the action is executed successfully, return the Future. // If your operation is async, you can return the corresponding future directly. - return configTaskExecutor.migrateRegion(statement); + return configTaskExecutor.migrateRegion(this); + } + + public Model getModel() { + return this.model; + } + + public MigrateRegionStatement getStatement() { + return statement; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/region/ReconstructRegionTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/region/ReconstructRegionTask.java index f5276ed64e7c..e1d7127f749e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/region/ReconstructRegionTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/region/ReconstructRegionTask.java @@ -19,24 +19,43 @@ package org.apache.iotdb.db.queryengine.plan.execution.config.metadata.region; +import org.apache.iotdb.common.rpc.thrift.Model; import org.apache.iotdb.db.queryengine.plan.execution.config.ConfigTaskResult; import org.apache.iotdb.db.queryengine.plan.execution.config.IConfigTask; import org.apache.iotdb.db.queryengine.plan.execution.config.executor.IConfigTaskExecutor; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ReconstructRegion; import org.apache.iotdb.db.queryengine.plan.statement.metadata.region.ReconstructRegionStatement; import com.google.common.util.concurrent.ListenableFuture; public class ReconstructRegionTask implements IConfigTask { - protected final ReconstructRegionStatement reconstructRegionStatement; + protected final ReconstructRegionStatement statement; + private final Model model; public ReconstructRegionTask(ReconstructRegionStatement reconstructRegionStatement) { - this.reconstructRegionStatement = reconstructRegionStatement; + this.statement = reconstructRegionStatement; + this.model = Model.TREE; + } + + public ReconstructRegionTask(ReconstructRegion reconstructRegion) { + this.statement = + new ReconstructRegionStatement( + reconstructRegion.getDataNodeId(), reconstructRegion.getRegionIds()); + this.model = Model.TABLE; } @Override public ListenableFuture execute(IConfigTaskExecutor configTaskExecutor) throws InterruptedException { - return configTaskExecutor.reconstructRegion(reconstructRegionStatement); + return configTaskExecutor.reconstructRegion(this); + } + + public Model getModel() { + return model; + } + + public ReconstructRegionStatement getStatement() { + return statement; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/region/RemoveRegionTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/region/RemoveRegionTask.java index 26ecea097f74..86d4bafc9db8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/region/RemoveRegionTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/region/RemoveRegionTask.java @@ -19,9 +19,11 @@ package org.apache.iotdb.db.queryengine.plan.execution.config.metadata.region; +import org.apache.iotdb.common.rpc.thrift.Model; import org.apache.iotdb.db.queryengine.plan.execution.config.ConfigTaskResult; import org.apache.iotdb.db.queryengine.plan.execution.config.IConfigTask; import org.apache.iotdb.db.queryengine.plan.execution.config.executor.IConfigTaskExecutor; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.RemoveRegion; import org.apache.iotdb.db.queryengine.plan.statement.metadata.region.RemoveRegionStatement; import com.google.common.util.concurrent.ListenableFuture; @@ -29,14 +31,30 @@ public class RemoveRegionTask implements IConfigTask { protected final RemoveRegionStatement statement; + private final Model model; public RemoveRegionTask(RemoveRegionStatement statement) { this.statement = statement; + this.model = Model.TREE; + } + + public RemoveRegionTask(RemoveRegion removeRegion) { + this.statement = + new RemoveRegionStatement(removeRegion.getRegionId(), removeRegion.getDataNodeId()); + this.model = Model.TABLE; } @Override public ListenableFuture execute(IConfigTaskExecutor configTaskExecutor) throws InterruptedException { - return configTaskExecutor.removeRegion(statement); + return configTaskExecutor.removeRegion(this); + } + + public RemoveRegionStatement getStatement() { + return statement; + } + + public Model getModel() { + return model; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/session/SetSqlDialectTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/session/SetSqlDialectTask.java new file mode 100644 index 000000000000..df4e3d3c641f --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/session/SetSqlDialectTask.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.execution.config.session; + +import org.apache.iotdb.db.protocol.session.IClientSession; +import org.apache.iotdb.db.queryengine.plan.execution.config.ConfigTaskResult; +import org.apache.iotdb.db.queryengine.plan.execution.config.IConfigTask; +import org.apache.iotdb.db.queryengine.plan.execution.config.executor.IConfigTaskExecutor; + +import com.google.common.util.concurrent.ListenableFuture; + +public class SetSqlDialectTask implements IConfigTask { + private final IClientSession.SqlDialect sqlDialect; + + public SetSqlDialectTask(IClientSession.SqlDialect sqlDialect) { + this.sqlDialect = sqlDialect; + } + + @Override + public ListenableFuture execute(IConfigTaskExecutor configTaskExecutor) + throws InterruptedException { + return configTaskExecutor.setSqlDialect(sqlDialect); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java index d7118d7e70ee..3b41ae014e52 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java @@ -39,6 +39,7 @@ import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.exception.sql.SemanticException; +import org.apache.iotdb.db.protocol.session.IClientSession; import org.apache.iotdb.db.qp.sql.IoTDBSqlParser; import org.apache.iotdb.db.qp.sql.IoTDBSqlParser.ConnectorAttributeClauseContext; import org.apache.iotdb.db.qp.sql.IoTDBSqlParser.ConstantContext; @@ -213,7 +214,10 @@ import org.apache.iotdb.db.queryengine.plan.statement.sys.KillQueryStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.LoadConfigurationStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.SetConfigurationStatement; +import org.apache.iotdb.db.queryengine.plan.statement.sys.SetSqlDialectStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.SetSystemStatusStatement; +import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowCurrentSqlDialectStatement; +import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowCurrentUserStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowQueriesStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowVersionStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.StartRepairDataStatement; @@ -1522,11 +1526,17 @@ private SelectComponent parseSelectClause( Map aliasToColumnMap = new HashMap<>(); for (IoTDBSqlParser.ResultColumnContext resultColumnContext : ctx.resultColumn()) { ResultColumn resultColumn = parseResultColumn(resultColumnContext); + String columnName = resultColumn.getExpression().getExpressionString(); // __endTime shouldn't be included in resultColumns - if (resultColumn.getExpression().getExpressionString().equals(ColumnHeaderConstant.ENDTIME)) { + if (columnName.equals(ColumnHeaderConstant.ENDTIME)) { queryStatement.setOutputEndTime(true); continue; } + // don't support pure time in select + if (columnName.equals(ColumnHeaderConstant.TIME)) { + throw new SemanticException( + "Time column is no need to appear in SELECT Clause explicitly, it will always be returned if possible"); + } if (resultColumn.hasAlias()) { String alias = resultColumn.getAlias(); if (aliasToColumnMap.containsKey(alias)) { @@ -2515,6 +2525,9 @@ private String[] parsePrivilege(String[] privileges) { continue; } else if (priv.equalsIgnoreCase("ALL")) { for (PrivilegeType type : PrivilegeType.values()) { + if (type.isRelationalPrivilege()) { + continue; + } privSet.add(type.toString()); } continue; @@ -4540,4 +4553,20 @@ private void parseWindowFunctionInInference( public Statement visitShowCurrentTimestamp(IoTDBSqlParser.ShowCurrentTimestampContext ctx) { return new ShowCurrentTimestampStatement(); } + + @Override + public Statement visitSetSqlDialect(IoTDBSqlParser.SetSqlDialectContext ctx) { + return new SetSqlDialectStatement( + ctx.TABLE() == null ? IClientSession.SqlDialect.TREE : IClientSession.SqlDialect.TABLE); + } + + @Override + public Statement visitShowCurrentSqlDialect(IoTDBSqlParser.ShowCurrentSqlDialectContext ctx) { + return new ShowCurrentSqlDialectStatement(); + } + + @Override + public Statement visitShowCurrentUser(IoTDBSqlParser.ShowCurrentUserContext ctx) { + return new ShowCurrentUserStatement(); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java index 967345567b92..860005397dbd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java @@ -635,14 +635,7 @@ public Operator visitInformationSchemaTableScan( return new InformationSchemaTableScanOperator( operatorContext, node.getPlanNodeId(), - getSupplier( - node.getQualifiedObjectName().getObjectName(), - dataTypes, - context - .getDriverContext() - .getFragmentInstanceContext() - .getSessionInfo() - .getUserName())); + getSupplier(node.getQualifiedObjectName().getObjectName(), dataTypes)); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadTsFilePieceNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadTsFilePieceNode.java index 5d4f02378fc2..f578148e014b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadTsFilePieceNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadTsFilePieceNode.java @@ -155,9 +155,9 @@ public static PlanNode deserialize(ByteBuffer buffer) { InputStream stream = new ByteArrayInputStream(buffer.array()); try { ReadWriteIOUtils.readShort(stream); // read PlanNodeType - File tsFile = new File(ReadWriteIOUtils.readString(stream)); - LoadTsFilePieceNode pieceNode = new LoadTsFilePieceNode(new PlanNodeId(""), tsFile); - int tsFileDataSize = ReadWriteIOUtils.readInt(stream); + final File tsFile = new File(ReadWriteIOUtils.readString(stream)); + final LoadTsFilePieceNode pieceNode = new LoadTsFilePieceNode(new PlanNodeId(""), tsFile); + final int tsFileDataSize = ReadWriteIOUtils.readInt(stream); for (int i = 0; i < tsFileDataSize; i++) { TsFileData tsFileData = TsFileData.deserialize(stream); pieceNode.addTsFileData(tsFileData); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java index 65ae1c148357..d4b34dd1eaa2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java @@ -536,15 +536,18 @@ private Scope visitInsert(WrappedInsertStatement insert, Optional scope) InsertBaseStatement innerInsert = insert.getInnerTreeStatement(); innerInsert.semanticCheck(); + innerInsert.toLowerCase(); + innerInsert = AnalyzeUtils.analyzeInsert( context, innerInsert, - () -> SchemaValidator.validate(metadata, insert, context), + () -> SchemaValidator.validate(metadata, insert, context, accessControl), metadata::getOrCreateDataPartition, AnalyzeUtils::computeTableDataPartitionParams, analysis, false); + insert.setInnerTreeStatement(innerInsert); analysis.setScope(insert, ret); @@ -554,7 +557,13 @@ private Scope visitInsert(WrappedInsertStatement insert, Optional scope) @Override protected Scope visitDelete(Delete node, Optional scope) { final Scope ret = Scope.create(); + accessControl.checkCanDeleteFromTable( + sessionContext.getUserName(), + new QualifiedObjectName( + AnalyzeUtils.getDatabaseName(node, queryContext), + node.getTable().getName().getSuffix())); AnalyzeUtils.analyzeDelete(node, queryContext); + analysis.setScope(node, ret); return ret; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java index 607b1c371355..8364ea489b36 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java @@ -541,6 +541,15 @@ && isIntegerNumber(argumentTypes.get(2)))) { + " must have at least two arguments, and first argument must be char type."); } return STRING; + } else if (TableBuiltinScalarFunction.GREATEST.getFunctionName().equalsIgnoreCase(functionName) + || TableBuiltinScalarFunction.LEAST.getFunctionName().equalsIgnoreCase(functionName)) { + if (argumentTypes.size() < 2 || !areAllTypesSameAndComparable(argumentTypes)) { + throw new SemanticException( + "Scalar function " + + functionName.toLowerCase(Locale.ENGLISH) + + " must have at least two arguments, and all type must be the same."); + } + return argumentTypes.get(0); } // builtin aggregation function @@ -884,6 +893,17 @@ public static boolean isTwoTypeComparable(List argumentTypes) { || ((isNumericType(left) || isCharType(left)) && isUnknownType(right)); } + public static boolean areAllTypesSameAndComparable(List argumentTypes) { + if (argumentTypes == null || argumentTypes.isEmpty()) { + return true; + } + Type firstType = argumentTypes.get(0); + if (!firstType.isComparable()) { + return false; + } + return argumentTypes.stream().allMatch(type -> type.equals(firstType)); + } + public static boolean isArithmeticType(Type type) { return INT32.equals(type) || INT64.equals(type) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCache.java index 195e6e896ced..579f8192f4b6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCache.java @@ -508,6 +508,18 @@ long getRequestCount() { return dualKeyCache.stats().requestCount(); } + long getMemoryUsage() { + return dualKeyCache.stats().memoryUsage(); + } + + long capacity() { + return dualKeyCache.stats().capacity(); + } + + long entriesCount() { + return dualKeyCache.stats().entriesCount(); + } + void invalidateLastCache(final @Nonnull String database) { readWriteLock.writeLock().lock(); @@ -668,6 +680,11 @@ public void invalidateTreeSchema() { } public void invalidateAll() { - dualKeyCache.invalidateAll(); + readWriteLock.writeLock().lock(); + try { + dualKeyCache.invalidateAll(); + } finally { + readWriteLock.writeLock().unlock(); + } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCacheMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCacheMetrics.java index 56e1b23de456..e65c418c0c1d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCacheMetrics.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCacheMetrics.java @@ -29,6 +29,10 @@ import java.util.Objects; public class TableDeviceSchemaCacheMetrics implements IMetricSet { + + private static final String SCHEMA_CACHE_TOTAL_USAGE = "schema_cache_total_usage"; + private static final String SCHEMA_CACHE_MEM_CAPACITY = "schema_cache_mem_capacity"; + private final TableDeviceSchemaCache tableDeviceSchemaCache; public TableDeviceSchemaCacheMetrics(final TableDeviceSchemaCache dataNodeSchemaCache) { @@ -55,6 +59,25 @@ public void bindTo(final AbstractMetricService metricService) { "SchemaCache", Tag.TYPE.toString(), "all"); + metricService.createAutoGauge( + Metric.MEM.toString(), + MetricLevel.IMPORTANT, + tableDeviceSchemaCache, + TableDeviceSchemaCache::getMemoryUsage, + Tag.NAME.toString(), + SCHEMA_CACHE_TOTAL_USAGE); + metricService.createAutoGauge( + Metric.MEM.toString(), + MetricLevel.IMPORTANT, + tableDeviceSchemaCache, + TableDeviceSchemaCache::capacity, + Tag.NAME.toString(), + SCHEMA_CACHE_MEM_CAPACITY); + metricService.createAutoGauge( + Metric.CACHE_ENTRIES_NUM.toString(), + MetricLevel.IMPORTANT, + tableDeviceSchemaCache, + TableDeviceSchemaCache::entriesCount); } @Override @@ -73,6 +96,17 @@ public void unbindFrom(final AbstractMetricService metricService) { "SchemaCache", Tag.TYPE.toString(), "all"); + metricService.remove( + MetricType.AUTO_GAUGE, + Metric.MEM.toString(), + Tag.NAME.toString(), + SCHEMA_CACHE_TOTAL_USAGE); + metricService.remove( + MetricType.AUTO_GAUGE, + Metric.MEM.toString(), + Tag.NAME.toString(), + SCHEMA_CACHE_MEM_CAPACITY); + metricService.remove(MetricType.AUTO_GAUGE, Metric.CACHE_ENTRIES_NUM.toString()); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/DataNodeLocationSupplierFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/DataNodeLocationSupplierFactory.java index b91c9eecaf42..c8d92110d7c5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/DataNodeLocationSupplierFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/DataNodeLocationSupplierFactory.java @@ -87,14 +87,20 @@ private static InformationSchemaTableDataNodeLocationSupplier getInstance() { @Override public List getDataNodeLocations(final String tableName) { - if (tableName.equals(InformationSchema.QUERIES)) { - return getReadableDataNodeLocations(); - } else if (tableName.equals(InformationSchema.DATABASES) - || tableName.equals(InformationSchema.TABLES) - || tableName.equals(InformationSchema.COLUMNS)) { - return Collections.singletonList(DataNodeEndPoints.getLocalDataNodeLocation()); - } else { - throw new UnsupportedOperationException("Unknown table: " + tableName); + switch (tableName) { + case InformationSchema.QUERIES: + return getReadableDataNodeLocations(); + case InformationSchema.DATABASES: + case InformationSchema.TABLES: + case InformationSchema.COLUMNS: + case InformationSchema.REGIONS: + case InformationSchema.PIPES: + case InformationSchema.PIPE_PLUGINS: + case InformationSchema.TOPICS: + case InformationSchema.SUBSCRIPTIONS: + return Collections.singletonList(DataNodeEndPoints.getLocalDataNodeLocation()); + default: + throw new UnsupportedOperationException("Unknown table: " + tableName); } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/security/AccessControl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/security/AccessControl.java index ad6494ab582b..a9724b2636cd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/security/AccessControl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/security/AccessControl.java @@ -140,4 +140,12 @@ public interface AccessControl { */ void checkUserCanRunRelationalAuthorStatement( String userName, RelationalAuthorStatement statement); + + /** + * Check if user is admin user + * + * @param userName name of user + * @throws AccessDeniedException if not allowed + */ + void checkUserIsAdmin(String userName); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/security/AccessControlImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/security/AccessControlImpl.java index fd9fd8012f25..cafdd13b138f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/security/AccessControlImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/security/AccessControlImpl.java @@ -21,12 +21,13 @@ import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.auth.entity.PrivilegeType; -import org.apache.iotdb.commons.exception.IoTDBException; +import org.apache.iotdb.commons.exception.auth.AccessDeniedException; import org.apache.iotdb.db.auth.AuthorityChecker; import org.apache.iotdb.db.queryengine.plan.relational.metadata.QualifiedObjectName; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.RelationalAuthorStatement; import org.apache.iotdb.db.queryengine.plan.relational.type.AuthorRType; -import org.apache.iotdb.rpc.TSStatusCode; + +import static org.apache.iotdb.db.auth.AuthorityChecker.ONLY_ADMIN_ALLOWED; public class AccessControlImpl implements AccessControl { @@ -105,10 +106,7 @@ public void checkUserCanRunRelationalAuthorStatement( case CREATE_USER: // admin cannot be created. if (AuthorityChecker.SUPER_USER.equals(statement.getUserName())) { - status = - AuthorityChecker.getTSStatus( - false, "Cannot create user has same name with admin user"); - throw new RuntimeException(new IoTDBException(status.getMessage(), status.getCode())); + throw new AccessDeniedException("Cannot create user has same name with admin user"); } if (AuthorityChecker.SUPER_USER.equals(userName)) { return; @@ -118,8 +116,7 @@ public void checkUserCanRunRelationalAuthorStatement( case DROP_USER: if (AuthorityChecker.SUPER_USER.equals(statement.getUserName()) || statement.getUserName().equals(userName)) { - status = AuthorityChecker.getTSStatus(false, "Cannot drop admin user or yourself"); - throw new RuntimeException(new IoTDBException(status.getMessage(), status.getCode())); + throw new AccessDeniedException("Cannot drop admin user or yourself"); } if (AuthorityChecker.SUPER_USER.equals(userName)) { return; @@ -142,10 +139,7 @@ public void checkUserCanRunRelationalAuthorStatement( return; case CREATE_ROLE: if (AuthorityChecker.SUPER_USER.equals(statement.getRoleName())) { - throw new RuntimeException( - new IoTDBException( - "Cannot create role has same name with admin user", - TSStatusCode.NO_PERMISSION.getStatusCode())); + throw new AccessDeniedException("Cannot create role has same name with admin user"); } if (AuthorityChecker.SUPER_USER.equals(userName)) { return; @@ -155,9 +149,7 @@ public void checkUserCanRunRelationalAuthorStatement( case DROP_ROLE: if (AuthorityChecker.SUPER_USER.equals(statement.getUserName())) { - throw new RuntimeException( - new IoTDBException( - "Cannot drop role with admin name", TSStatusCode.NO_PERMISSION.getStatusCode())); + throw new AccessDeniedException("Cannot drop role with admin name"); } if (AuthorityChecker.SUPER_USER.equals(userName)) { return; @@ -167,9 +159,7 @@ public void checkUserCanRunRelationalAuthorStatement( case GRANT_USER_ROLE: if (AuthorityChecker.SUPER_USER.equals(statement.getUserName())) { - throw new RuntimeException( - new IoTDBException( - "Cannot grant role to admin", TSStatusCode.NO_PERMISSION.getStatusCode())); + throw new AccessDeniedException("Cannot grant role to admin"); } if (AuthorityChecker.SUPER_USER.equals(userName)) { return; @@ -179,9 +169,7 @@ public void checkUserCanRunRelationalAuthorStatement( case REVOKE_USER_ROLE: if (AuthorityChecker.SUPER_USER.equals(statement.getUserName())) { - throw new RuntimeException( - new IoTDBException( - "Cannot revoke role from admin", TSStatusCode.NO_PERMISSION.getStatusCode())); + throw new AccessDeniedException("Cannot revoke role from admin"); } if (AuthorityChecker.SUPER_USER.equals(userName)) { return; @@ -208,10 +196,7 @@ public void checkUserCanRunRelationalAuthorStatement( case REVOKE_ROLE_ANY: case REVOKE_USER_ANY: if (AuthorityChecker.SUPER_USER.equals(statement.getUserName())) { - throw new RuntimeException( - new IoTDBException( - "Cannot grant/revoke privileges to/from admin", - TSStatusCode.NO_PERMISSION.getStatusCode())); + throw new AccessDeniedException("Cannot grant/revoke privileges of admin user"); } if (AuthorityChecker.SUPER_USER.equals(userName)) { return; @@ -226,10 +211,7 @@ public void checkUserCanRunRelationalAuthorStatement( case GRANT_USER_ALL: case REVOKE_USER_ALL: if (AuthorityChecker.SUPER_USER.equals(statement.getUserName())) { - throw new RuntimeException( - new IoTDBException( - "Cannot grant/revoke all privileges to/from admin", - TSStatusCode.NO_PERMISSION.getStatusCode())); + throw new AccessDeniedException("Cannot grant/revoke all privileges of admin user"); } if (AuthorityChecker.SUPER_USER.equals(userName)) { return; @@ -248,10 +230,7 @@ public void checkUserCanRunRelationalAuthorStatement( case REVOKE_USER_DB: case REVOKE_ROLE_DB: if (AuthorityChecker.SUPER_USER.equals(statement.getUserName())) { - throw new RuntimeException( - new IoTDBException( - "Cannot grant/revoke privileges of admin user", - TSStatusCode.NO_PERMISSION.getStatusCode())); + throw new AccessDeniedException("Cannot grant/revoke privileges of admin user"); } if (AuthorityChecker.SUPER_USER.equals(userName)) { return; @@ -268,10 +247,7 @@ public void checkUserCanRunRelationalAuthorStatement( case REVOKE_USER_TB: case REVOKE_ROLE_TB: if (AuthorityChecker.SUPER_USER.equals(statement.getUserName())) { - throw new RuntimeException( - new IoTDBException( - "Cannot grant/revoke privileges of admin user", - TSStatusCode.NO_PERMISSION.getStatusCode())); + throw new AccessDeniedException("Cannot grant/revoke privileges of admin user"); } if (AuthorityChecker.SUPER_USER.equals(userName)) { return; @@ -289,10 +265,7 @@ public void checkUserCanRunRelationalAuthorStatement( case REVOKE_USER_SYS: case REVOKE_ROLE_SYS: if (AuthorityChecker.SUPER_USER.equals(statement.getUserName())) { - throw new RuntimeException( - new IoTDBException( - "Cannot grant/revoke privileges of admin user", - TSStatusCode.NO_PERMISSION.getStatusCode())); + throw new AccessDeniedException("Cannot grant/revoke privileges of admin user"); } if (AuthorityChecker.SUPER_USER.equals(userName)) { return; @@ -305,4 +278,11 @@ public void checkUserCanRunRelationalAuthorStatement( break; } } + + @Override + public void checkUserIsAdmin(String userName) { + if (!AuthorityChecker.SUPER_USER.equals(userName)) { + throw new AccessDeniedException(ONLY_ADMIN_ALLOWED); + } + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/security/AllowAllAccessControl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/security/AllowAllAccessControl.java index cea0393c7dcc..7843e1601e51 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/security/AllowAllAccessControl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/security/AllowAllAccessControl.java @@ -88,4 +88,9 @@ public void checkUserCanRunRelationalAuthorStatement( String userName, RelationalAuthorStatement statement) { // allow anything } + + @Override + public void checkUserIsAdmin(String userName) { + // allow anything + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java index ed67e02098de..efd4e59fa141 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java @@ -433,6 +433,14 @@ protected R visitStopRepairData(StopRepairData node, C context) { return visitStatement(node, context); } + protected R visitLoadConfiguration(LoadConfiguration node, C context) { + return visitStatement(node, context); + } + + protected R visitSetSystemStatus(SetSystemStatus node, C context) { + return visitStatement(node, context); + } + protected R visitInsertRow(InsertRow node, C context) { return visitStatement(node, context); } @@ -636,4 +644,24 @@ protected R visitKillQuery(KillQuery node, C context) { protected R visitRelationalAuthorPlan(RelationalAuthorStatement node, C context) { return visitStatement(node, context); } + + protected R visitMigrateRegion(MigrateRegion node, C context) { + return visitStatement(node, context); + } + + protected R visitReconstructRegion(ReconstructRegion node, C context) { + return visitStatement(node, context); + } + + protected R visitExtendRegion(ExtendRegion node, C context) { + return visitStatement(node, context); + } + + protected R visitRemoveRegion(RemoveRegion node, C context) { + return visitStatement(node, context); + } + + protected R visitSetSqlDialect(SetSqlDialect node, C context) { + return visitStatement(node, context); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ExtendRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ExtendRegion.java new file mode 100644 index 000000000000..687777c85588 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ExtendRegion.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; + +import com.google.common.collect.ImmutableList; + +import java.util.List; +import java.util.Objects; + +public class ExtendRegion extends Statement { + + private final int regionId; + private final int dataNodeId; + + public ExtendRegion(int regionId, int dataNodeId) { + super(null); + this.regionId = regionId; + this.dataNodeId = dataNodeId; + } + + @Override + public List getChildren() { + return ImmutableList.of(); + } + + @Override + public int hashCode() { + return Objects.hash(ExtendRegion.class, regionId, dataNodeId); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof ExtendRegion)) { + return false; + } + ExtendRegion another = (ExtendRegion) obj; + return regionId == another.regionId && dataNodeId == another.dataNodeId; + } + + @Override + public String toString() { + return String.format("extend region %d to datanode %d", regionId, dataNodeId); + } + + @Override + public R accept(AstVisitor visitor, C context) { + return visitor.visitExtendRegion(this, context); + } + + public int getRegionId() { + return regionId; + } + + public int getDataNodeId() { + return dataNodeId; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertRows.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertRows.java index cdfb4c3c39f3..c55c02da0573 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertRows.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertRows.java @@ -61,7 +61,7 @@ public void updateAfterSchemaValidation(MPPQueryContext context) throws QueryPro @Override public String getTableName() { - return getInnerTreeStatement().getDevicePath().getFullPath(); + return getInnerTreeStatement().getInsertRowStatementList().get(0).getDevicePath().getFullPath(); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/LoadConfiguration.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/LoadConfiguration.java new file mode 100644 index 000000000000..b67ed2b3acf9 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/LoadConfiguration.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; + +import org.apache.iotdb.db.queryengine.common.MPPQueryContext; +import org.apache.iotdb.db.queryengine.plan.statement.Statement; + +public class LoadConfiguration extends WrappedStatement { + public LoadConfiguration(Statement innerTreeStatement, MPPQueryContext context) { + super(innerTreeStatement, context); + } + + @Override + public R accept(AstVisitor visitor, C context) { + return visitor.visitLoadConfiguration(this, context); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/MigrateRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/MigrateRegion.java new file mode 100644 index 000000000000..4ec5497f471e --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/MigrateRegion.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; + +import com.google.common.collect.ImmutableList; + +import javax.annotation.Nullable; + +import java.util.List; +import java.util.Objects; + +public class MigrateRegion extends Statement { + private final int regionId; + + private final int fromId; + + private final int toId; + + public MigrateRegion(int regionId, int fromId, int toId) { + this(null, regionId, fromId, toId); + } + + public MigrateRegion(@Nullable NodeLocation location, int regionId, int fromId, int toId) { + super(location); + this.regionId = regionId; + this.fromId = fromId; + this.toId = toId; + } + + @Override + public List getChildren() { + return ImmutableList.of(); + } + + @Override + public int hashCode() { + return Objects.hash(MigrateRegion.class, regionId, fromId, toId); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof MigrateRegion)) { + return false; + } + MigrateRegion another = (MigrateRegion) obj; + return regionId == another.regionId && fromId == another.fromId && toId == another.toId; + } + + @Override + public String toString() { + return String.format("migrate region %d from %d to %d", regionId, fromId, toId); + } + + @Override + public R accept(AstVisitor visitor, C context) { + return visitor.visitMigrateRegion(this, context); + } + + public int getRegionId() { + return regionId; + } + + public int getFromId() { + return fromId; + } + + public int getToId() { + return toId; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ReconstructRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ReconstructRegion.java new file mode 100644 index 000000000000..df4aad226064 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ReconstructRegion.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; + +import com.google.common.collect.ImmutableList; + +import java.util.List; +import java.util.Objects; + +public class ReconstructRegion extends Statement { + final int dataNodeId; + final List regionIds; + + public ReconstructRegion(int dataNodeId, List regionIds) { + super(null); + this.dataNodeId = dataNodeId; + this.regionIds = regionIds; + } + + @Override + public List getChildren() { + return ImmutableList.of(); + } + + @Override + public int hashCode() { + return Objects.hash(ReconstructRegion.class, dataNodeId, regionIds); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof ReconstructRegion)) { + return false; + } + ReconstructRegion another = (ReconstructRegion) obj; + return dataNodeId == another.dataNodeId && regionIds.equals(another.regionIds); + } + + @Override + public String toString() { + return String.format("reconstruct region %s of datanode %d", regionIds, dataNodeId); + } + + @Override + public R accept(AstVisitor visitor, C context) { + return visitor.visitReconstructRegion(this, context); + } + + public int getDataNodeId() { + return dataNodeId; + } + + public List getRegionIds() { + return regionIds; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/RemoveRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/RemoveRegion.java new file mode 100644 index 000000000000..8c14433167a8 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/RemoveRegion.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; + +import com.google.common.collect.ImmutableList; + +import java.util.List; +import java.util.Objects; + +public class RemoveRegion extends Statement { + + private final int regionId; + private final int dataNodeId; + + public RemoveRegion(int regionId, int dataNodeId) { + super(null); + this.regionId = regionId; + this.dataNodeId = dataNodeId; + } + + @Override + public List getChildren() { + return ImmutableList.of(); + } + + @Override + public int hashCode() { + return Objects.hash(RemoveRegion.class, regionId, dataNodeId); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof RemoveRegion)) { + return false; + } + RemoveRegion another = (RemoveRegion) obj; + return regionId == another.regionId && dataNodeId == another.dataNodeId; + } + + @Override + public String toString() { + return String.format("remove region %d from %d", regionId, dataNodeId); + } + + @Override + public R accept(AstVisitor visitor, C context) { + return visitor.visitRemoveRegion(this, context); + } + + public int getRegionId() { + return regionId; + } + + public int getDataNodeId() { + return dataNodeId; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SetSqlDialect.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SetSqlDialect.java new file mode 100644 index 000000000000..5f4092d5c31b --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SetSqlDialect.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; + +import org.apache.iotdb.db.protocol.session.IClientSession; + +import com.google.common.collect.ImmutableList; + +import javax.annotation.Nullable; + +import java.util.List; +import java.util.Objects; + +public class SetSqlDialect extends Statement { + private final IClientSession.SqlDialect sqlDialect; + + public SetSqlDialect(IClientSession.SqlDialect sqlDialect, @Nullable NodeLocation location) { + super(location); + this.sqlDialect = sqlDialect; + } + + public IClientSession.SqlDialect getSqlDialect() { + return sqlDialect; + } + + @Override + public List getChildren() { + return ImmutableList.of(); + } + + @Override + public R accept(AstVisitor visitor, C context) { + return visitor.visitSetSqlDialect(this, context); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + SetSqlDialect other = (SetSqlDialect) o; + return Objects.equals(sqlDialect, other.sqlDialect); + } + + @Override + public int hashCode() { + return Objects.hash(sqlDialect); + } + + @Override + public boolean shallowEquals(Node other) { + return sameClass(this, other); + } + + @Override + public String toString() { + return "SET SQL_DIALECT=" + sqlDialect; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SetSystemStatus.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SetSystemStatus.java new file mode 100644 index 000000000000..c767b67c96f9 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SetSystemStatus.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; + +import org.apache.iotdb.db.queryengine.common.MPPQueryContext; +import org.apache.iotdb.db.queryengine.plan.statement.Statement; + +public class SetSystemStatus extends WrappedStatement { + public SetSystemStatus(Statement innerTreeStatement, MPPQueryContext context) { + super(innerTreeStatement, context); + } + + @Override + public R accept(AstVisitor visitor, C context) { + return visitor.visitSetSystemStatus(this, context); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java index 27465268674c..82ff69b564e1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java @@ -21,6 +21,7 @@ import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType; import org.apache.iotdb.commons.auth.entity.PrivilegeType; +import org.apache.iotdb.commons.cluster.NodeStatus; import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.schema.cache.CacheClearOptions; @@ -82,6 +83,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Explain; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ExplainAnalyze; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ExtendRegion; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Fill; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Flush; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FunctionCall; @@ -105,9 +107,11 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LikePredicate; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Limit; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Literal; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LoadConfiguration; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LoadTsFile; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LogicalExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LongLiteral; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.MigrateRegion; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.NaturalJoin; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Node; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.NodeLocation; @@ -124,9 +128,11 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Query; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.QueryBody; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.QuerySpecification; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ReconstructRegion; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Relation; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.RelationalAuthorStatement; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.RemoveDataNode; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.RemoveRegion; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.RenameColumn; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.RenameTable; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Row; @@ -135,6 +141,8 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SelectItem; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SetConfiguration; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SetProperties; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SetSqlDialect; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SetSystemStatus; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowAINodes; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowCluster; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowClusterId; @@ -187,7 +195,9 @@ import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowsStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.FlushStatement; +import org.apache.iotdb.db.queryengine.plan.statement.sys.LoadConfigurationStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.SetConfigurationStatement; +import org.apache.iotdb.db.queryengine.plan.statement.sys.SetSystemStatusStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.StartRepairDataStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.StopRepairDataStatement; import org.apache.iotdb.db.relational.grammar.sql.RelationalSqlBaseVisitor; @@ -1141,7 +1151,34 @@ public Node visitShowSeriesSlotListStatement( @Override public Node visitMigrateRegionStatement(RelationalSqlParser.MigrateRegionStatementContext ctx) { - return super.visitMigrateRegionStatement(ctx); + return new MigrateRegion( + Integer.parseInt(ctx.regionId.getText()), + Integer.parseInt(ctx.fromId.getText()), + Integer.parseInt(ctx.toId.getText())); + } + + @Override + public Node visitReconstructRegionStatement( + RelationalSqlParser.ReconstructRegionStatementContext ctx) { + int dataNodeId = Integer.parseInt(ctx.targetDataNodeId.getText()); + List regionIds = + ctx.regionIds.stream() + .map(Token::getText) + .map(Integer::parseInt) + .collect(Collectors.toList()); + return new ReconstructRegion(dataNodeId, regionIds); + } + + @Override + public Node visitExtendRegionStatement(RelationalSqlParser.ExtendRegionStatementContext ctx) { + return new ExtendRegion( + Integer.parseInt(ctx.regionId.getText()), Integer.parseInt(ctx.targetDataNodeId.getText())); + } + + @Override + public Node visitRemoveRegionStatement(RelationalSqlParser.RemoveRegionStatementContext ctx) { + return new RemoveRegion( + Integer.parseInt(ctx.regionId.getText()), Integer.parseInt(ctx.targetDataNodeId.getText())); } @Override @@ -1199,7 +1236,17 @@ public Node visitClearCacheStatement(final RelationalSqlParser.ClearCacheStateme @Override public Node visitSetSystemStatusStatement( RelationalSqlParser.SetSystemStatusStatementContext ctx) { - return super.visitSetSystemStatusStatement(ctx); + SetSystemStatusStatement setSystemStatusStatement = new SetSystemStatusStatement(); + setSystemStatusStatement.setOnCluster( + ctx.localOrClusterMode() == null || ctx.localOrClusterMode().LOCAL() == null); + if (ctx.RUNNING() != null) { + setSystemStatusStatement.setStatus(NodeStatus.Running); + } else if (ctx.READONLY() != null) { + setSystemStatusStatement.setStatus(NodeStatus.ReadOnly); + } else { + throw new SemanticException("Unknown system status in set system command."); + } + return new SetSystemStatus(setSystemStatusStatement, null); } @Override @@ -1213,6 +1260,13 @@ public Node visitShowCurrentSqlDialectStatement( return new ShowCurrentSqlDialect(getLocation(ctx)); } + @Override + public Node visitSetSqlDialectStatement(RelationalSqlParser.SetSqlDialectStatementContext ctx) { + return new SetSqlDialect( + ctx.TABLE() == null ? IClientSession.SqlDialect.TREE : IClientSession.SqlDialect.TABLE, + getLocation(ctx)); + } + @Override public Node visitShowCurrentDatabaseStatement( RelationalSqlParser.ShowCurrentDatabaseStatementContext ctx) { @@ -1277,7 +1331,11 @@ public Node visitKillQueryStatement(RelationalSqlParser.KillQueryStatementContex @Override public Node visitLoadConfigurationStatement( RelationalSqlParser.LoadConfigurationStatementContext ctx) { - return super.visitLoadConfigurationStatement(ctx); + LoadConfigurationStatement loadConfigurationStatement = + new LoadConfigurationStatement(StatementType.LOAD_CONFIGURATION); + loadConfigurationStatement.setOnCluster( + ctx.localOrClusterMode() == null || ctx.localOrClusterMode().LOCAL() == null); + return new LoadConfiguration(loadConfigurationStatement, null); } @Override @@ -1312,7 +1370,8 @@ public Node visitStartRepairDataStatement( RelationalSqlParser.StartRepairDataStatementContext ctx) { StartRepairDataStatement startRepairDataStatement = new StartRepairDataStatement(StatementType.START_REPAIR_DATA); - startRepairDataStatement.setOnCluster(ctx.localOrClusterMode().LOCAL() == null); + startRepairDataStatement.setOnCluster( + ctx.localOrClusterMode() == null || ctx.localOrClusterMode().LOCAL() == null); return new StartRepairData(startRepairDataStatement, null); } @@ -1320,7 +1379,8 @@ public Node visitStartRepairDataStatement( public Node visitStopRepairDataStatement(RelationalSqlParser.StopRepairDataStatementContext ctx) { StopRepairDataStatement stopRepairDataStatement = new StopRepairDataStatement(StatementType.STOP_REPAIR_DATA); - stopRepairDataStatement.setOnCluster(ctx.localOrClusterMode().LOCAL() == null); + stopRepairDataStatement.setOnCluster( + ctx.localOrClusterMode() == null || ctx.localOrClusterMode().LOCAL() == null); return new StopRepairData(stopRepairDataStatement, null); } @@ -1425,12 +1485,22 @@ public Node visitListRolePrivilegeStatement( @Override public Node visitListUserStatement(RelationalSqlParser.ListUserStatementContext ctx) { - return new RelationalAuthorStatement(AuthorRType.LIST_USER); + RelationalAuthorStatement stmt = new RelationalAuthorStatement(AuthorRType.LIST_USER); + if (ctx.OF() != null) { + String roleName = ctx.roleName.getText(); + stmt.setRoleName(roleName); + } + return stmt; } @Override public Node visitListRoleStatement(RelationalSqlParser.ListRoleStatementContext ctx) { - return new RelationalAuthorStatement(AuthorRType.LIST_ROLE); + RelationalAuthorStatement stmt = new RelationalAuthorStatement(AuthorRType.LIST_ROLE); + if (ctx.OF() != null) { + String userName = ctx.userName.getText(); + stmt.setUserName(userName); + } + return stmt; } private Set parseSystemPrivilege(RelationalSqlParser.SystemPrivilegesContext ctx) { @@ -1444,10 +1514,18 @@ private Set parseSystemPrivilege(RelationalSqlParser.SystemPrivil private Set parseRelationalPrivilege( RelationalSqlParser.ObjectPrivilegesContext ctx) { - List privilegeContexts = ctx.objectPrivilege(); Set privileges = new HashSet<>(); - for (RelationalSqlParser.ObjectPrivilegeContext privilege : privilegeContexts) { - privileges.add(PrivilegeType.valueOf(privilege.getText().toUpperCase())); + if (ctx.ALL() != null) { + for (PrivilegeType privilegeType : PrivilegeType.values()) { + if (privilegeType.isRelationalPrivilege()) { + privileges.add(privilegeType); + } + } + } else { + List privilegeContexts = ctx.objectPrivilege(); + for (RelationalSqlParser.ObjectPrivilegeContext privilege : privilegeContexts) { + privileges.add(PrivilegeType.valueOf(privilege.getText().toUpperCase())); + } } return privileges; } @@ -1533,7 +1611,7 @@ public Node visitRevokeStatement(RelationalSqlParser.RevokeStatementContext ctx) fromUser = ctx.holderType().getText().equalsIgnoreCase("user"); name = ctx.holderName.getText(); boolean grantOption = ctx.revokeGrantOpt() != null; - boolean fromTable; + boolean fromTable = false; Set privileges = new HashSet<>(); // SYSTEM PRIVILEGES OR ALL PRIVILEGES @@ -1555,38 +1633,48 @@ public Node visitRevokeStatement(RelationalSqlParser.RevokeStatementContext ctx) } } else { privileges = parseRelationalPrivilege(ctx.privilegeObjectScope().objectPrivileges()); + boolean revokeAll = ctx.privilegeObjectScope().objectPrivileges().ALL() != null; + String databaseName = ""; + String tableName = ""; + // ON TABLE / DB if (ctx.privilegeObjectScope().objectType() != null) { fromTable = ctx.privilegeObjectScope().objectType().getText().equalsIgnoreCase("table"); - String databaseName = ""; if (fromTable) { databaseName = clientSession.getDatabaseName(); if (databaseName == null) { throw new SemanticException("Database is not set yet."); } + tableName = ctx.privilegeObjectScope().objectName.getText().toLowerCase(); + } else { + databaseName = ctx.privilegeObjectScope().objectName.getText().toLowerCase(); } - String obj = ctx.privilegeObjectScope().objectName.getText(); - return new RelationalAuthorStatement( - fromUser - ? fromTable ? AuthorRType.REVOKE_USER_TB : AuthorRType.REVOKE_USER_DB - : fromTable ? AuthorRType.REVOKE_ROLE_TB : AuthorRType.REVOKE_ROLE_DB, - fromUser ? name : "", - fromUser ? "" : name, - fromTable ? databaseName.toLowerCase() : obj.toLowerCase(), - fromTable ? obj.toLowerCase() : "", - privileges, - grantOption, - null); } else if (ctx.privilegeObjectScope().objectScope() != null) { - String db = ctx.privilegeObjectScope().objectScope().dbname.getText().toLowerCase(); - String tb = ctx.privilegeObjectScope().objectScope().tbname.getText().toLowerCase(); + fromTable = true; + databaseName = ctx.privilegeObjectScope().objectScope().dbname.getText().toLowerCase(); + tableName = ctx.privilegeObjectScope().objectScope().tbname.getText().toLowerCase(); + } + + // The REVOKE ALL command can revoke privileges for users, databases, and tables. + // When AuthorRType is REVOKE_USER_ALL: + // If both database and table are empty, it clears all privileges globally. + // If a database is specified (non-empty), it revokes all privileges within that database + // scope. + // If a table is specified (non-empty), it revokes privileges specifically for that table. + // For operations involving the ANY scope, REVOKE_USER_ALL cannot be combined with + // database/table + // specifications. However, since ALL privileges are resolved as concrete privileges in this + // context, + // equivalent effects can be achieved by supplementing with REVOKE_USER_ANY operations. + + if (revokeAll && ctx.privilegeObjectScope().ANY() == null) { return new RelationalAuthorStatement( - fromUser ? AuthorRType.REVOKE_USER_TB : AuthorRType.REVOKE_ROLE_TB, + fromUser ? AuthorRType.REVOKE_USER_ALL : AuthorRType.REVOKE_ROLE_ALL, fromUser ? name : "", fromUser ? "" : name, - db, - tb, - privileges, + databaseName, + tableName, + Collections.emptySet(), grantOption, null); } else if (ctx.privilegeObjectScope().ANY() != null) { @@ -1596,10 +1684,20 @@ public Node visitRevokeStatement(RelationalSqlParser.RevokeStatementContext ctx) fromUser ? name : "", fromUser ? "" : name, grantOption); + } else { + return new RelationalAuthorStatement( + fromUser + ? fromTable ? AuthorRType.REVOKE_USER_TB : AuthorRType.REVOKE_USER_DB + : fromTable ? AuthorRType.REVOKE_ROLE_TB : AuthorRType.REVOKE_ROLE_DB, + fromUser ? name : "", + fromUser ? "" : name, + databaseName, + tableName, + privileges, + grantOption, + null); } } - // will not get here. - throw new SemanticException("author statement parser error"); } // ********************** query expressions ******************** diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileScheduler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileScheduler.java index d42ee3616d55..5a76988163fd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileScheduler.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileScheduler.java @@ -94,6 +94,7 @@ import java.util.HashSet; import java.util.Iterator; import java.util.List; +import java.util.ListIterator; import java.util.Map; import java.util.Objects; import java.util.Set; @@ -215,8 +216,7 @@ public void start() { long startTime = System.nanoTime(); final boolean isFirstPhaseSuccess; try { - isFirstPhaseSuccess = - firstPhaseWithRetry(node, CONFIG.getLoadTsFileRetryCountOnRegionChange()); + isFirstPhaseSuccess = firstPhase(node); } finally { LOAD_TSFILE_COST_METRICS_SET.recordPhaseTimeCost( LoadTsFileCostMetricsSet.FIRST_PHASE, System.nanoTime() - startTime); @@ -270,9 +270,23 @@ public void start() { if (isLoadSuccess) { stateMachine.transitionToFinished(); } else { + final StringBuilder failedTsFiles = + new StringBuilder( + !tsFileNodeList.isEmpty() + ? tsFileNodeList.get(0).getTsFileResource().getTsFilePath() + : ""); + final ListIterator iterator = failedTsFileNodeIndexes.listIterator(1); + while (iterator.hasNext()) { + failedTsFiles + .append(", ") + .append(tsFileNodeList.get(iterator.next()).getTsFileResource().getTsFilePath()); + } final long startTime = System.nanoTime(); try { // if failed to load some TsFiles, then try to convert the TsFiles to Tablets + LOGGER.info( + "Load TsFile(s) failed, will try to convert to tablets and insert. Failed TsFiles: {}", + failedTsFiles); convertFailedTsFilesToTabletsAndRetry(); } finally { LOAD_TSFILE_COST_METRICS_SET.recordPhaseTimeCost( @@ -284,30 +298,7 @@ public void start() { } } - private boolean firstPhaseWithRetry(LoadSingleTsFileNode node, int retryCountOnRegionChange) { - retryCountOnRegionChange = Math.max(0, retryCountOnRegionChange); - while (true) { - try { - return firstPhase(node); - } catch (RegionReplicaSetChangedException e) { - if (retryCountOnRegionChange > 0) { - LOGGER.warn( - "Region replica set changed during loading TsFile {}, maybe due to region migration, will retry for {} times.", - node.getTsFileResource(), - retryCountOnRegionChange); - retryCountOnRegionChange--; - } else { - stateMachine.transitionToFailed(e); - LOGGER.warn( - "Region replica set changed during loading TsFile {} after retry.", - node.getTsFileResource()); - return false; - } - } - } - } - - private boolean firstPhase(LoadSingleTsFileNode node) throws RegionReplicaSetChangedException { + private boolean firstPhase(LoadSingleTsFileNode node) { final TsFileDataManager tsFileDataManager = new TsFileDataManager(this, node, block); try { new TsFileSplitter( @@ -317,8 +308,6 @@ private boolean firstPhase(LoadSingleTsFileNode node) throws RegionReplicaSetCha stateMachine.transitionToFailed(new TSStatus(TSStatusCode.LOAD_FILE_ERROR.getStatusCode())); return false; } - } catch (RegionReplicaSetChangedException e) { - throw e; } catch (IllegalStateException e) { stateMachine.transitionToFailed(e); LOGGER.warn( @@ -698,7 +687,7 @@ private boolean addOrSendChunkData(ChunkData chunkData) throws LoadFileException dataSize -= pieceNode.getDataSize(); block.reduceMemoryUsage(pieceNode.getDataSize()); - regionId2ReplicaSetAndNode.put( + regionId2ReplicaSetAndNode.replace( sortedRegionId, new Pair<>( replicaSet, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java index 333d21aeccd4..49d8f9f97c2f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java @@ -122,7 +122,10 @@ import org.apache.iotdb.db.queryengine.plan.statement.sys.LoadConfigurationStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.MergeStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.SetConfigurationStatement; +import org.apache.iotdb.db.queryengine.plan.statement.sys.SetSqlDialectStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.SetSystemStatusStatement; +import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowCurrentSqlDialectStatement; +import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowCurrentUserStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowQueriesStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowVersionStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.StartRepairDataStatement; @@ -659,4 +662,17 @@ public R visitShowCurrentTimestamp( ShowCurrentTimestampStatement showCurrentTimestampStatement, C context) { return visitStatement(showCurrentTimestampStatement, context); } + + public R visitSetSqlDialect(SetSqlDialectStatement setSqlDialectStatement, C context) { + return visitStatement(setSqlDialectStatement, context); + } + + public R visitShowCurrentSqlDialect( + ShowCurrentSqlDialectStatement showCurrentSqlDialectStatement, C context) { + return visitStatement(showCurrentSqlDialectStatement, context); + } + + public R visitShowCurrentUser(ShowCurrentUserStatement showCurrentUserStatement, C context) { + return visitStatement(showCurrentUserStatement, context); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/sys/SetSqlDialectStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/sys/SetSqlDialectStatement.java new file mode 100644 index 000000000000..edb63ef4893b --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/sys/SetSqlDialectStatement.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.statement.sys; + +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.db.protocol.session.IClientSession; +import org.apache.iotdb.db.queryengine.plan.analyze.QueryType; +import org.apache.iotdb.db.queryengine.plan.statement.IConfigStatement; +import org.apache.iotdb.db.queryengine.plan.statement.Statement; +import org.apache.iotdb.db.queryengine.plan.statement.StatementVisitor; + +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +import static org.apache.iotdb.db.auth.AuthorityChecker.SUCCEED; + +public class SetSqlDialectStatement extends Statement implements IConfigStatement { + private final IClientSession.SqlDialect sqlDialect; + + public SetSqlDialectStatement(IClientSession.SqlDialect sqlDialect) { + this.sqlDialect = sqlDialect; + } + + public IClientSession.SqlDialect getSqlDialect() { + return sqlDialect; + } + + @Override + public int hashCode() { + return Objects.hash(sqlDialect); + } + + @Override + public String toString() { + return "SET SQL_DIALECT=" + sqlDialect; + } + + @Override + public List getPaths() { + return Collections.emptyList(); + } + + @Override + public QueryType getQueryType() { + return QueryType.WRITE; + } + + @Override + public R accept(StatementVisitor visitor, C context) { + return visitor.visitSetSqlDialect(this, context); + } + + @Override + public TSStatus checkPermissionBeforeProcess(final String userName) { + return SUCCEED; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/sys/ShowCurrentSqlDialectStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/sys/ShowCurrentSqlDialectStatement.java new file mode 100644 index 000000000000..73a40db282f8 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/sys/ShowCurrentSqlDialectStatement.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.statement.sys; + +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.db.queryengine.plan.analyze.QueryType; +import org.apache.iotdb.db.queryengine.plan.statement.IConfigStatement; +import org.apache.iotdb.db.queryengine.plan.statement.StatementVisitor; +import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowStatement; + +import static org.apache.iotdb.db.auth.AuthorityChecker.SUCCEED; + +public class ShowCurrentSqlDialectStatement extends ShowStatement implements IConfigStatement { + @Override + public R accept(StatementVisitor visitor, C context) { + return visitor.visitShowCurrentSqlDialect(this, context); + } + + @Override + public TSStatus checkPermissionBeforeProcess(String userName) { + return SUCCEED; + } + + @Override + public QueryType getQueryType() { + return QueryType.READ; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/sys/ShowCurrentUserStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/sys/ShowCurrentUserStatement.java new file mode 100644 index 000000000000..81650ef21aed --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/sys/ShowCurrentUserStatement.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.statement.sys; + +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.db.queryengine.plan.analyze.QueryType; +import org.apache.iotdb.db.queryengine.plan.statement.IConfigStatement; +import org.apache.iotdb.db.queryengine.plan.statement.StatementVisitor; +import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowStatement; + +import static org.apache.iotdb.db.auth.AuthorityChecker.SUCCEED; + +public class ShowCurrentUserStatement extends ShowStatement implements IConfigStatement { + @Override + public R accept(StatementVisitor visitor, C context) { + return visitor.visitShowCurrentUser(this, context); + } + + @Override + public TSStatus checkPermissionBeforeProcess(String userName) { + return SUCCEED; + } + + @Override + public QueryType getQueryType() { + return QueryType.READ; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/AbstractGreatestLeastColumnTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/AbstractGreatestLeastColumnTransformer.java new file mode 100644 index 000000000000..f92c5fa2811e --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/AbstractGreatestLeastColumnTransformer.java @@ -0,0 +1,109 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.db.queryengine.transformation.dag.column.multi; + +import org.apache.iotdb.db.queryengine.transformation.dag.column.ColumnTransformer; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.read.common.type.Type; +import org.apache.tsfile.read.common.type.TypeEnum; + +import java.util.List; + +public abstract class AbstractGreatestLeastColumnTransformer extends MultiColumnTransformer { + + protected AbstractGreatestLeastColumnTransformer( + Type returnType, List columnTransformerList) { + super(returnType, columnTransformerList); + } + + @Override + protected void doTransform( + List childrenColumns, ColumnBuilder builder, int positionCount) { + for (int i = 0; i < positionCount; i++) { + transform(builder, childrenColumns, i); + } + } + + @Override + protected void doTransform( + List childrenColumns, ColumnBuilder builder, int positionCount, boolean[] selection) { + for (int i = 0; i < positionCount; i++) { + if (selection[i]) { + transform(builder, childrenColumns, i); + } else { + builder.appendNull(); + } + } + } + + protected abstract void transform(ColumnBuilder builder, List childrenColumns, int index); + + public static ColumnTransformer getGreatestColumnTransformer( + Type type, List columnTransformers) { + TypeEnum typeEnum = type.getTypeEnum(); + switch (typeEnum) { + case BOOLEAN: + return new BooleanGreatestColumnTransformer(type, columnTransformers); + case INT32: + case DATE: + return new Int32GreatestColumnTransformer(type, columnTransformers); + case INT64: + case TIMESTAMP: + return new Int64GreatestColumnTransformer(type, columnTransformers); + case FLOAT: + return new FloatGreatestColumnTransformer(type, columnTransformers); + case DOUBLE: + return new DoubleGreatestColumnTransformer(type, columnTransformers); + case STRING: + case TEXT: + return new BinaryGreatestColumnTransformer(type, columnTransformers); + default: + throw new UnsupportedOperationException( + String.format("Unsupported data type: %s", typeEnum)); + } + } + + public static ColumnTransformer getLeastColumnTransformer( + Type type, List columnTransformers) { + TypeEnum typeEnum = type.getTypeEnum(); + switch (typeEnum) { + case BOOLEAN: + return new BooleanLeastColumnTransformer(type, columnTransformers); + case INT32: + case DATE: + return new Int32LeastColumnTransformer(type, columnTransformers); + case INT64: + case TIMESTAMP: + return new Int64LeastColumnTransformer(type, columnTransformers); + case FLOAT: + return new FloatLeastColumnTransformer(type, columnTransformers); + case DOUBLE: + return new DoubleLeastColumnTransformer(type, columnTransformers); + case STRING: + case TEXT: + return new BinaryLeastColumnTransformer(type, columnTransformers); + default: + throw new UnsupportedOperationException( + String.format("Unsupported data type: %s", typeEnum)); + } + } + + @Override + protected void checkType() { + // do nothing + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/BinaryGreatestColumnTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/BinaryGreatestColumnTransformer.java new file mode 100644 index 000000000000..e81487bbac3e --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/BinaryGreatestColumnTransformer.java @@ -0,0 +1,53 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.db.queryengine.transformation.dag.column.multi; + +import org.apache.iotdb.db.queryengine.transformation.dag.column.ColumnTransformer; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.read.common.type.Type; +import org.apache.tsfile.utils.Binary; + +import java.util.List; + +public class BinaryGreatestColumnTransformer extends AbstractGreatestLeastColumnTransformer { + protected BinaryGreatestColumnTransformer( + Type returnType, List columnTransformerList) { + super(returnType, columnTransformerList); + } + + @Override + protected void transform(ColumnBuilder builder, List childrenColumns, int index) { + Column firstColumn = childrenColumns.get(0); + Binary maxValue = firstColumn.getBinary(index); + boolean allNull = firstColumn.isNull(index); + for (int i = 1; i < childrenColumns.size(); i++) { + Column column = childrenColumns.get(i); + if (!column.isNull(index)) { + Binary value = column.getBinary(index); + if (allNull || value.compareTo(maxValue) > 0) { + allNull = false; + maxValue = value; + } + } + } + if (allNull) { + builder.appendNull(); + } else { + returnType.writeBinary(builder, maxValue); + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/BinaryLeastColumnTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/BinaryLeastColumnTransformer.java new file mode 100644 index 000000000000..6cde29f4c845 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/BinaryLeastColumnTransformer.java @@ -0,0 +1,53 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.db.queryengine.transformation.dag.column.multi; + +import org.apache.iotdb.db.queryengine.transformation.dag.column.ColumnTransformer; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.read.common.type.Type; +import org.apache.tsfile.utils.Binary; + +import java.util.List; + +public class BinaryLeastColumnTransformer extends AbstractGreatestLeastColumnTransformer { + protected BinaryLeastColumnTransformer( + Type returnType, List columnTransformerList) { + super(returnType, columnTransformerList); + } + + @Override + protected void transform(ColumnBuilder builder, List childrenColumns, int index) { + Column firstColumn = childrenColumns.get(0); + Binary minValue = firstColumn.getBinary(index); + boolean allNull = firstColumn.isNull(index); + for (int i = 1; i < childrenColumns.size(); i++) { + Column column = childrenColumns.get(i); + if (!column.isNull(index)) { + Binary value = column.getBinary(index); + if (allNull || value.compareTo(minValue) < 0) { + allNull = false; + minValue = value; + } + } + } + if (allNull) { + builder.appendNull(); + } else { + returnType.writeBinary(builder, minValue); + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/BooleanGreatestColumnTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/BooleanGreatestColumnTransformer.java new file mode 100644 index 000000000000..2bc56e6f5bc9 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/BooleanGreatestColumnTransformer.java @@ -0,0 +1,52 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.db.queryengine.transformation.dag.column.multi; + +import org.apache.iotdb.db.queryengine.transformation.dag.column.ColumnTransformer; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.read.common.type.Type; + +import java.util.List; + +public class BooleanGreatestColumnTransformer extends AbstractGreatestLeastColumnTransformer { + protected BooleanGreatestColumnTransformer( + Type returnType, List columnTransformerList) { + super(returnType, columnTransformerList); + } + + @Override + protected void transform(ColumnBuilder builder, List childrenColumns, int index) { + boolean allNull = true; + for (Column column : childrenColumns) { + if (!column.isNull(index)) { + allNull = false; + if (column.getBoolean(index)) { + // find max value true + returnType.writeBoolean(builder, true); + break; + } + } + } + if (allNull) { + // all value are null return null + builder.appendNull(); + } else { + // do not have true, but have false + returnType.writeBoolean(builder, false); + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/BooleanLeastColumnTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/BooleanLeastColumnTransformer.java new file mode 100644 index 000000000000..4ea2b717092c --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/BooleanLeastColumnTransformer.java @@ -0,0 +1,49 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.db.queryengine.transformation.dag.column.multi; + +import org.apache.iotdb.db.queryengine.transformation.dag.column.ColumnTransformer; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.read.common.type.Type; + +import java.util.List; + +public class BooleanLeastColumnTransformer extends AbstractGreatestLeastColumnTransformer { + protected BooleanLeastColumnTransformer( + Type returnType, List columnTransformerList) { + super(returnType, columnTransformerList); + } + + @Override + protected void transform(ColumnBuilder builder, List childrenColumns, int index) { + boolean allNull = true; + for (Column column : childrenColumns) { + if (!column.isNull(index)) { + allNull = false; + if (!column.getBoolean(index)) { + returnType.writeBoolean(builder, false); + break; + } + } + } + if (allNull) { + builder.appendNull(); + } else { + returnType.writeBoolean(builder, true); + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/DoubleGreatestColumnTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/DoubleGreatestColumnTransformer.java new file mode 100644 index 000000000000..e09f0b6df4b2 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/DoubleGreatestColumnTransformer.java @@ -0,0 +1,52 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.db.queryengine.transformation.dag.column.multi; + +import org.apache.iotdb.db.queryengine.transformation.dag.column.ColumnTransformer; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.read.common.type.Type; + +import java.util.List; + +public class DoubleGreatestColumnTransformer extends AbstractGreatestLeastColumnTransformer { + protected DoubleGreatestColumnTransformer( + Type returnType, List columnTransformerList) { + super(returnType, columnTransformerList); + } + + @Override + protected void transform(ColumnBuilder builder, List childrenColumns, int index) { + Column firstColumn = childrenColumns.get(0); + double maxValue = firstColumn.getDouble(index); + boolean allNull = firstColumn.isNull(index); + for (int i = 1; i < childrenColumns.size(); i++) { + Column column = childrenColumns.get(i); + if (!column.isNull(index)) { + double value = column.getDouble(index); + if (allNull || value > maxValue) { + allNull = false; + maxValue = value; + } + } + } + if (allNull) { + builder.appendNull(); + } else { + returnType.writeDouble(builder, maxValue); + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/DoubleLeastColumnTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/DoubleLeastColumnTransformer.java new file mode 100644 index 000000000000..2dc19ec75e7a --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/DoubleLeastColumnTransformer.java @@ -0,0 +1,52 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.db.queryengine.transformation.dag.column.multi; + +import org.apache.iotdb.db.queryengine.transformation.dag.column.ColumnTransformer; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.read.common.type.Type; + +import java.util.List; + +public class DoubleLeastColumnTransformer extends AbstractGreatestLeastColumnTransformer { + protected DoubleLeastColumnTransformer( + Type returnType, List columnTransformerList) { + super(returnType, columnTransformerList); + } + + @Override + protected void transform(ColumnBuilder builder, List childrenColumns, int index) { + Column firstColumn = childrenColumns.get(0); + double minValue = firstColumn.getDouble(index); + boolean allNull = firstColumn.isNull(index); + for (int i = 1; i < childrenColumns.size(); i++) { + Column column = childrenColumns.get(i); + if (!column.isNull(index)) { + double value = column.getDouble(index); + if (allNull || value < minValue) { + allNull = false; + minValue = value; + } + } + } + if (allNull) { + builder.appendNull(); + } else { + returnType.writeDouble(builder, minValue); + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/FloatGreatestColumnTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/FloatGreatestColumnTransformer.java new file mode 100644 index 000000000000..bc5bfc982f8c --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/FloatGreatestColumnTransformer.java @@ -0,0 +1,52 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.db.queryengine.transformation.dag.column.multi; + +import org.apache.iotdb.db.queryengine.transformation.dag.column.ColumnTransformer; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.read.common.type.Type; + +import java.util.List; + +public class FloatGreatestColumnTransformer extends AbstractGreatestLeastColumnTransformer { + protected FloatGreatestColumnTransformer( + Type returnType, List columnTransformerList) { + super(returnType, columnTransformerList); + } + + @Override + protected void transform(ColumnBuilder builder, List childrenColumns, int index) { + Column firstColumn = childrenColumns.get(0); + float maxValue = firstColumn.getFloat(index); + boolean allNull = firstColumn.isNull(index); + for (int i = 1; i < childrenColumns.size(); i++) { + Column column = childrenColumns.get(i); + if (!column.isNull(index)) { + float value = column.getFloat(index); + if (allNull || value > maxValue) { + allNull = false; + maxValue = value; + } + } + } + if (allNull) { + builder.appendNull(); + } else { + returnType.writeFloat(builder, maxValue); + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/FloatLeastColumnTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/FloatLeastColumnTransformer.java new file mode 100644 index 000000000000..223f63673390 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/FloatLeastColumnTransformer.java @@ -0,0 +1,52 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.db.queryengine.transformation.dag.column.multi; + +import org.apache.iotdb.db.queryengine.transformation.dag.column.ColumnTransformer; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.read.common.type.Type; + +import java.util.List; + +public class FloatLeastColumnTransformer extends AbstractGreatestLeastColumnTransformer { + protected FloatLeastColumnTransformer( + Type returnType, List columnTransformerList) { + super(returnType, columnTransformerList); + } + + @Override + protected void transform(ColumnBuilder builder, List childrenColumns, int index) { + Column firstColumn = childrenColumns.get(0); + float minValue = firstColumn.getFloat(index); + boolean allNull = firstColumn.isNull(index); + for (int i = 1; i < childrenColumns.size(); i++) { + Column column = childrenColumns.get(i); + if (!column.isNull(index)) { + float value = column.getFloat(index); + if (allNull || value < minValue) { + allNull = false; + minValue = value; + } + } + } + if (allNull) { + builder.appendNull(); + } else { + returnType.writeFloat(builder, minValue); + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/Int32GreatestColumnTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/Int32GreatestColumnTransformer.java new file mode 100644 index 000000000000..551970f470a6 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/Int32GreatestColumnTransformer.java @@ -0,0 +1,52 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.db.queryengine.transformation.dag.column.multi; + +import org.apache.iotdb.db.queryengine.transformation.dag.column.ColumnTransformer; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.read.common.type.Type; + +import java.util.List; + +public class Int32GreatestColumnTransformer extends AbstractGreatestLeastColumnTransformer { + protected Int32GreatestColumnTransformer( + Type returnType, List columnTransformerList) { + super(returnType, columnTransformerList); + } + + @Override + protected void transform(ColumnBuilder builder, List childrenColumns, int index) { + Column firstColumn = childrenColumns.get(0); + int maxValue = firstColumn.getInt(index); + boolean allNull = firstColumn.isNull(index); + for (int i = 1; i < childrenColumns.size(); i++) { + Column column = childrenColumns.get(i); + if (!column.isNull(index)) { + int value = column.getInt(index); + if (allNull || value > maxValue) { + allNull = false; + maxValue = value; + } + } + } + if (allNull) { + builder.appendNull(); + } else { + returnType.writeInt(builder, maxValue); + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/Int32LeastColumnTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/Int32LeastColumnTransformer.java new file mode 100644 index 000000000000..26b64349fb4f --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/Int32LeastColumnTransformer.java @@ -0,0 +1,52 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.db.queryengine.transformation.dag.column.multi; + +import org.apache.iotdb.db.queryengine.transformation.dag.column.ColumnTransformer; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.read.common.type.Type; + +import java.util.List; + +public class Int32LeastColumnTransformer extends AbstractGreatestLeastColumnTransformer { + protected Int32LeastColumnTransformer( + Type returnType, List columnTransformerList) { + super(returnType, columnTransformerList); + } + + @Override + protected void transform(ColumnBuilder builder, List childrenColumns, int index) { + Column firstColumn = childrenColumns.get(0); + int minValue = firstColumn.getInt(index); + boolean allNull = firstColumn.isNull(index); + for (int i = 1; i < childrenColumns.size(); i++) { + Column column = childrenColumns.get(i); + if (!column.isNull(index)) { + int value = column.getInt(index); + if (allNull || value < minValue) { + allNull = false; + minValue = value; + } + } + } + if (allNull) { + builder.appendNull(); + } else { + returnType.writeInt(builder, minValue); + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/Int64GreatestColumnTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/Int64GreatestColumnTransformer.java new file mode 100644 index 000000000000..7818d217d0aa --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/Int64GreatestColumnTransformer.java @@ -0,0 +1,52 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.db.queryengine.transformation.dag.column.multi; + +import org.apache.iotdb.db.queryengine.transformation.dag.column.ColumnTransformer; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.read.common.type.Type; + +import java.util.List; + +public class Int64GreatestColumnTransformer extends AbstractGreatestLeastColumnTransformer { + protected Int64GreatestColumnTransformer( + Type returnType, List columnTransformerList) { + super(returnType, columnTransformerList); + } + + @Override + protected void transform(ColumnBuilder builder, List childrenColumns, int index) { + Column firstColumn = childrenColumns.get(0); + long maxValue = firstColumn.getLong(index); + boolean allNull = firstColumn.isNull(index); + for (int i = 1; i < childrenColumns.size(); i++) { + Column column = childrenColumns.get(i); + if (!column.isNull(index)) { + long value = column.getLong(index); + if (allNull || value > maxValue) { + allNull = false; + maxValue = value; + } + } + } + if (allNull) { + builder.appendNull(); + } else { + returnType.writeLong(builder, maxValue); + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/Int64LeastColumnTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/Int64LeastColumnTransformer.java new file mode 100644 index 000000000000..0efaccb37869 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/Int64LeastColumnTransformer.java @@ -0,0 +1,52 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.db.queryengine.transformation.dag.column.multi; + +import org.apache.iotdb.db.queryengine.transformation.dag.column.ColumnTransformer; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.read.common.type.Type; + +import java.util.List; + +public class Int64LeastColumnTransformer extends AbstractGreatestLeastColumnTransformer { + protected Int64LeastColumnTransformer( + Type returnType, List columnTransformerList) { + super(returnType, columnTransformerList); + } + + @Override + protected void transform(ColumnBuilder builder, List childrenColumns, int index) { + Column firstColumn = childrenColumns.get(0); + long minValue = firstColumn.getLong(index); + boolean allNull = firstColumn.isNull(index); + for (int i = 1; i < childrenColumns.size(); i++) { + Column column = childrenColumns.get(i); + if (!column.isNull(index)) { + long value = column.getLong(index); + if (allNull || value < minValue) { + allNull = false; + minValue = value; + } + } + } + if (allNull) { + builder.appendNull(); + } else { + returnType.writeLong(builder, minValue); + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java index dbd32b47276f..12393fb2a209 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java @@ -37,12 +37,16 @@ import org.apache.iotdb.db.utils.datastructure.TVList; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; +import org.apache.tsfile.file.metadata.AbstractAlignedTimeSeriesMetadata; import org.apache.tsfile.file.metadata.AlignedChunkMetadata; import org.apache.tsfile.file.metadata.AlignedTimeSeriesMetadata; import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.ITimeSeriesMetadata; +import org.apache.tsfile.file.metadata.TableDeviceChunkMetadata; +import org.apache.tsfile.file.metadata.TableDeviceTimeSeriesMetadata; import org.apache.tsfile.file.metadata.TimeseriesMetadata; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; @@ -108,7 +112,7 @@ public AlignedResourceByPathUtils(IFullPath fullPath) { * have chunkMetadata, but query will use these, so we need to generate it for them. */ @Override - public AlignedTimeSeriesMetadata generateTimeSeriesMetadata( + public AbstractAlignedTimeSeriesMetadata generateTimeSeriesMetadata( List readOnlyMemChunk, List chunkMetadataList) { TimeseriesMetadata timeTimeSeriesMetadata = new TimeseriesMetadata(); timeTimeSeriesMetadata.setDataSizeOfChunkMetaDataList(-1); @@ -131,8 +135,11 @@ public AlignedTimeSeriesMetadata generateTimeSeriesMetadata( boolean[] exist = new boolean[alignedFullPath.getSchemaList().size()]; boolean modified = false; + boolean isTable = false; for (IChunkMetadata chunkMetadata : chunkMetadataList) { - AlignedChunkMetadata alignedChunkMetadata = (AlignedChunkMetadata) chunkMetadata; + AbstractAlignedChunkMetadata alignedChunkMetadata = + (AbstractAlignedChunkMetadata) chunkMetadata; + isTable = isTable || (alignedChunkMetadata instanceof TableDeviceChunkMetadata); modified = (modified || alignedChunkMetadata.isModified()); timeStatistics.mergeStatistics(alignedChunkMetadata.getTimeChunkMetadata().getStatistics()); for (int i = 0; i < valueTimeSeriesMetadataList.size(); i++) { @@ -149,8 +156,9 @@ public AlignedTimeSeriesMetadata generateTimeSeriesMetadata( for (ReadOnlyMemChunk memChunk : readOnlyMemChunk) { if (!memChunk.isEmpty()) { - AlignedChunkMetadata alignedChunkMetadata = - (AlignedChunkMetadata) memChunk.getChunkMetaData(); + AbstractAlignedChunkMetadata alignedChunkMetadata = + (AbstractAlignedChunkMetadata) memChunk.getChunkMetaData(); + isTable = isTable || (alignedChunkMetadata instanceof TableDeviceChunkMetadata); timeStatistics.mergeStatistics(alignedChunkMetadata.getTimeChunkMetadata().getStatistics()); for (int i = 0; i < valueTimeSeriesMetadataList.size(); i++) { if (alignedChunkMetadata.getValueChunkMetadataList().get(i) != null) { @@ -174,7 +182,9 @@ public AlignedTimeSeriesMetadata generateTimeSeriesMetadata( } } - return new AlignedTimeSeriesMetadata(timeTimeSeriesMetadata, valueTimeSeriesMetadataList); + return isTable + ? new TableDeviceTimeSeriesMetadata(timeTimeSeriesMetadata, valueTimeSeriesMetadataList) + : new AlignedTimeSeriesMetadata(timeTimeSeriesMetadata, valueTimeSeriesMetadataList); } @Override @@ -268,7 +278,7 @@ public List getVisibleMetadataListFromWriter( QueryContext context, long timeLowerBound) { - List chunkMetadataList = new ArrayList<>(); + List chunkMetadataList = new ArrayList<>(); List timeChunkMetadataList = writer.getVisibleMetadataList( alignedFullPath.getDeviceId(), AlignedFullPath.VECTOR_PLACEHOLDER, TSDataType.VECTOR); @@ -285,7 +295,10 @@ public List getVisibleMetadataListFromWriter( // only need time column if (alignedFullPath.getMeasurementList().isEmpty()) { chunkMetadataList.add( - new AlignedChunkMetadata(timeChunkMetadataList.get(i), Collections.emptyList())); + context.isIgnoreAllNullRows() + ? new AlignedChunkMetadata(timeChunkMetadataList.get(i), Collections.emptyList()) + : new TableDeviceChunkMetadata( + timeChunkMetadataList.get(i), Collections.emptyList())); } else { List valueChunkMetadata = new ArrayList<>(); // if all the sub sensors doesn't exist, it will be false @@ -298,7 +311,9 @@ public List getVisibleMetadataListFromWriter( } if (!context.isIgnoreAllNullRows() || exits) { chunkMetadataList.add( - new AlignedChunkMetadata(timeChunkMetadataList.get(i), valueChunkMetadata)); + context.isIgnoreAllNullRows() + ? new AlignedChunkMetadata(timeChunkMetadataList.get(i), valueChunkMetadata) + : new TableDeviceChunkMetadata(timeChunkMetadataList.get(i), valueChunkMetadata)); } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java index 6496aaf256c1..f958c4c5def2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java @@ -910,10 +910,20 @@ public TSStatus writeLoadTsFileNode( LoadTsFileRateLimiter.getInstance().acquire(pieceNode.getDataSize()); + final DataRegion dataRegion = getDataRegion(dataRegionId); + if (dataRegion == null) { + LOGGER.warn( + "DataRegion {} not found on this DataNode when writing piece node" + + "of TsFile {} (maybe due to region migration), will skip.", + dataRegionId, + pieceNode.getTsFile()); + return RpcUtils.SUCCESS_STATUS; + } + try { - loadTsFileManager.writeToDataRegion(getDataRegion(dataRegionId), pieceNode, uuid); + loadTsFileManager.writeToDataRegion(dataRegion, pieceNode, uuid); } catch (IOException e) { - LOGGER.error( + LOGGER.warn( "IO error when writing piece node of TsFile {} to DataRegion {}.", pieceNode.getTsFile(), dataRegionId, @@ -921,6 +931,15 @@ public TSStatus writeLoadTsFileNode( status.setCode(TSStatusCode.LOAD_FILE_ERROR.getStatusCode()); status.setMessage(e.getMessage()); return status; + } catch (Exception e) { + LOGGER.warn( + "Exception occurred when writing piece node of TsFile {} to DataRegion {}.", + pieceNode.getTsFile(), + dataRegionId, + e); + status.setCode(TSStatusCode.LOAD_FILE_ERROR.getStatusCode()); + status.setMessage(e.getMessage()); + return status; } return RpcUtils.SUCCESS_STATUS; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/ReadChunkCompactionPerformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/ReadChunkCompactionPerformer.java index ca5929399767..c86c02e51b2e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/ReadChunkCompactionPerformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/ReadChunkCompactionPerformer.java @@ -36,7 +36,7 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.exception.write.PageException; -import org.apache.tsfile.file.metadata.AlignedChunkMetadata; +import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.read.TsFileSequenceReader; @@ -179,8 +179,9 @@ private void compactAlignedSeries( MultiTsFileDeviceIterator deviceIterator) throws IOException, InterruptedException, IllegalPathException, PageException { checkThreadInterrupted(); - LinkedList>> readerAndChunkMetadataList = - deviceIterator.getReaderAndChunkMetadataForCurrentAlignedSeries(); + LinkedList>> + readerAndChunkMetadataList = + deviceIterator.getReaderAndChunkMetadataForCurrentAlignedSeries(); if (!checkAlignedSeriesExists(readerAndChunkMetadataList)) { return; } @@ -212,9 +213,9 @@ private void checkThreadInterrupted() throws InterruptedException { } private boolean checkAlignedSeriesExists( - LinkedList>> + LinkedList>> readerAndChunkMetadataList) { - for (Pair> readerListPair : + for (Pair> readerListPair : readerAndChunkMetadataList) { if (!readerListPair.right.isEmpty()) { return true; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java index 32a1593c4a65..1c8aed4ca62d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java @@ -34,7 +34,7 @@ import org.apache.iotdb.db.utils.ModificationUtils; import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.file.metadata.AlignedChunkMetadata; +import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; import org.apache.tsfile.file.metadata.IDeviceID; @@ -384,14 +384,14 @@ public Map getAllSchemasOfCurrentDevice() throws IOEx * @throws IOException if io errors occurred */ @SuppressWarnings({"squid:S1319", "squid:S135"}) - public LinkedList>> + public LinkedList>> getReaderAndChunkMetadataForCurrentAlignedSeries() throws IOException, IllegalPathException { if (currentDevice == null || !currentDevice.right) { return new LinkedList<>(); } - LinkedList>> readerAndChunkMetadataList = - new LinkedList<>(); + LinkedList>> + readerAndChunkMetadataList = new LinkedList<>(); for (TsFileResource tsFileResource : tsFileResourcesSortedByAsc) { if (!deviceIteratorMap.containsKey(tsFileResource)) { continue; @@ -403,7 +403,7 @@ public Map getAllSchemasOfCurrentDevice() throws IOEx MetadataIndexNode firstMeasurementNodeOfCurrentDevice = iterator.getFirstMeasurementNodeOfCurrentDevice(); TsFileSequenceReader reader = readerMap.get(tsFileResource); - List alignedChunkMetadataList = + List alignedChunkMetadataList = reader.getAlignedChunkMetadataByMetadataIndexNode( currentDevice.left, firstMeasurementNodeOfCurrentDevice, ignoreAllNullRows); applyModificationForAlignedChunkMetadataList(tsFileResource, alignedChunkMetadataList); @@ -420,7 +420,7 @@ public Map getAllSchemasOfCurrentDevice() throws IOEx * @param alignedChunkMetadataList list of aligned chunk metadata */ private void applyModificationForAlignedChunkMetadataList( - TsFileResource tsFileResource, List alignedChunkMetadataList) + TsFileResource tsFileResource, List alignedChunkMetadataList) throws IllegalPathException { if (alignedChunkMetadataList.isEmpty()) { // all the value chunks is empty chunk @@ -437,7 +437,7 @@ private void applyModificationForAlignedChunkMetadataList( tsFileResource, r -> new ArrayList<>(tsFileResource.getAllModEntries())); // construct the input params List> for QueryUtils.modifyAlignedChunkMetaData - AlignedChunkMetadata alignedChunkMetadata = alignedChunkMetadataList.get(0); + AbstractAlignedChunkMetadata alignedChunkMetadata = alignedChunkMetadataList.get(0); List valueChunkMetadataList = alignedChunkMetadata.getValueChunkMetadataList(); // match time column modifications diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/batch/BatchedFastAlignedSeriesCompactionExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/batch/BatchedFastAlignedSeriesCompactionExecutor.java index c2044da7f630..636fba8c2ca8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/batch/BatchedFastAlignedSeriesCompactionExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/batch/BatchedFastAlignedSeriesCompactionExecutor.java @@ -43,7 +43,7 @@ import org.apache.tsfile.common.constant.TsFileConstant; import org.apache.tsfile.exception.write.PageException; -import org.apache.tsfile.file.metadata.AlignedChunkMetadata; +import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; import org.apache.tsfile.file.metadata.IDeviceID; @@ -72,7 +72,7 @@ public class BatchedFastAlignedSeriesCompactionExecutor private final List valueMeasurementSchemas; private final List sortedSourceFiles; - private final Map> alignedChunkMetadataCache; + private final Map> alignedChunkMetadataCache; private final BatchCompactionPlan batchCompactionPlan; private final int batchSize = IoTDBDescriptor.getInstance().getConfig().getCompactionMaxAlignedSeriesNumInOneBatch(); @@ -110,11 +110,11 @@ public BatchedFastAlignedSeriesCompactionExecutor( this.batchCompactionPlan = new BatchCompactionPlan(); } - private List getAlignedChunkMetadataListBySelectedValueColumn( + private List getAlignedChunkMetadataListBySelectedValueColumn( TsFileResource tsFileResource, List selectedValueMeasurementSchemas) throws IOException, IllegalPathException { // 1. get Full AlignedChunkMetadata from cache - List alignedChunkMetadataList = null; + List alignedChunkMetadataList = null; if (alignedChunkMetadataCache.containsKey(tsFileResource)) { alignedChunkMetadataList = alignedChunkMetadataCache.get(tsFileResource); } else { @@ -124,8 +124,8 @@ private List getAlignedChunkMetadataListBySelectedValueCol } // 2. generate AlignedChunkMetadata list by selected value columns - List filteredAlignedChunkMetadataList = new ArrayList<>(); - for (AlignedChunkMetadata alignedChunkMetadata : alignedChunkMetadataList) { + List filteredAlignedChunkMetadataList = new ArrayList<>(); + for (AbstractAlignedChunkMetadata alignedChunkMetadata : alignedChunkMetadataList) { filteredAlignedChunkMetadataList.add( AlignedSeriesBatchCompactionUtils.filterAlignedChunkMetadataByIndex( alignedChunkMetadata, batchColumnSelection.getSelectedColumnIndexList())); @@ -264,8 +264,8 @@ public void execute() } @Override - protected List getAlignedChunkMetadataList(TsFileResource resource) - throws IOException, IllegalPathException { + protected List getAlignedChunkMetadataList( + TsFileResource resource) throws IOException, IllegalPathException { return getAlignedChunkMetadataListBySelectedValueColumn(resource, measurementSchemas); } @@ -301,10 +301,10 @@ protected ModifiedStatus isPageModified(PageElement pageElement) { IChunkMetadata batchedAlignedChunkMetadata = alignedPageElement.getChunkMetadataElement().chunkMetadata; TsFileResource resource = alignedPageElement.getChunkMetadataElement().fileElement.resource; - List alignedChunkMetadataListOfFile = + List alignedChunkMetadataListOfFile = alignedChunkMetadataCache.get(resource); - AlignedChunkMetadata originAlignedChunkMetadata = null; - for (AlignedChunkMetadata alignedChunkMetadata : alignedChunkMetadataListOfFile) { + AbstractAlignedChunkMetadata originAlignedChunkMetadata = null; + for (AbstractAlignedChunkMetadata alignedChunkMetadata : alignedChunkMetadataListOfFile) { if (alignedChunkMetadata.getOffsetOfChunkHeader() == batchedAlignedChunkMetadata.getOffsetOfChunkHeader()) { originAlignedChunkMetadata = alignedChunkMetadata; @@ -383,8 +383,8 @@ protected ModifiedStatus isPageModified(PageElement pageElement) { } @Override - protected List getAlignedChunkMetadataList(TsFileResource resource) - throws IOException, IllegalPathException { + protected List getAlignedChunkMetadataList( + TsFileResource resource) throws IOException, IllegalPathException { return getAlignedChunkMetadataListBySelectedValueColumn(resource, measurementSchemas); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/batch/BatchedReadChunkAlignedSeriesCompactionExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/batch/BatchedReadChunkAlignedSeriesCompactionExecutor.java index c0d16841ab10..e343ea59b929 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/batch/BatchedReadChunkAlignedSeriesCompactionExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/batch/BatchedReadChunkAlignedSeriesCompactionExecutor.java @@ -38,7 +38,7 @@ import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.tsfile.exception.write.PageException; -import org.apache.tsfile.file.metadata.AlignedChunkMetadata; +import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.statistics.Statistics; @@ -67,13 +67,14 @@ public class BatchedReadChunkAlignedSeriesCompactionExecutor private final int batchSize = IoTDBDescriptor.getInstance().getConfig().getCompactionMaxAlignedSeriesNumInOneBatch(); private final AlignedSeriesBatchCompactionUtils.BatchColumnSelection batchColumnSelection; - private final LinkedList>> + private final LinkedList>> originReaderAndChunkMetadataList; public BatchedReadChunkAlignedSeriesCompactionExecutor( IDeviceID device, TsFileResource targetResource, - LinkedList>> readerAndChunkMetadataList, + LinkedList>> + readerAndChunkMetadataList, CompactionTsFileWriter writer, CompactionTaskSummary summary, boolean ignoreAllNullRows) @@ -113,7 +114,7 @@ private void compactFirstBatch() throws IOException, PageException { selectedColumnSchemaList = batchColumnSelection.getCurrentSelectedColumnSchemaList(); } - LinkedList>> + LinkedList>> batchedReaderAndChunkMetadataList = filterAlignedChunkMetadataList(readerAndChunkMetadataList, selectedColumnIndexList); @@ -137,7 +138,7 @@ private void compactFirstBatch() throws IOException, PageException { private void compactLeftBatches() throws PageException, IOException { while (batchColumnSelection.hasNext()) { batchColumnSelection.next(); - LinkedList>> + LinkedList>> groupReaderAndChunkMetadataList = filterAlignedChunkMetadataList( readerAndChunkMetadataList, batchColumnSelection.getSelectedColumnIndexList()); @@ -155,16 +156,19 @@ private void compactLeftBatches() throws PageException, IOException { } } - private LinkedList>> + private LinkedList>> filterAlignedChunkMetadataList( - List>> readerAndChunkMetadataList, + List>> + readerAndChunkMetadataList, List selectedMeasurementIndexs) { - LinkedList>> + LinkedList>> groupReaderAndChunkMetadataList = new LinkedList<>(); - for (Pair> pair : readerAndChunkMetadataList) { - List alignedChunkMetadataList = pair.getRight(); - List selectedColumnAlignedChunkMetadataList = new LinkedList<>(); - for (AlignedChunkMetadata alignedChunkMetadata : alignedChunkMetadataList) { + for (Pair> pair : + readerAndChunkMetadataList) { + List alignedChunkMetadataList = pair.getRight(); + List selectedColumnAlignedChunkMetadataList = + new LinkedList<>(); + for (AbstractAlignedChunkMetadata alignedChunkMetadata : alignedChunkMetadataList) { selectedColumnAlignedChunkMetadataList.add( AlignedSeriesBatchCompactionUtils.filterAlignedChunkMetadataByIndex( alignedChunkMetadata, selectedMeasurementIndexs)); @@ -181,7 +185,7 @@ public class FirstBatchedReadChunkAlignedSeriesCompactionExecutor public FirstBatchedReadChunkAlignedSeriesCompactionExecutor( IDeviceID device, TsFileResource targetResource, - LinkedList>> + LinkedList>> readerAndChunkMetadataList, CompactionTsFileWriter writer, CompactionTaskSummary summary, @@ -224,8 +228,8 @@ protected boolean isAllValuePageEmpty(PageLoader timePage, List valu String file = timePage.getFile(); ChunkMetadata timeChunkMetadata = timePage.getChunkMetadata(); - List alignedChunkMetadataList = Collections.emptyList(); - for (Pair> pair : + List alignedChunkMetadataList = Collections.emptyList(); + for (Pair> pair : originReaderAndChunkMetadataList) { TsFileSequenceReader reader = pair.getLeft(); if (reader.getFileName().equals(file)) { @@ -234,8 +238,8 @@ protected boolean isAllValuePageEmpty(PageLoader timePage, List valu } } - AlignedChunkMetadata originAlignedChunkMetadata = null; - for (AlignedChunkMetadata alignedChunkMetadata : alignedChunkMetadataList) { + AbstractAlignedChunkMetadata originAlignedChunkMetadata = null; + for (AbstractAlignedChunkMetadata alignedChunkMetadata : alignedChunkMetadataList) { if (alignedChunkMetadata.getOffsetOfChunkHeader() == timeChunkMetadata.getOffsetOfChunkHeader()) { originAlignedChunkMetadata = alignedChunkMetadata; @@ -303,7 +307,7 @@ public class FollowingBatchedReadChunkAlignedSeriesGroupCompactionExecutor public FollowingBatchedReadChunkAlignedSeriesGroupCompactionExecutor( IDeviceID device, TsFileResource targetResource, - LinkedList>> + LinkedList>> readerAndChunkMetadataList, CompactionTsFileWriter writer, CompactionTaskSummary summary, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/batch/utils/AlignedSeriesBatchCompactionUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/batch/utils/AlignedSeriesBatchCompactionUtils.java index 25613fc2a1a8..c3c58e5bb6a1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/batch/utils/AlignedSeriesBatchCompactionUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/batch/utils/AlignedSeriesBatchCompactionUtils.java @@ -21,7 +21,7 @@ import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.executor.ModifiedStatus; -import org.apache.tsfile.file.metadata.AlignedChunkMetadata; +import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; import org.apache.tsfile.read.TsFileSequenceReader; @@ -43,17 +43,18 @@ public class AlignedSeriesBatchCompactionUtils { private AlignedSeriesBatchCompactionUtils() {} public static void markAlignedChunkHasDeletion( - LinkedList>> + LinkedList>> readerAndChunkMetadataList) { - for (Pair> pair : readerAndChunkMetadataList) { - List alignedChunkMetadataList = pair.getRight(); + for (Pair> pair : + readerAndChunkMetadataList) { + List alignedChunkMetadataList = pair.getRight(); markAlignedChunkHasDeletion(alignedChunkMetadataList); } } public static void markAlignedChunkHasDeletion( - List alignedChunkMetadataList) { - for (AlignedChunkMetadata alignedChunkMetadata : alignedChunkMetadataList) { + List alignedChunkMetadataList) { + for (AbstractAlignedChunkMetadata alignedChunkMetadata : alignedChunkMetadataList) { IChunkMetadata timeChunkMetadata = alignedChunkMetadata.getTimeChunkMetadata(); for (IChunkMetadata iChunkMetadata : alignedChunkMetadata.getValueChunkMetadataList()) { if (iChunkMetadata != null && iChunkMetadata.isModified()) { @@ -68,8 +69,8 @@ public static boolean isTimeChunk(ChunkMetadata chunkMetadata) { return chunkMetadata.getMeasurementUid().isEmpty(); } - public static AlignedChunkMetadata filterAlignedChunkMetadataByIndex( - AlignedChunkMetadata alignedChunkMetadata, List selectedMeasurements) { + public static AbstractAlignedChunkMetadata filterAlignedChunkMetadataByIndex( + AbstractAlignedChunkMetadata alignedChunkMetadata, List selectedMeasurements) { IChunkMetadata[] valueChunkMetadataArr = new IChunkMetadata[selectedMeasurements.size()]; List originValueChunkMetadataList = alignedChunkMetadata.getValueChunkMetadataList(); @@ -77,12 +78,13 @@ public static AlignedChunkMetadata filterAlignedChunkMetadataByIndex( int columnIndex = selectedMeasurements.get(i); valueChunkMetadataArr[i] = originValueChunkMetadataList.get(columnIndex); } - return new AlignedChunkMetadata( + return alignedChunkMetadata.createNewChunkMetadata( alignedChunkMetadata.getTimeChunkMetadata(), Arrays.asList(valueChunkMetadataArr)); } - public static AlignedChunkMetadata fillAlignedChunkMetadataBySchemaList( - AlignedChunkMetadata originAlignedChunkMetadata, List schemaList) { + public static AbstractAlignedChunkMetadata fillAlignedChunkMetadataBySchemaList( + AbstractAlignedChunkMetadata originAlignedChunkMetadata, + List schemaList) { List originValueChunkMetadataList = originAlignedChunkMetadata.getValueChunkMetadataList(); IChunkMetadata[] newValueChunkMetadataArr = new IChunkMetadata[schemaList.size()]; @@ -109,14 +111,14 @@ public static AlignedChunkMetadata fillAlignedChunkMetadataBySchemaList( currentValueChunkMetadataIndex++; } } - return new AlignedChunkMetadata( + return originAlignedChunkMetadata.createNewChunkMetadata( originAlignedChunkMetadata.getTimeChunkMetadata(), Arrays.asList(newValueChunkMetadataArr)); } public static ModifiedStatus calculateAlignedPageModifiedStatus( long startTime, long endTime, - AlignedChunkMetadata originAlignedChunkMetadata, + AbstractAlignedChunkMetadata originAlignedChunkMetadata, boolean ignoreAllNullRows) { ModifiedStatus timePageModifiedStatus = checkIsModified( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastAlignedSeriesCompactionExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastAlignedSeriesCompactionExecutor.java index 64fb585542d7..b489630894bd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastAlignedSeriesCompactionExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastAlignedSeriesCompactionExecutor.java @@ -44,10 +44,12 @@ import org.apache.tsfile.exception.write.PageException; import org.apache.tsfile.file.header.ChunkHeader; import org.apache.tsfile.file.header.PageHeader; +import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; import org.apache.tsfile.file.metadata.AlignedChunkMetadata; import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.file.metadata.TableDeviceChunkMetadata; import org.apache.tsfile.read.TsFileSequenceReader; import org.apache.tsfile.read.common.Chunk; import org.apache.tsfile.utils.Pair; @@ -155,7 +157,8 @@ void deserializeFileIntoChunkMetadataQueue(List fileElements) throws IOException, IllegalPathException { for (FileElement fileElement : fileElements) { TsFileResource resource = fileElement.resource; - List alignedChunkMetadataList = getAlignedChunkMetadataList(resource); + List alignedChunkMetadataList = + getAlignedChunkMetadataList(resource); if (alignedChunkMetadataList.isEmpty()) { // all chunks have been deleted in this file or current file does not contain this aligned @@ -174,7 +177,7 @@ void deserializeFileIntoChunkMetadataQueue(List fileElements) } } - protected List getAlignedChunkMetadataList(TsFileResource resource) + protected List getAlignedChunkMetadataList(TsFileResource resource) throws IOException, IllegalPathException { // read time chunk metadatas and value chunk metadatas in the current file List timeChunkMetadatas = null; @@ -216,7 +219,7 @@ protected List getAlignedChunkMetadataList(TsFileResource } } - List alignedChunkMetadataList = new ArrayList<>(); + List alignedChunkMetadataList = new ArrayList<>(); // if current file contains this aligned device,then construct aligned chunk metadatas if (timeChunkMetadatas != null) { for (int i = 0; i < timeChunkMetadatas.size(); i++) { @@ -228,8 +231,10 @@ protected List getAlignedChunkMetadataList(TsFileResource valueChunkMetadataList.add(chunkMetadata.get(i)); } } - AlignedChunkMetadata alignedChunkMetadata = - new AlignedChunkMetadata(timeChunkMetadatas.get(i), valueChunkMetadataList); + AbstractAlignedChunkMetadata alignedChunkMetadata = + ignoreAllNullRows + ? new AlignedChunkMetadata(timeChunkMetadatas.get(i), valueChunkMetadataList) + : new TableDeviceChunkMetadata(timeChunkMetadatas.get(i), valueChunkMetadataList); alignedChunkMetadataList.add(alignedChunkMetadata); } @@ -345,8 +350,8 @@ protected void deserializeChunkIntoPageQueue(ChunkMetadataElement chunkMetadataE @Override void readChunk(ChunkMetadataElement chunkMetadataElement) throws IOException { updateSummary(chunkMetadataElement, ChunkStatus.READ_IN); - AlignedChunkMetadata alignedChunkMetadata = - (AlignedChunkMetadata) chunkMetadataElement.chunkMetadata; + AbstractAlignedChunkMetadata alignedChunkMetadata = + (AbstractAlignedChunkMetadata) chunkMetadataElement.chunkMetadata; TsFileSequenceReader reader = readerCacheMap.get(chunkMetadataElement.fileElement.resource); chunkMetadataElement.chunk = readChunk(reader, (ChunkMetadata) alignedChunkMetadata.getTimeChunkMetadata()); @@ -418,8 +423,8 @@ protected boolean flushPageToCompactionWriter(PageElement pageElement) protected ModifiedStatus isPageModified(PageElement pageElement) { long startTime = pageElement.getStartTime(); long endTime = pageElement.getEndTime(); - AlignedChunkMetadata alignedChunkMetadata = - (AlignedChunkMetadata) pageElement.getChunkMetadataElement().chunkMetadata; + AbstractAlignedChunkMetadata alignedChunkMetadata = + (AbstractAlignedChunkMetadata) pageElement.getChunkMetadataElement().chunkMetadata; return AlignedSeriesBatchCompactionUtils.calculateAlignedPageModifiedStatus( startTime, endTime, alignedChunkMetadata, ignoreAllNullRows); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/SeriesCompactionExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/SeriesCompactionExecutor.java index 83dab2e76907..c2a5c6c7558f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/SeriesCompactionExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/SeriesCompactionExecutor.java @@ -36,7 +36,7 @@ import org.apache.iotdb.db.utils.datastructure.PatternTreeMapFactory; import org.apache.tsfile.exception.write.PageException; -import org.apache.tsfile.file.metadata.AlignedChunkMetadata; +import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.read.TimeValuePair; @@ -495,14 +495,14 @@ protected void updateSummary(ChunkMetadataElement chunkMetadataElement, ChunkSta case READ_IN: summary.increaseProcessChunkNum( isAligned - ? ((AlignedChunkMetadata) chunkMetadataElement.chunkMetadata) + ? ((AbstractAlignedChunkMetadata) chunkMetadataElement.chunkMetadata) .getValueChunkMetadataList() .size() + 1 : 1); if (isAligned) { for (IChunkMetadata valueChunkMetadata : - ((AlignedChunkMetadata) chunkMetadataElement.chunkMetadata) + ((AbstractAlignedChunkMetadata) chunkMetadataElement.chunkMetadata) .getValueChunkMetadataList()) { if (valueChunkMetadata == null) { continue; @@ -517,7 +517,7 @@ protected void updateSummary(ChunkMetadataElement chunkMetadataElement, ChunkSta case DIRECTORY_FLUSH: if (isAligned) { summary.increaseDirectlyFlushChunkNum( - ((AlignedChunkMetadata) (chunkMetadataElement.chunkMetadata)) + ((AbstractAlignedChunkMetadata) (chunkMetadataElement.chunkMetadata)) .getValueChunkMetadataList() .size() + 1); @@ -528,7 +528,7 @@ protected void updateSummary(ChunkMetadataElement chunkMetadataElement, ChunkSta case DESERIALIZE_CHUNK: if (isAligned) { summary.increaseDeserializedChunkNum( - ((AlignedChunkMetadata) (chunkMetadataElement.chunkMetadata)) + ((AbstractAlignedChunkMetadata) (chunkMetadataElement.chunkMetadata)) .getValueChunkMetadataList() .size() + 1); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/ReadChunkAlignedSeriesCompactionExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/ReadChunkAlignedSeriesCompactionExecutor.java index 12c205c472ed..17281c87051e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/ReadChunkAlignedSeriesCompactionExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/ReadChunkAlignedSeriesCompactionExecutor.java @@ -39,7 +39,7 @@ import org.apache.tsfile.exception.write.PageException; import org.apache.tsfile.file.header.ChunkHeader; import org.apache.tsfile.file.header.PageHeader; -import org.apache.tsfile.file.metadata.AlignedChunkMetadata; +import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; import org.apache.tsfile.file.metadata.IDeviceID; @@ -68,7 +68,7 @@ public class ReadChunkAlignedSeriesCompactionExecutor { protected final IDeviceID device; - protected final LinkedList>> + protected final LinkedList>> readerAndChunkMetadataList; protected final TsFileResource targetResource; protected final CompactionTsFileWriter writer; @@ -85,7 +85,8 @@ public class ReadChunkAlignedSeriesCompactionExecutor { public ReadChunkAlignedSeriesCompactionExecutor( IDeviceID device, TsFileResource targetResource, - LinkedList>> readerAndChunkMetadataList, + LinkedList>> + readerAndChunkMetadataList, CompactionTsFileWriter writer, CompactionTaskSummary summary, boolean ignoreAllNullRows) @@ -108,7 +109,8 @@ public ReadChunkAlignedSeriesCompactionExecutor( public ReadChunkAlignedSeriesCompactionExecutor( IDeviceID device, TsFileResource targetResource, - LinkedList>> readerAndChunkMetadataList, + LinkedList>> + readerAndChunkMetadataList, CompactionTsFileWriter writer, CompactionTaskSummary summary, IMeasurementSchema timeSchema, @@ -131,11 +133,11 @@ public ReadChunkAlignedSeriesCompactionExecutor( private void collectValueColumnSchemaList() throws IOException { Map measurementSchemaMap = new HashMap<>(); for (int i = this.readerAndChunkMetadataList.size() - 1; i >= 0; i--) { - Pair> pair = + Pair> pair = this.readerAndChunkMetadataList.get(i); CompactionTsFileReader reader = (CompactionTsFileReader) pair.getLeft(); - List alignedChunkMetadataList = pair.getRight(); - for (AlignedChunkMetadata alignedChunkMetadata : alignedChunkMetadataList) { + List alignedChunkMetadataList = pair.getRight(); + for (AbstractAlignedChunkMetadata alignedChunkMetadata : alignedChunkMetadataList) { if (alignedChunkMetadata == null) { continue; } @@ -176,10 +178,11 @@ private void collectValueColumnSchemaList() throws IOException { } private void fillAlignedChunkMetadataToMatchSchemaList() { - for (Pair> pair : readerAndChunkMetadataList) { - List alignedChunkMetadataList = pair.getRight(); + for (Pair> pair : + readerAndChunkMetadataList) { + List alignedChunkMetadataList = pair.getRight(); for (int i = 0; i < alignedChunkMetadataList.size(); i++) { - AlignedChunkMetadata alignedChunkMetadata = alignedChunkMetadataList.get(i); + AbstractAlignedChunkMetadata alignedChunkMetadata = alignedChunkMetadataList.get(i); alignedChunkMetadataList.set( i, AlignedSeriesBatchCompactionUtils.fillAlignedChunkMetadataBySchemaList( @@ -193,15 +196,15 @@ protected AlignedChunkWriterImpl constructAlignedChunkWriter() { } public void execute() throws IOException, PageException { - for (Pair> readerListPair : + for (Pair> readerListPair : readerAndChunkMetadataList) { TsFileSequenceReader reader = readerListPair.left; - List alignedChunkMetadataList = readerListPair.right; + List alignedChunkMetadataList = readerListPair.right; if (reader instanceof CompactionTsFileReader) { ((CompactionTsFileReader) reader).markStartOfAlignedSeries(); } - for (AlignedChunkMetadata alignedChunkMetadata : alignedChunkMetadataList) { + for (AbstractAlignedChunkMetadata alignedChunkMetadata : alignedChunkMetadataList) { compactWithAlignedChunk(reader, alignedChunkMetadata); } if (reader instanceof CompactionTsFileReader) { @@ -215,7 +218,7 @@ public void execute() throws IOException, PageException { } private void compactWithAlignedChunk( - TsFileSequenceReader reader, AlignedChunkMetadata alignedChunkMetadata) + TsFileSequenceReader reader, AbstractAlignedChunkMetadata alignedChunkMetadata) throws IOException, PageException { ChunkLoader timeChunk = getChunkLoader(reader, (ChunkMetadata) alignedChunkMetadata.getTimeChunkMetadata()); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/FastCrossCompactionWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/FastCrossCompactionWriter.java index ae5bc2e75911..d4b20039109e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/FastCrossCompactionWriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/FastCrossCompactionWriter.java @@ -26,7 +26,7 @@ import org.apache.tsfile.exception.write.PageException; import org.apache.tsfile.file.header.PageHeader; -import org.apache.tsfile.file.metadata.AlignedChunkMetadata; +import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; import org.apache.tsfile.read.TsFileSequenceReader; @@ -97,8 +97,8 @@ public boolean flushNonAlignedChunk(Chunk chunk, ChunkMetadata chunkMetadata, in @Override public boolean flushAlignedChunk(ChunkMetadataElement chunkMetadataElement, int subTaskId) throws IOException { - AlignedChunkMetadata alignedChunkMetadata = - (AlignedChunkMetadata) chunkMetadataElement.chunkMetadata; + AbstractAlignedChunkMetadata alignedChunkMetadata = + (AbstractAlignedChunkMetadata) chunkMetadataElement.chunkMetadata; IChunkMetadata timeChunkMetadata = alignedChunkMetadata.getTimeChunkMetadata(); List valueChunkMetadatas = alignedChunkMetadata.getValueChunkMetadataList(); Chunk timeChunk = chunkMetadataElement.chunk; @@ -132,8 +132,8 @@ public boolean flushBatchedValueChunk( int subTaskId, AbstractCompactionFlushController flushController) throws IOException { - AlignedChunkMetadata alignedChunkMetadata = - (AlignedChunkMetadata) chunkMetadataElement.chunkMetadata; + AbstractAlignedChunkMetadata alignedChunkMetadata = + (AbstractAlignedChunkMetadata) chunkMetadataElement.chunkMetadata; IChunkMetadata timeChunkMetadata = alignedChunkMetadata.getTimeChunkMetadata(); List valueChunkMetadatas = alignedChunkMetadata.getValueChunkMetadataList(); List valueChunks = chunkMetadataElement.valueChunks; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/FastInnerCompactionWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/FastInnerCompactionWriter.java index e9709af3dbf6..39e621cfaf40 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/FastInnerCompactionWriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/FastInnerCompactionWriter.java @@ -26,7 +26,7 @@ import org.apache.tsfile.exception.write.PageException; import org.apache.tsfile.file.header.PageHeader; -import org.apache.tsfile.file.metadata.AlignedChunkMetadata; +import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; import org.apache.tsfile.read.common.Chunk; @@ -89,8 +89,8 @@ public boolean flushNonAlignedChunk(Chunk chunk, ChunkMetadata chunkMetadata, in @Override public boolean flushAlignedChunk(ChunkMetadataElement chunkMetadataElement, int subTaskId) throws IOException { - AlignedChunkMetadata alignedChunkMetadata = - (AlignedChunkMetadata) chunkMetadataElement.chunkMetadata; + AbstractAlignedChunkMetadata alignedChunkMetadata = + (AbstractAlignedChunkMetadata) chunkMetadataElement.chunkMetadata; IChunkMetadata timeChunkMetadata = alignedChunkMetadata.getTimeChunkMetadata(); List valueChunkMetadatas = alignedChunkMetadata.getValueChunkMetadataList(); Chunk timeChunk = chunkMetadataElement.chunk; @@ -121,8 +121,8 @@ public boolean flushBatchedValueChunk( int subTaskId, AbstractCompactionFlushController flushController) throws IOException { - AlignedChunkMetadata alignedChunkMetadata = - (AlignedChunkMetadata) chunkMetadataElement.chunkMetadata; + AbstractAlignedChunkMetadata alignedChunkMetadata = + (AbstractAlignedChunkMetadata) chunkMetadataElement.chunkMetadata; IChunkMetadata timeChunkMetadata = alignedChunkMetadata.getTimeChunkMetadata(); List valueChunkMetadatas = alignedChunkMetadata.getValueChunkMetadataList(); List valueChunks = chunkMetadataElement.valueChunks; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/io/CompactionDiskTSMIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/io/CompactionDiskTSMIterator.java new file mode 100644 index 000000000000..b90769dba37f --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/io/CompactionDiskTSMIterator.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.storageengine.dataregion.compaction.io; + +import org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.constant.CompactionType; + +import org.apache.tsfile.file.metadata.ChunkGroupMetadata; +import org.apache.tsfile.write.writer.tsmiterator.DiskTSMIterator; + +import java.io.File; +import java.io.IOException; +import java.util.LinkedList; +import java.util.List; + +public class CompactionDiskTSMIterator extends DiskTSMIterator { + + public CompactionDiskTSMIterator( + CompactionType compactionType, + File cmtFile, + List chunkGroupMetadataList, + LinkedList endPosForEachDevice) + throws IOException { + super(cmtFile, chunkGroupMetadataList, endPosForEachDevice); + this.input = new CompactionTsFileInput(compactionType, this.input); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/io/CompactionTsFileInput.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/io/CompactionTsFileInput.java index b13475e4fe67..e51f26da14cb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/io/CompactionTsFileInput.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/io/CompactionTsFileInput.java @@ -19,6 +19,11 @@ package org.apache.iotdb.db.storageengine.dataregion.compaction.io; +import org.apache.iotdb.db.service.metrics.CompactionMetrics; +import org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.CompactionTaskManager; +import org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.constant.CompactionIoDataType; +import org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.constant.CompactionType; + import org.apache.tsfile.exception.StopReadTsFileByInterruptException; import org.apache.tsfile.read.reader.TsFileInput; @@ -29,10 +34,33 @@ public class CompactionTsFileInput implements TsFileInput { private final TsFileInput tsFileInput; - public CompactionTsFileInput(TsFileInput tsFileInput) { + private long metadataOffset = -1; + + /** The type of compaction running. */ + private final CompactionType compactionType; + + /** A flag that indicates if an aligned series is being read. */ + private volatile boolean readingAlignedSeries = false; + + public CompactionTsFileInput(CompactionType compactionType, TsFileInput tsFileInput) { + this.compactionType = compactionType; this.tsFileInput = tsFileInput; } + public void setMetadataOffset(long metadataOffset) { + this.metadataOffset = metadataOffset; + } + + /** Marks the start of reading an aligned series. */ + public void markStartOfAlignedSeries() { + readingAlignedSeries = true; + } + + /** Marks the end of reading an aligned series. */ + public void markEndOfAlignedSeries() { + readingAlignedSeries = false; + } + @Override public long size() throws IOException { try { @@ -71,7 +99,9 @@ public TsFileInput position(long newPosition) throws IOException { @Override public int read(ByteBuffer dst) throws IOException { + acquireReadDataSizeWithCompactionReadRateLimiter(dst.remaining()); int readSize = tsFileInput.read(dst); + updateMetrics(position(), readSize); if (Thread.currentThread().isInterrupted()) { throw new StopReadTsFileByInterruptException(); } @@ -80,7 +110,9 @@ public int read(ByteBuffer dst) throws IOException { @Override public int read(ByteBuffer dst, long position) throws IOException { + acquireReadDataSizeWithCompactionReadRateLimiter(dst.remaining()); int readSize = tsFileInput.read(dst, position); + updateMetrics(position, readSize); if (Thread.currentThread().isInterrupted()) { throw new StopReadTsFileByInterruptException(); } @@ -89,7 +121,7 @@ public int read(ByteBuffer dst, long position) throws IOException { @Override public InputStream wrapAsInputStream() throws IOException { - return tsFileInput.wrapAsInputStream(); + return new CompactionTsFileInputStreamWrapper(tsFileInput.wrapAsInputStream()); } @Override @@ -101,4 +133,90 @@ public void close() throws IOException { public String getFilePath() { return tsFileInput.getFilePath(); } + + private void acquireReadDataSizeWithCompactionReadRateLimiter(int readDataSize) { + CompactionTaskManager.getInstance().getCompactionReadOperationRateLimiter().acquire(1); + CompactionTaskManager.getInstance().getCompactionReadRateLimiter().acquire(readDataSize); + } + + private void updateMetrics(long position, long totalSize) { + if (position >= metadataOffset) { + CompactionMetrics.getInstance() + .recordReadInfo(compactionType, CompactionIoDataType.METADATA, totalSize); + } else { + CompactionMetrics.getInstance() + .recordReadInfo( + compactionType, + readingAlignedSeries + ? CompactionIoDataType.ALIGNED + : CompactionIoDataType.NOT_ALIGNED, + totalSize); + } + } + + private class CompactionTsFileInputStreamWrapper extends InputStream { + + private final InputStream inputStream; + + public CompactionTsFileInputStreamWrapper(InputStream inputStream) { + this.inputStream = inputStream; + } + + @Override + public int read() throws IOException { + acquireReadDataSizeWithCompactionReadRateLimiter(1); + long position = position(); + int readSize = inputStream.read(); + updateMetrics(position, readSize); + return readSize; + } + + @Override + public int read(byte[] b) throws IOException { + acquireReadDataSizeWithCompactionReadRateLimiter(b.length); + long position = position(); + int readSize = inputStream.read(b); + updateMetrics(position, readSize); + return readSize; + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + acquireReadDataSizeWithCompactionReadRateLimiter(len); + long position = position(); + int readSize = inputStream.read(b, off, len); + updateMetrics(position, readSize); + return readSize; + } + + @Override + public long skip(long n) throws IOException { + return inputStream.skip(n); + } + + @Override + public int available() throws IOException { + return inputStream.available(); + } + + @Override + public void close() throws IOException { + inputStream.close(); + } + + @Override + public synchronized void mark(int readlimit) { + inputStream.mark(readlimit); + } + + @Override + public synchronized void reset() throws IOException { + inputStream.reset(); + } + + @Override + public boolean markSupported() { + return inputStream.markSupported(); + } + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/io/CompactionTsFileReader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/io/CompactionTsFileReader.java index bf98611db2c2..7aabb0856433 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/io/CompactionTsFileReader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/io/CompactionTsFileReader.java @@ -19,29 +19,22 @@ package org.apache.iotdb.db.storageengine.dataregion.compaction.io; -import org.apache.iotdb.db.service.metrics.CompactionMetrics; -import org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.CompactionTaskManager; -import org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.constant.CompactionIoDataType; import org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.constant.CompactionType; import org.apache.tsfile.file.IMetadataIndexEntry; import org.apache.tsfile.file.header.ChunkHeader; -import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.MetadataIndexNode; import org.apache.tsfile.file.metadata.TimeseriesMetadata; import org.apache.tsfile.file.metadata.enums.MetadataIndexNodeType; import org.apache.tsfile.read.TsFileSequenceReader; -import org.apache.tsfile.read.common.Chunk; import org.apache.tsfile.utils.Pair; import java.io.IOException; -import java.io.InputStream; import java.nio.ByteBuffer; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.function.LongConsumer; /** * This class extends the TsFileSequenceReader class to read and manage TsFile with a focus on @@ -50,14 +43,6 @@ */ public class CompactionTsFileReader extends TsFileSequenceReader { - private long metadataOffset = 0; - - /** The type of compaction running. */ - CompactionType compactionType; - - /** A flag that indicates if an aligned series is being read. */ - private volatile boolean readingAlignedSeries = false; - /** * Constructs a new instance of CompactionTsFileReader. * @@ -67,44 +52,20 @@ public class CompactionTsFileReader extends TsFileSequenceReader { */ public CompactionTsFileReader(String file, CompactionType compactionType) throws IOException { super(file); - this.tsFileInput = new CompactionTsFileInput(tsFileInput); - this.compactionType = compactionType; - this.metadataOffset = readFileMetadata().getMetaOffset(); - } - - @Override - protected ByteBuffer readData(long position, int totalSize, LongConsumer ioSizeRecorder) - throws IOException { - acquireReadDataSizeWithCompactionReadRateLimiter(totalSize); - ByteBuffer buffer = super.readData(position, totalSize, ioSizeRecorder); - if (position >= metadataOffset) { - CompactionMetrics.getInstance() - .recordReadInfo(compactionType, CompactionIoDataType.METADATA, totalSize); - } else { - CompactionMetrics.getInstance() - .recordReadInfo( - compactionType, - readingAlignedSeries - ? CompactionIoDataType.ALIGNED - : CompactionIoDataType.NOT_ALIGNED, - totalSize); - } - return buffer; + CompactionTsFileInput compactionTsFileInput = + new CompactionTsFileInput(compactionType, tsFileInput); + this.tsFileInput = compactionTsFileInput; + compactionTsFileInput.setMetadataOffset(readFileMetadata().getMetaOffset()); } /** Marks the start of reading an aligned series. */ public void markStartOfAlignedSeries() { - readingAlignedSeries = true; + ((CompactionTsFileInput) tsFileInput).markStartOfAlignedSeries(); } /** Marks the end of reading an aligned series. */ public void markEndOfAlignedSeries() { - readingAlignedSeries = false; - } - - @Override - public Chunk readMemChunk(ChunkMetadata metaData) throws IOException { - return super.readMemChunk(metaData); + ((CompactionTsFileInput) tsFileInput).markEndOfAlignedSeries(); } @SuppressWarnings("java:S2177") @@ -112,10 +73,6 @@ public ChunkHeader readChunkHeader(long position) throws IOException { return ChunkHeader.deserializeFrom(tsFileInput, position); } - public InputStream wrapAsInputStream() throws IOException { - return this.tsFileInput.wrapAsInputStream(); - } - public Map>> getTimeseriesMetadataAndOffsetByDevice( MetadataIndexNode measurementNode, @@ -186,11 +143,6 @@ public Map> getTimeseriesMetadataOffsetByDevice( return timeseriesMetadataOffsetMap; } - private void acquireReadDataSizeWithCompactionReadRateLimiter(int readDataSize) { - CompactionTaskManager.getInstance().getCompactionReadOperationRateLimiter().acquire(1); - CompactionTaskManager.getInstance().getCompactionReadRateLimiter().acquire(readDataSize); - } - @Override public boolean equals(Object o) { return super.equals(o); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/io/CompactionTsFileWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/io/CompactionTsFileWriter.java index 7d89747d2f65..fcea93d6c2e3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/io/CompactionTsFileWriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/io/CompactionTsFileWriter.java @@ -36,6 +36,7 @@ import org.apache.tsfile.write.chunk.IChunkWriter; import org.apache.tsfile.write.record.Tablet.ColumnCategory; import org.apache.tsfile.write.writer.TsFileIOWriter; +import org.apache.tsfile.write.writer.tsmiterator.TSMIterator; import java.io.File; import java.io.IOException; @@ -150,6 +151,14 @@ public void endFile() throws IOException { .recordWriteInfo(type, CompactionIoDataType.METADATA, writtenDataSize); } + @Override + protected TSMIterator getTSMIterator() throws IOException { + return hasChunkMetadataInDisk + ? new CompactionDiskTSMIterator( + type, chunkMetadataTempFile, chunkGroupMetadataList, endPosInCMTForDevice) + : TSMIterator.getTSMIteratorInMemory(chunkGroupMetadataList); + } + public boolean isEmptyTargetFile() { return isEmptyTargetFile; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedReadOnlyMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedReadOnlyMemChunk.java index 569db7347ab5..a3eb66b018d3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedReadOnlyMemChunk.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedReadOnlyMemChunk.java @@ -30,6 +30,7 @@ import org.apache.tsfile.file.metadata.AlignedChunkMetadata; import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; +import org.apache.tsfile.file.metadata.TableDeviceChunkMetadata; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.file.metadata.statistics.Statistics; import org.apache.tsfile.read.common.TimeRange; @@ -75,10 +76,11 @@ public AlignedReadOnlyMemChunk( timeColumnDeletion, valueColumnsDeletionList, context.isIgnoreAllNullRows()); - initAlignedChunkMetaFromTsBlock(); + initAlignedChunkMetaFromTsBlock(context.isIgnoreAllNullRows()); } - private void initAlignedChunkMetaFromTsBlock() throws QueryProcessException { + private void initAlignedChunkMetaFromTsBlock(boolean ignoreAllNullRows) + throws QueryProcessException { // Time chunk Statistics timeStatistics = Statistics.getStatsByType(TSDataType.VECTOR); IChunkMetadata timeChunkMetadata = @@ -160,7 +162,9 @@ private void initAlignedChunkMetaFromTsBlock() throws QueryProcessException { } } IChunkMetadata alignedChunkMetadata = - new AlignedChunkMetadata(timeChunkMetadata, valueChunkMetadataList); + ignoreAllNullRows + ? new AlignedChunkMetadata(timeChunkMetadata, valueChunkMetadataList) + : new TableDeviceChunkMetadata(timeChunkMetadata, valueChunkMetadataList); alignedChunkMetadata.setChunkLoader(new MemAlignedChunkLoader(context, this)); alignedChunkMetadata.setVersion(Long.MAX_VALUE); cachedMetaData = alignedChunkMetadata; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java index 3b2c5fe29449..b6f5629528fa 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java @@ -80,6 +80,7 @@ import org.apache.iotdb.rpc.TSStatusCode; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; import org.apache.tsfile.file.metadata.AlignedChunkMetadata; import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; @@ -1788,7 +1789,7 @@ RestorableTsFileIOWriter getWriter() { private void processAlignedChunkMetaDataFromFlushedMemTable( IDeviceID deviceID, - AlignedChunkMetadata alignedChunkMetadata, + AbstractAlignedChunkMetadata alignedChunkMetadata, Map> measurementToChunkMetaMap, Map> measurementToChunkHandleMap, String filePath) { @@ -1841,10 +1842,10 @@ private void buildChunkHandleForFlushedMemTable( Map> measurementToChunkMetaList, Map> measurementToChunkHandleList) { for (IChunkMetadata chunkMetadata : chunkMetadataList) { - if (chunkMetadata instanceof AlignedChunkMetadata) { + if (chunkMetadata instanceof AbstractAlignedChunkMetadata) { processAlignedChunkMetaDataFromFlushedMemTable( deviceID, - (AlignedChunkMetadata) chunkMetadata, + (AbstractAlignedChunkMetadata) chunkMetadata, measurementToChunkMetaList, measurementToChunkHandleList, this.tsFileResource.getTsFilePath()); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/DiskAlignedChunkLoader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/DiskAlignedChunkLoader.java index adef246ec8d9..a7c6eb96d429 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/DiskAlignedChunkLoader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/DiskAlignedChunkLoader.java @@ -25,7 +25,7 @@ import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileID; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; -import org.apache.tsfile.file.metadata.AlignedChunkMetadata; +import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; import org.apache.tsfile.read.common.Chunk; @@ -76,7 +76,8 @@ public IChunkReader getChunkReader(IChunkMetadata chunkMetaData, Filter globalTi throws IOException { long t1 = System.nanoTime(); try { - AlignedChunkMetadata alignedChunkMetadata = (AlignedChunkMetadata) chunkMetaData; + AbstractAlignedChunkMetadata alignedChunkMetadata = + (AbstractAlignedChunkMetadata) chunkMetaData; ChunkMetadata timeChunkMetadata = (ChunkMetadata) alignedChunkMetadata.getTimeChunkMetadata(); Chunk timeChunk = ChunkCache.getInstance() diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedChunkReader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedChunkReader.java index 33c85408f0d3..12ac281dffc5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedChunkReader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedChunkReader.java @@ -21,7 +21,7 @@ import org.apache.iotdb.db.storageengine.dataregion.memtable.AlignedReadOnlyMemChunk; -import org.apache.tsfile.file.metadata.AlignedChunkMetadata; +import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; import org.apache.tsfile.read.common.BatchData; import org.apache.tsfile.read.filter.basic.Filter; import org.apache.tsfile.read.reader.IChunkReader; @@ -42,7 +42,7 @@ public MemAlignedChunkReader(AlignedReadOnlyMemChunk readableChunk, Filter globa Collections.singletonList( new MemAlignedPageReader( readableChunk.getTsBlock(), - (AlignedChunkMetadata) readableChunk.getChunkMetaData(), + (AbstractAlignedChunkMetadata) readableChunk.getChunkMetaData(), globalTimeFilter)); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedPageReader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedPageReader.java index 473cd7366c33..6adfe09c0542 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedPageReader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedPageReader.java @@ -22,7 +22,7 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.file.metadata.AlignedChunkMetadata; +import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; import org.apache.tsfile.file.metadata.statistics.Statistics; import org.apache.tsfile.read.common.BatchData; import org.apache.tsfile.read.common.BatchDataFactory; @@ -45,7 +45,7 @@ public class MemAlignedPageReader implements IPageReader { private final TsBlock tsBlock; - private final AlignedChunkMetadata chunkMetadata; + private final AbstractAlignedChunkMetadata chunkMetadata; private Filter recordFilter; private PaginationController paginationController = UNLIMITED_PAGINATION_CONTROLLER; @@ -53,7 +53,7 @@ public class MemAlignedPageReader implements IPageReader { private TsBlockBuilder builder; public MemAlignedPageReader( - TsBlock tsBlock, AlignedChunkMetadata chunkMetadata, Filter recordFilter) { + TsBlock tsBlock, AbstractAlignedChunkMetadata chunkMetadata, Filter recordFilter) { this.tsBlock = tsBlock; this.chunkMetadata = chunkMetadata; this.recordFilter = recordFilter; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/metadata/DiskAlignedChunkMetadataLoader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/metadata/DiskAlignedChunkMetadataLoader.java index afc57079c88b..f243b51ee1a7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/metadata/DiskAlignedChunkMetadataLoader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/metadata/DiskAlignedChunkMetadataLoader.java @@ -26,8 +26,8 @@ import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.iotdb.db.utils.ModificationUtils; +import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; import org.apache.tsfile.file.metadata.AbstractAlignedTimeSeriesMetadata; -import org.apache.tsfile.file.metadata.AlignedChunkMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; import org.apache.tsfile.file.metadata.ITimeSeriesMetadata; import org.apache.tsfile.read.controller.IChunkMetadataLoader; @@ -83,7 +83,7 @@ public DiskAlignedChunkMetadataLoader( public List loadChunkMetadataList(ITimeSeriesMetadata timeSeriesMetadata) { final long t1 = System.nanoTime(); try { - List alignedChunkMetadataList = + List alignedChunkMetadataList = ((AbstractAlignedTimeSeriesMetadata) timeSeriesMetadata).getCopiedChunkMetadataList(); // when alignedChunkMetadataList.size() == 1, it means that the chunk statistics is same as diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java index 41bbbafe901c..74bf838a8d3f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java @@ -77,7 +77,7 @@ import java.util.stream.Stream; import static org.apache.iotdb.db.utils.constant.SqlConstant.ROOT; -import static org.apache.iotdb.db.utils.constant.SqlConstant.ROOT_DOT; +import static org.apache.iotdb.db.utils.constant.SqlConstant.TREE_MODEL_DATABASE_PREFIX; /** * {@link LoadTsFileManager} is used for dealing with {@link LoadTsFilePieceNode} and {@link @@ -445,7 +445,8 @@ private void write(DataPartitionInfo partitionInfo, ChunkData chunkData) throws // Table model needs to register TableSchema final String tableName = chunkData.getDevice() != null ? chunkData.getDevice().getTableName() : null; - if (tableName != null && !(tableName.startsWith(ROOT_DOT) || tableName.equals(ROOT))) { + if (tableName != null + && !(tableName.startsWith(TREE_MODEL_DATABASE_PREFIX) || tableName.equals(ROOT))) { writer .getSchema() .getTableSchemaMap() diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTableStatementDataTypeConvertExecutionVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTableStatementDataTypeConvertExecutionVisitor.java index de9dd0908879..646185126a0e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTableStatementDataTypeConvertExecutionVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTableStatementDataTypeConvertExecutionVisitor.java @@ -123,6 +123,10 @@ public Optional visitLoadTsFile( || result.getCode() == TSStatusCode.REDIRECTION_RECOMMEND.getStatusCode() || result.getCode() == TSStatusCode.LOAD_IDEMPOTENT_CONFLICT_EXCEPTION.getStatusCode())) { + LOGGER.warn( + "Failed to convert data type for LoadTsFileStatement: {}, status code is {}.", + loadTsFileStatement, + result.getCode()); return Optional.empty(); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTreeStatementDataTypeConvertExecutionVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTreeStatementDataTypeConvertExecutionVisitor.java index 6f7d732e6fb4..eb7efba36185 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTreeStatementDataTypeConvertExecutionVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTreeStatementDataTypeConvertExecutionVisitor.java @@ -109,6 +109,10 @@ file, new IoTDBTreePattern(null), Long.MIN_VALUE, Long.MAX_VALUE, null, null)) { || result.getCode() == TSStatusCode.REDIRECTION_RECOMMEND.getStatusCode() || result.getCode() == TSStatusCode.LOAD_IDEMPOTENT_CONFLICT_EXCEPTION.getStatusCode())) { + LOGGER.warn( + "Failed to convert data type for LoadTsFileStatement: {}, status code is {}.", + loadTsFileStatement, + result.getCode()); return Optional.empty(); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingQueue.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingQueue.java index cf78aff6d4a2..ca9bf096efa8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingQueue.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingQueue.java @@ -97,6 +97,9 @@ public abstract class SubscriptionPrefetchingQueue { private final SubscriptionPrefetchingQueueStates states; + private static final long STATE_REPORT_INTERVAL_IN_MS = 10_000L; + private long lastStateReportTimestamp = System.currentTimeMillis(); + private volatile boolean isCompleted = false; private volatile boolean isClosed = false; @@ -259,6 +262,7 @@ public boolean executePrefetch() { if (isClosed()) { return false; } + reportStateIfNeeded(); // TODO: more refined behavior (prefetch/serialize/...) control if (states.shouldPrefetch()) { tryPrefetch(); @@ -274,6 +278,13 @@ public boolean executePrefetch() { } } + private void reportStateIfNeeded() { + if (System.currentTimeMillis() - lastStateReportTimestamp > STATE_REPORT_INTERVAL_IN_MS) { + LOGGER.info("Subscription: SubscriptionPrefetchingQueue state {}", this); + lastStateReportTimestamp = System.currentTimeMillis(); + } + } + @SafeVarargs private final void remapInFlightEventsSnapshot( final RemappingFunction... functions) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/response/SubscriptionEventTabletResponse.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/response/SubscriptionEventTabletResponse.java index cd93cd350ba0..c6150536f21a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/response/SubscriptionEventTabletResponse.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/response/SubscriptionEventTabletResponse.java @@ -116,11 +116,6 @@ public void fetchNextResponse(final long offset /* unused */) throws Exception { @Override public synchronized void nack() { - if (nextOffset.get() == 1) { - // do nothing if with complete tablets - return; - } - cleanUp(); // should not reset the iterator of batch when init diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/utils/TsFileRewriteOverPrecisedI64Scan.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/utils/TsFileRewriteOverPrecisedI64Scan.java new file mode 100644 index 000000000000..a93570905346 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/utils/TsFileRewriteOverPrecisedI64Scan.java @@ -0,0 +1,223 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.tools.utils; + +import org.apache.tsfile.encoding.encoder.TSEncodingBuilder; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.header.ChunkHeader; +import org.apache.tsfile.file.header.PageHeader; +import org.apache.tsfile.file.metadata.statistics.Statistics; +import org.apache.tsfile.read.common.BatchData; +import org.apache.tsfile.read.common.Chunk; +import org.apache.tsfile.read.reader.IPageReader; +import org.apache.tsfile.read.reader.chunk.ChunkReader; +import org.apache.tsfile.read.reader.chunk.TableChunkReader; +import org.apache.tsfile.utils.Pair; +import org.apache.tsfile.utils.TsPrimitiveType; +import org.apache.tsfile.write.chunk.ChunkWriterImpl; +import org.apache.tsfile.write.chunk.ValueChunkWriter; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.apache.tsfile.write.writer.TsFileIOWriter; + +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Collections; +import java.util.List; + +public class TsFileRewriteOverPrecisedI64Scan extends TsFileSequenceScan { + + private final File target; + private TsFileIOWriter writer; + private Chunk currTimeChunk; + + public TsFileRewriteOverPrecisedI64Scan(File target) { + this.target = target; + } + + public static void main(String[] args) { + File sourceFile = new File(args[0]); + File targetFile = new File(args[1]); + TsFileRewriteOverPrecisedI64Scan scan = new TsFileRewriteOverPrecisedI64Scan(targetFile); + scan.scanTsFile(sourceFile); + + long sourceLength = sourceFile.length(); + long targetLength = targetFile.length(); + System.out.printf( + "Before rewrite %d, after rewrite %d, ratio %f%n", + sourceLength, targetLength, sourceLength * 1.0 / targetLength); + } + + @Override + protected boolean onFileOpen(File file) throws IOException { + boolean shouldScan = super.onFileOpen(file); + if (shouldScan) { + writer = new TsFileIOWriter(target); + } + return shouldScan; + } + + @Override + protected void onFileEnd() throws IOException { + writer.endChunkGroup(); + writer.endFile(); + } + + @Override + protected void onChunkGroup() throws IOException { + if (currDeviceID != null) { + writer.endChunkGroup(); + } + super.onChunkGroup(); + writer.startChunkGroup(currDeviceID); + } + + @Override + protected void onChunk(PageVisitor pageVisitor) throws IOException { + reader.position(reader.position() - 1); + Chunk chunk = reader.readMemChunk(reader.position()); + chunk = + new Chunk( + chunk.getHeader(), + chunk.getData(), + Collections.emptyList(), + Statistics.getStatsByType(chunk.getHeader().getDataType())); + currMeasurementID = chunk.getHeader().getMeasurementID(); + currTimeseriesID = new Pair<>(currDeviceID, currMeasurementID); + if (!currDeviceAligned) { + onNonAlignedChunk(chunk); + } else { + onAlignedChunk(chunk); + } + System.out.println("Processed a chunk of " + currDeviceID + "." + currMeasurementID); + reader.position( + reader.position() + + chunk.getHeader().getSerializedSize() + + chunk.getHeader().getDataSize()); + } + + private void onNonAlignedChunk(Chunk chunk) throws IOException { + if (chunk.getHeader().getDataType() != TSDataType.INT64) { + writer.writeChunk(chunk); + } else { + if (!rewriteInt64ChunkNonAligned(chunk)) { + writer.writeChunk(chunk); + } + } + } + + private void onAlignedChunk(Chunk chunk) throws IOException { + if (isTimeChunk || chunk.getHeader().getDataType() != TSDataType.INT64) { + writer.writeChunk(chunk); + if (isTimeChunk) { + currTimeChunk = chunk; + } + } else { + if (!rewriteInt64ChunkAligned(chunk)) { + writer.writeChunk(chunk); + } + } + } + + private boolean rewriteInt64ChunkNonAligned(Chunk chunk) throws IOException { + ChunkReader chunkReader = new ChunkReader(chunk); + ChunkHeader header = chunk.getHeader(); + List pageReaders = chunkReader.loadPageReaderList(); + ChunkWriterImpl chunkWriter = + new ChunkWriterImpl( + new MeasurementSchema( + header.getMeasurementID(), + TSDataType.INT32, + header.getEncodingType(), + header.getCompressionType())); + + for (IPageReader pageReader : pageReaders) { + BatchData batchData = pageReader.getAllSatisfiedPageData(); + while (batchData.hasCurrent()) { + int intVal = (int) batchData.getLong(); + if (intVal != batchData.getLong()) { + // the chunk is not over-precised + return false; + } + chunkWriter.write(batchData.currentTime(), (int) batchData.getLong()); + } + chunkWriter.sealCurrentPage(); + } + chunkWriter.writeToFileWriter(writer); + return true; + } + + private boolean rewriteInt64ChunkAligned(Chunk chunk) throws IOException { + // use TableChunkReader so that nulls will not be skipped + TableChunkReader chunkReader = + new TableChunkReader(currTimeChunk, Collections.singletonList(chunk), null); + ChunkHeader header = chunk.getHeader(); + List pageReaders = chunkReader.loadPageReaderList(); + ValueChunkWriter valueChunkWriter = + new ValueChunkWriter( + header.getMeasurementID(), + header.getCompressionType(), + TSDataType.INT32, + header.getEncodingType(), + TSEncodingBuilder.getEncodingBuilder(header.getEncodingType()) + .getEncoder(TSDataType.INT32)); + + for (IPageReader pageReader : pageReaders) { + BatchData batchData = pageReader.getAllSatisfiedPageData(); + while (batchData.hasCurrent()) { + TsPrimitiveType[] vector = batchData.getVector(); + boolean isNull = vector[0] == null; + if (!isNull) { + int intVal = (int) vector[0].getLong(); + if (intVal != vector[0].getLong()) { + // the chunk is not over-precised + return false; + } + valueChunkWriter.write(batchData.currentTime(), intVal, false); + } else { + valueChunkWriter.write(batchData.currentTime(), 0, true); + } + } + valueChunkWriter.sealCurrentPage(); + } + valueChunkWriter.writeToFileWriter(writer); + return true; + } + + @Override + protected void onTimePage(PageHeader pageHeader, ByteBuffer pageData, ChunkHeader chunkHeader) { + // do nothing + } + + @Override + protected void onValuePage(PageHeader pageHeader, ByteBuffer pageData, ChunkHeader chunkHeader) { + // do nothing + } + + @Override + protected void onNonAlignedPage( + PageHeader pageHeader, ByteBuffer pageData, ChunkHeader chunkHeader) { + // do nothing + } + + @Override + protected void onException(Throwable t) { + throw new RuntimeException(t); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/utils/TsFileRewriteSmallRangeI64Scan.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/utils/TsFileRewriteSmallRangeI64Scan.java new file mode 100644 index 000000000000..360fc1c05768 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/utils/TsFileRewriteSmallRangeI64Scan.java @@ -0,0 +1,230 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.tools.utils; + +import org.apache.tsfile.encoding.encoder.TSEncodingBuilder; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.header.ChunkHeader; +import org.apache.tsfile.file.header.PageHeader; +import org.apache.tsfile.file.metadata.statistics.Statistics; +import org.apache.tsfile.read.common.BatchData; +import org.apache.tsfile.read.common.Chunk; +import org.apache.tsfile.read.reader.IPageReader; +import org.apache.tsfile.read.reader.chunk.ChunkReader; +import org.apache.tsfile.read.reader.chunk.TableChunkReader; +import org.apache.tsfile.utils.Pair; +import org.apache.tsfile.utils.TsPrimitiveType; +import org.apache.tsfile.write.chunk.ChunkWriterImpl; +import org.apache.tsfile.write.chunk.ValueChunkWriter; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.apache.tsfile.write.writer.TsFileIOWriter; + +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Collections; +import java.util.List; + +public class TsFileRewriteSmallRangeI64Scan extends TsFileSequenceScan { + + private final File target; + private TsFileIOWriter writer; + private Chunk currTimeChunk; + + public TsFileRewriteSmallRangeI64Scan(File target) { + this.target = target; + } + + public static void main(String[] args) throws IOException { + File sourceFile = new File(args[0]); + File targetFile = new File(args[1]); + TsFileRewriteSmallRangeI64Scan scan = new TsFileRewriteSmallRangeI64Scan(targetFile); + scan.scanTsFile(sourceFile); + + long sourceLength = sourceFile.length(); + long targetLength = targetFile.length(); + System.out.printf( + "Before rewrite %d, after rewrite %d, ratio %f%n", + sourceLength, targetLength, sourceLength * 1.0 / targetLength); + } + + @Override + protected boolean onFileOpen(File file) throws IOException { + boolean shouldScan = super.onFileOpen(file); + if (shouldScan) { + writer = new TsFileIOWriter(target); + } + return shouldScan; + } + + @Override + protected void onFileEnd() throws IOException { + writer.endChunkGroup(); + writer.endFile(); + } + + @Override + protected void onChunkGroup() throws IOException { + if (currDeviceID != null) { + writer.endChunkGroup(); + } + super.onChunkGroup(); + writer.startChunkGroup(currDeviceID); + } + + @Override + protected void onChunk(PageVisitor pageVisitor) throws IOException { + reader.position(reader.position() - 1); + Chunk chunk = reader.readMemChunk(reader.position()); + chunk = + new Chunk( + chunk.getHeader(), + chunk.getData(), + Collections.emptyList(), + Statistics.getStatsByType(chunk.getHeader().getDataType())); + currMeasurementID = chunk.getHeader().getMeasurementID(); + currTimeseriesID = new Pair<>(currDeviceID, currMeasurementID); + if (!currDeviceAligned) { + onNonAlignedChunk(chunk); + } else { + onAlignedChunk(chunk); + } + System.out.println("Processed a chunk of " + currDeviceID + "." + currMeasurementID); + reader.position( + reader.position() + + chunk.getHeader().getSerializedSize() + + chunk.getHeader().getDataSize()); + } + + private void onNonAlignedChunk(Chunk chunk) throws IOException { + if (chunk.getHeader().getDataType() != TSDataType.INT64) { + writer.writeChunk(chunk); + } else { + if (!rewriteInt64ChunkNonAligned(chunk)) { + writer.writeChunk(chunk); + } + } + } + + private void onAlignedChunk(Chunk chunk) throws IOException { + if (isTimeChunk || chunk.getHeader().getDataType() != TSDataType.INT64) { + writer.writeChunk(chunk); + if (isTimeChunk) { + currTimeChunk = chunk; + } + } else { + if (!rewriteInt64ChunkAligned(chunk)) { + writer.writeChunk(chunk); + } + } + } + + private boolean rewriteInt64ChunkNonAligned(Chunk chunk) throws IOException { + ChunkReader chunkReader = new ChunkReader(chunk); + ChunkHeader header = chunk.getHeader(); + List pageReaders = chunkReader.loadPageReaderList(); + ChunkWriterImpl chunkWriter = + new ChunkWriterImpl( + new MeasurementSchema( + header.getMeasurementID(), + TSDataType.INT32, + header.getEncodingType(), + header.getCompressionType())); + + Long firstVal = null; + for (IPageReader pageReader : pageReaders) { + BatchData batchData = pageReader.getAllSatisfiedPageData(); + while (batchData.hasCurrent()) { + if (firstVal == null) { + firstVal = batchData.getLong(); + } + long diff = batchData.getLong() - firstVal; + if (diff > Integer.MAX_VALUE) { + return false; + } + chunkWriter.write(batchData.currentTime(), (int) diff); + batchData.next(); + } + chunkWriter.sealCurrentPage(); + } + chunkWriter.writeToFileWriter(writer); + return true; + } + + private boolean rewriteInt64ChunkAligned(Chunk chunk) throws IOException { + // use TableChunkReader so that nulls will not be skipped + TableChunkReader chunkReader = + new TableChunkReader(currTimeChunk, Collections.singletonList(chunk), null); + ChunkHeader header = chunk.getHeader(); + List pageReaders = chunkReader.loadPageReaderList(); + ValueChunkWriter valueChunkWriter = + new ValueChunkWriter( + header.getMeasurementID(), + header.getCompressionType(), + TSDataType.INT32, + header.getEncodingType(), + TSEncodingBuilder.getEncodingBuilder(header.getEncodingType()) + .getEncoder(TSDataType.INT32)); + + Long firstVal = null; + for (IPageReader pageReader : pageReaders) { + BatchData batchData = pageReader.getAllSatisfiedPageData(); + while (batchData.hasCurrent()) { + TsPrimitiveType[] vector = batchData.getVector(); + boolean isNull = vector[0] == null; + if (!isNull) { + if (firstVal == null) { + firstVal = vector[0].getLong(); + } + long diff = vector[0].getLong() - firstVal; + if (diff > Integer.MAX_VALUE) { + return false; + } + valueChunkWriter.write(batchData.currentTime(), (int) diff, false); + } else { + valueChunkWriter.write(batchData.currentTime(), 0, true); + } + } + valueChunkWriter.sealCurrentPage(); + } + valueChunkWriter.writeToFileWriter(writer); + return true; + } + + @Override + protected void onTimePage(PageHeader pageHeader, ByteBuffer pageData, ChunkHeader chunkHeader) { + // do nothing + } + + @Override + protected void onValuePage(PageHeader pageHeader, ByteBuffer pageData, ChunkHeader chunkHeader) { + // do nothing + } + + @Override + protected void onNonAlignedPage( + PageHeader pageHeader, ByteBuffer pageData, ChunkHeader chunkHeader) { + // do nothing + } + + @Override + protected void onException(Throwable t) { + throw new RuntimeException(t); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/utils/TsFileSequenceScan.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/utils/TsFileSequenceScan.java index d9ea636948f8..eddb7a0e80c4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/utils/TsFileSequenceScan.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/utils/TsFileSequenceScan.java @@ -44,6 +44,9 @@ public abstract class TsFileSequenceScan { protected String currMeasurementID; protected Pair currTimeseriesID; protected boolean currChunkOnePage; + protected ChunkHeader currChunkHeader; + protected boolean currDeviceAligned; + protected boolean isTimeChunk; public TsFileSequenceScan() {} @@ -60,23 +63,26 @@ protected boolean onFileOpen(File file) throws IOException { protected void onFileEnd() throws IOException {} protected void onChunk(PageVisitor pageVisitor) throws IOException { - ChunkHeader chunkHeader = reader.readChunkHeader(marker); - if (chunkHeader.getDataSize() == 0) { + currChunkHeader = reader.readChunkHeader(marker); + if (currChunkHeader.getDataSize() == 0) { // empty value chunk return; } - currMeasurementID = chunkHeader.getMeasurementID(); + currMeasurementID = currChunkHeader.getMeasurementID(); currTimeseriesID = new Pair<>(currDeviceID, currMeasurementID); - int dataSize = chunkHeader.getDataSize(); + int dataSize = currChunkHeader.getDataSize(); + onChunkData(pageVisitor, dataSize); + } + protected void onChunkData(PageVisitor pageVisitor, int dataSize) throws IOException { while (dataSize > 0) { PageHeader pageHeader = reader.readPageHeader( - chunkHeader.getDataType(), - (chunkHeader.getChunkType() & 0x3F) == MetaMarker.CHUNK_HEADER); - ByteBuffer pageData = reader.readPage(pageHeader, chunkHeader.getCompressionType()); - pageVisitor.onPage(pageHeader, pageData, chunkHeader); + currChunkHeader.getDataType(), + (currChunkHeader.getChunkType() & 0x3F) == MetaMarker.CHUNK_HEADER); + ByteBuffer pageData = reader.readPage(pageHeader, currChunkHeader.getCompressionType()); + pageVisitor.onPage(pageHeader, pageData, currChunkHeader); dataSize -= pageHeader.getSerializedPageSize(); } } @@ -123,26 +129,38 @@ public void scanTsFile(File tsFile) { switch (marker) { case MetaMarker.ONLY_ONE_PAGE_CHUNK_HEADER: currChunkOnePage = true; + currDeviceAligned = false; + isTimeChunk = false; onChunk(this::onNonAlignedPage); break; case MetaMarker.CHUNK_HEADER: currChunkOnePage = false; + currDeviceAligned = false; + isTimeChunk = false; onChunk(this::onNonAlignedPage); break; case MetaMarker.ONLY_ONE_PAGE_TIME_CHUNK_HEADER: currChunkOnePage = true; + currDeviceAligned = true; + isTimeChunk = true; onChunk(this::onTimePage); break; case MetaMarker.TIME_CHUNK_HEADER: currChunkOnePage = false; + currDeviceAligned = true; + isTimeChunk = true; onChunk(this::onTimePage); break; case MetaMarker.ONLY_ONE_PAGE_VALUE_CHUNK_HEADER: currChunkOnePage = true; + currDeviceAligned = true; + isTimeChunk = false; onChunk(this::onValuePage); break; case MetaMarker.VALUE_CHUNK_HEADER: currChunkOnePage = false; + currDeviceAligned = true; + isTimeChunk = false; onChunk(this::onValuePage); break; case MetaMarker.CHUNK_GROUP_HEADER: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/utils/TsFileStatisticScan.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/utils/TsFileStatisticScan.java new file mode 100644 index 000000000000..094f12dcc846 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/utils/TsFileStatisticScan.java @@ -0,0 +1,266 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.tools.utils; + +import org.apache.tsfile.common.conf.TSFileDescriptor; +import org.apache.tsfile.encoding.decoder.Decoder; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.header.ChunkHeader; +import org.apache.tsfile.file.header.PageHeader; +import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.file.metadata.statistics.Statistics; +import org.apache.tsfile.read.common.BatchData; +import org.apache.tsfile.read.reader.page.AlignedPageReader; +import org.apache.tsfile.read.reader.page.PageReader; +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.Pair; +import org.apache.tsfile.utils.TsPrimitiveType; + +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Collections; +import java.util.EnumMap; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class TsFileStatisticScan extends TsFileSequenceScan { + + // (deviceId, measurementId) -> data type + private final Map, TSDataType> seriesDataTypeMap = new HashMap<>(); + private final Map dataTypeSizeMap = new EnumMap<>(TSDataType.class); + private final Map dataTypePointMap = new EnumMap<>(TSDataType.class); + private final Map dataTypeChunkMap = new EnumMap<>(TSDataType.class); + private final List distinctBinaryValueNumInChunks = new LinkedList<>(); + // the number of Int64 chunks that can be represented by Int32 + private int overPrecisedInt64ChunkNum; + // the number of Int64 chunks that cannot be represented by Int32 + private int justPrecisedInt64ChunkNum; + // the number of Int64 chunks whose range can be represented by Int32 + private int smallRangeInt64ChunkNum; + // the number of Int64 chunks whose range cannot be represented by Int32 + private int largeRangeInt64ChunkNum; + private boolean currChunkJustPrecised; + private boolean currChunkLargeRange; + private Set distinctBinarySet = new HashSet<>(); + private PageHeader currTimePageHeader; + private ByteBuffer currTimePageBuffer; + + public static void main(String[] args) { + TsFileStatisticScan t = new TsFileStatisticScan(); + t.scanTsFile(new File(args[0])); + } + + @Override + protected void onFileEnd() throws IOException { + super.onFileEnd(); + Map seriesTypeCountMap = new EnumMap<>(TSDataType.class); + for (TSDataType type : seriesDataTypeMap.values()) { + seriesTypeCountMap.compute(type, (t, v) -> v == null ? 1 : v + 1); + } + System.out.println("Series data type count: " + seriesTypeCountMap); + System.out.println( + "Int64 series statistics: overPrecised " + + overPrecisedInt64ChunkNum + + ", justPrecised " + + justPrecisedInt64ChunkNum + + ", smallRange " + + smallRangeInt64ChunkNum + + ", largeRange " + + largeRangeInt64ChunkNum); + System.out.println("data type -> size: " + dataTypeSizeMap); + System.out.println("data type -> point count: " + dataTypePointMap); + System.out.println("data type -> chunk count: " + dataTypeChunkMap); + System.out.println( + "average distinct binary value num: " + + distinctBinaryValueNumInChunks.stream().mapToInt(i -> i).average().orElse(0.0)); + } + + @Override + protected void onChunk(PageVisitor pageVisitor) throws IOException { + currChunkJustPrecised = false; + currChunkLargeRange = false; + + super.onChunk(pageVisitor); + if (!isTimeChunk) { + seriesDataTypeMap.computeIfAbsent(currTimeseriesID, cid -> currChunkHeader.getDataType()); + dataTypeSizeMap.compute( + currChunkHeader.getDataType(), + (type, size) -> + size == null + ? currChunkHeader.getSerializedSize() + currChunkHeader.getDataSize() + : size + currChunkHeader.getSerializedSize() + currChunkHeader.getDataSize()); + dataTypeChunkMap.compute( + currChunkHeader.getDataType(), (type, size) -> size == null ? 1 : size + 1); + } + if (currChunkHeader.getDataType() == TSDataType.INT64) { + if (currChunkJustPrecised) { + justPrecisedInt64ChunkNum++; + } else { + overPrecisedInt64ChunkNum++; + } + + if (currChunkLargeRange) { + largeRangeInt64ChunkNum++; + } else { + smallRangeInt64ChunkNum++; + } + } else if (currChunkHeader.getDataType() == TSDataType.TEXT + || currChunkHeader.getDataType() == TSDataType.STRING) { + distinctBinaryValueNumInChunks.add(distinctBinarySet.size()); + distinctBinarySet.clear(); + } + } + + @Override + protected void onTimePage(PageHeader pageHeader, ByteBuffer pageData, ChunkHeader chunkHeader) + throws IOException { + currTimePageHeader = pageHeader; + currTimePageBuffer = pageData; + } + + @Override + protected void onValuePage(PageHeader pageHeader, ByteBuffer pageData, ChunkHeader chunkHeader) + throws IOException { + TSDataType dataType = chunkHeader.getDataType(); + if (dataType == TSDataType.INT64) { + Statistics statistics = (Statistics) pageHeader.getStatistics(); + Long minValue = statistics.getMinValue(); + Long maxValue = statistics.getMaxValue(); + if (minValue < Integer.MIN_VALUE || maxValue > Integer.MAX_VALUE) { + currChunkJustPrecised = true; + } + if (maxValue - minValue > Integer.MAX_VALUE) { + currChunkLargeRange = true; + } + } else if (dataType == TSDataType.TEXT || dataType == TSDataType.STRING) { + AlignedPageReader pageReader = + new AlignedPageReader( + currTimePageHeader, + currTimePageBuffer, + Decoder.getDecoderByType( + TSEncoding.valueOf(TSFileDescriptor.getInstance().getConfig().getTimeEncoder()), + TSDataType.INT64), + Collections.singletonList(pageHeader), + Collections.singletonList(pageData), + Collections.singletonList(dataType), + Collections.singletonList( + Decoder.getDecoderByType(chunkHeader.getEncodingType(), dataType)), + null); + BatchData batchData = pageReader.getAllSatisfiedPageData(); + while (batchData.hasCurrent()) { + TsPrimitiveType[] vector = batchData.getVector(); + if (vector[0] != null) { + distinctBinarySet.add(vector[0].getBinary()); + } + batchData.next(); + } + } + dataTypePointMap.compute( + currChunkHeader.getDataType(), + (type, size) -> + size == null + ? pageHeader.getStatistics().getCount() + : size + pageHeader.getStatistics().getCount()); + } + + @Override + protected void onNonAlignedPage( + PageHeader pageHeader, ByteBuffer pageData, ChunkHeader chunkHeader) throws IOException { + TSDataType dataType = chunkHeader.getDataType(); + if (pageHeader.getStatistics() != null) { + if (dataType == TSDataType.INT64) { + Statistics statistics = (Statistics) pageHeader.getStatistics(); + Long minValue = statistics.getMinValue(); + Long maxValue = statistics.getMaxValue(); + if (minValue < Integer.MIN_VALUE || maxValue > Integer.MAX_VALUE) { + currChunkJustPrecised = true; + } + if (maxValue - minValue > Integer.MAX_VALUE) { + currChunkLargeRange = true; + } + } else if (dataType == TSDataType.TEXT || dataType == TSDataType.STRING) { + PageReader pageReader = + new PageReader( + pageHeader, + pageData, + chunkHeader.getDataType(), + Decoder.getDecoderByType(chunkHeader.getEncodingType(), chunkHeader.getDataType()), + Decoder.getDecoderByType( + TSEncoding.valueOf(TSFileDescriptor.getInstance().getConfig().getTimeEncoder()), + TSDataType.INT64)); + BatchData allSatisfiedPageData = pageReader.getAllSatisfiedPageData(true); + while (allSatisfiedPageData.hasCurrent()) { + distinctBinarySet.add(allSatisfiedPageData.getBinary()); + allSatisfiedPageData.next(); + } + } + dataTypePointMap.compute( + currChunkHeader.getDataType(), + (type, size) -> + size == null + ? pageHeader.getStatistics().getCount() + : size + pageHeader.getStatistics().getCount()); + } else { + PageReader pageReader = + new PageReader( + pageHeader, + pageData, + chunkHeader.getDataType(), + Decoder.getDecoderByType(chunkHeader.getEncodingType(), chunkHeader.getDataType()), + Decoder.getDecoderByType( + TSEncoding.valueOf(TSFileDescriptor.getInstance().getConfig().getTimeEncoder()), + TSDataType.INT64)); + BatchData allSatisfiedPageData = pageReader.getAllSatisfiedPageData(true); + long minLongValue = Long.MAX_VALUE; + long maxLongValue = Long.MIN_VALUE; + int cnt = 0; + while (allSatisfiedPageData.hasCurrent()) { + if (dataType == TSDataType.INT64) { + long val = (long) allSatisfiedPageData.currentValue(); + if (val < Integer.MIN_VALUE || val > Integer.MAX_VALUE) { + currChunkJustPrecised = true; + } + minLongValue = Math.min(minLongValue, val); + maxLongValue = Math.max(maxLongValue, val); + } else if (dataType == TSDataType.TEXT || dataType == TSDataType.STRING) { + distinctBinarySet.add(allSatisfiedPageData.getBinary()); + } + cnt++; + allSatisfiedPageData.next(); + } + int finalCnt = cnt; + dataTypePointMap.compute( + currChunkHeader.getDataType(), (type, size) -> size == null ? finalCnt : size + finalCnt); + if (maxLongValue - minLongValue > Integer.MAX_VALUE) { + currChunkLargeRange = true; + } + } + } + + @Override + protected void onException(Throwable t) { + t.printStackTrace(); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ErrorHandlingUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ErrorHandlingUtils.java index 247a0a3fe828..57527c85af86 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ErrorHandlingUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ErrorHandlingUtils.java @@ -106,7 +106,11 @@ public static TSStatus onQueryException(Exception e, String operation, TSStatusC || status.getCode() == TSStatusCode.ILLEGAL_PATH.getStatusCode() || status.getCode() == TSStatusCode.NUMERIC_VALUE_OUT_OF_RANGE.getStatusCode() || status.getCode() == TSStatusCode.DIVISION_BY_ZERO.getStatusCode() - || status.getCode() == TSStatusCode.DATE_OUT_OF_RANGE.getStatusCode()) { + || status.getCode() == TSStatusCode.DATE_OUT_OF_RANGE.getStatusCode() + || status.getCode() == TSStatusCode.TABLE_NOT_EXISTS.getStatusCode() + || status.getCode() == TSStatusCode.TABLE_ALREADY_EXISTS.getStatusCode() + || status.getCode() == TSStatusCode.COLUMN_NOT_EXISTS.getStatusCode() + || status.getCode() == TSStatusCode.COLUMN_ALREADY_EXISTS.getStatusCode()) { LOGGER.info(message); } else { LOGGER.warn(message, e); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ModificationUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ModificationUtils.java index 599662076480..f9a6d28e2b98 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ModificationUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ModificationUtils.java @@ -25,6 +25,7 @@ import org.apache.iotdb.db.storageengine.dataregion.modification.TableDeletionEntry; import org.apache.iotdb.db.storageengine.dataregion.modification.TreeDeletionEntry; +import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; import org.apache.tsfile.file.metadata.AlignedChunkMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; import org.apache.tsfile.file.metadata.IDeviceID; @@ -101,7 +102,7 @@ public static void modifyAlignedChunkMetaData( } private static void modifyValueColumns( - AlignedChunkMetadata metaData, List> valueColumnsModifications) { + AbstractAlignedChunkMetadata metaData, List> valueColumnsModifications) { List valueChunkMetadataList = metaData.getValueChunkMetadataList(); // deal with each sub sensor for (int j = 0; j < valueChunkMetadataList.size(); j++) { @@ -116,7 +117,7 @@ private static void modifyValueColumns( } private static boolean areAllValueColumnsDeleted( - AlignedChunkMetadata alignedChunkMetadata, boolean modified) { + AbstractAlignedChunkMetadata alignedChunkMetadata, boolean modified) { // the whole aligned path need to be removed, only set to be true if all the sub sensors // are deleted and ignoreAllNullRows is true @@ -156,11 +157,11 @@ private static boolean areAllValueColumnsDeleted( } public static void modifyAlignedChunkMetaData( - List chunkMetaData, + List chunkMetaData, List timeColumnModifications, List> valueColumnsModifications, boolean ignoreAllNullRows) { - for (AlignedChunkMetadata metaData : chunkMetaData) { + for (AbstractAlignedChunkMetadata metaData : chunkMetaData) { IChunkMetadata timeColumnChunkMetadata = metaData.getTimeChunkMetadata(); for (ModEntry modification : timeColumnModifications) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TimestampPrecisionUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TimestampPrecisionUtils.java index 768d0030e96d..676a50237824 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TimestampPrecisionUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TimestampPrecisionUtils.java @@ -70,7 +70,7 @@ public static void checkTimestampPrecision(long time) { } switch (TIMESTAMP_PRECISION) { case "ms": - if (time > 10_000_000_000_000L) { + if (time > 10_000_000_000_000L || time < -10_000_000_000_000L) { throw new SemanticException( String.format( "Current system timestamp precision is %s, " @@ -79,7 +79,7 @@ public static void checkTimestampPrecision(long time) { } break; case "us": - if (time > 10_000_000_000_000_000L) { + if (time > 10_000_000_000_000_000L || time < -10_000_000_000_000L) { throw new SemanticException( String.format( "Current system timestamp precision is %s, " diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/constant/SqlConstant.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/constant/SqlConstant.java index cad8f229c4eb..625af8935614 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/constant/SqlConstant.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/constant/SqlConstant.java @@ -36,7 +36,7 @@ protected SqlConstant() { public static final String ROOT = "root"; - public static final String ROOT_DOT = "root."; + public static final String TREE_MODEL_DATABASE_PREFIX = "root."; public static final String QUOTE = "'"; public static final String DQUOTE = "\""; diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/Utils.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/Utils.java index 0c6eac0dbed1..327d4a34c39e 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/Utils.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/Utils.java @@ -41,6 +41,7 @@ public static List createMockTsBlocks(int numOfTsBlocks, long mockTsBlo for (int i = 0; i < numOfTsBlocks; i++) { TsBlock mockTsBlock = Mockito.mock(TsBlock.class); Mockito.when(mockTsBlock.getRetainedSizeInBytes()).thenReturn(mockTsBlockSize); + Mockito.when(mockTsBlock.getSizeInBytes()).thenReturn(mockTsBlockSize); mockTsBlocks.add(mockTsBlock); } @@ -50,6 +51,7 @@ public static List createMockTsBlocks(int numOfTsBlocks, long mockTsBlo public static TsBlock createMockTsBlock(long mockTsBlockSize) { TsBlock mockTsBlock = Mockito.mock(TsBlock.class); Mockito.when(mockTsBlock.getRetainedSizeInBytes()).thenReturn(mockTsBlockSize); + Mockito.when(mockTsBlock.getSizeInBytes()).thenReturn(mockTsBlockSize); return mockTsBlock; } @@ -144,6 +146,7 @@ public static TsBlockSerde createMockTsBlockSerde(long mockTsBlockSize) { TsBlockSerde mockTsBlockSerde = Mockito.mock(TsBlockSerde.class); TsBlock mockTsBlock = Mockito.mock(TsBlock.class); Mockito.when(mockTsBlock.getRetainedSizeInBytes()).thenReturn(mockTsBlockSize); + Mockito.when(mockTsBlock.getSizeInBytes()).thenReturn(mockTsBlockSize); Mockito.when(mockTsBlockSerde.deserialize(Mockito.any(ByteBuffer.class))) .thenReturn(mockTsBlock); return mockTsBlockSerde; diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/MergeTreeSortOperatorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/MergeTreeSortOperatorTest.java index b479a5787907..6041f37bcec4 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/MergeTreeSortOperatorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/MergeTreeSortOperatorTest.java @@ -55,6 +55,7 @@ import org.apache.iotdb.db.storageengine.dataregion.read.reader.series.SeriesReaderTestUtil; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.iotdb.db.utils.datastructure.SortKey; +import org.apache.iotdb.isession.SessionConfig; import com.google.common.collect.ImmutableList; import com.google.common.util.concurrent.ListenableFuture; @@ -1844,6 +1845,11 @@ public IClientSession.SqlDialect getSQLDialect() { return IClientSession.SqlDialect.TREE; } + @Override + public String getUser() { + return SessionConfig.DEFAULT_USER; + } + @Override public void start() {} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/OperatorMemoryTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/OperatorMemoryTest.java index 1bba28545bf3..e7f662ab573d 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/OperatorMemoryTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/OperatorMemoryTest.java @@ -253,6 +253,7 @@ public void pipelineExchangeOperatorTest() { public void lastCacheScanOperatorTest() { TsBlock tsBlock = Mockito.mock(TsBlock.class); Mockito.when(tsBlock.getRetainedSizeInBytes()).thenReturn(1024L); + Mockito.when(tsBlock.getSizeInBytes()).thenReturn(1024L); LastCacheScanOperator lastCacheScanOperator = new LastCacheScanOperator(null, null, tsBlock); assertEquals(1024, lastCacheScanOperator.calculateMaxPeekMemory()); @@ -383,6 +384,7 @@ public void lastQueryOperatorTest() { public void lastQuerySortOperatorTest() { TsBlock tsBlock = Mockito.mock(TsBlock.class); Mockito.when(tsBlock.getRetainedSizeInBytes()).thenReturn(16 * 1024L); + Mockito.when(tsBlock.getSizeInBytes()).thenReturn(16 * 1024L); Mockito.when(tsBlock.getPositionCount()).thenReturn(16); List children = new ArrayList<>(4); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGeneratorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGeneratorTest.java index 06b51954d533..30024d5e59c2 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGeneratorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGeneratorTest.java @@ -783,6 +783,9 @@ public void testComplexGrantRevoke() { // 1. test complex privilege on single path :"root.**" Set allPriv = new HashSet<>(); for (PrivilegeType type : PrivilegeType.values()) { + if (type.isRelationalPrivilege()) { + continue; + } allPriv.add(type.toString()); } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ShowQueriesTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ShowQueriesTest.java index bdf44401aade..63b7b783d747 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ShowQueriesTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ShowQueriesTest.java @@ -32,6 +32,7 @@ import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.QUERY_ID_TABLE_MODEL; import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.START_TIME_TABLE_MODEL; import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.STATEMENT; +import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.USER; import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanAssert.assertPlan; import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.collect; import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.exchange; @@ -60,7 +61,8 @@ public void testNormal() { START_TIME_TABLE_MODEL, DATA_NODE_ID_TABLE_MODEL, ELAPSED_TIME_TABLE_MODEL, - STATEMENT.toLowerCase(Locale.ENGLISH))))); + STATEMENT.toLowerCase(Locale.ENGLISH), + USER.toLowerCase(Locale.ENGLISH))))); // - Exchange // Output - Collect - Exchange @@ -137,6 +139,7 @@ public void testNonSelectAll() { START_TIME_TABLE_MODEL, DATA_NODE_ID_TABLE_MODEL, ELAPSED_TIME_TABLE_MODEL, - STATEMENT.toLowerCase(Locale.ENGLISH)))))); + STATEMENT.toLowerCase(Locale.ENGLISH), + USER.toLowerCase(Locale.ENGLISH)))))); } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/FastCompactionPerformerWithInconsistentCompressionTypeAndEncodingTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/FastCompactionPerformerWithInconsistentCompressionTypeAndEncodingTest.java index 55cb8625da37..f62480c55291 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/FastCompactionPerformerWithInconsistentCompressionTypeAndEncodingTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/FastCompactionPerformerWithInconsistentCompressionTypeAndEncodingTest.java @@ -32,7 +32,7 @@ import org.apache.tsfile.file.MetaMarker; import org.apache.tsfile.file.header.ChunkHeader; import org.apache.tsfile.file.header.PageHeader; -import org.apache.tsfile.file.metadata.AlignedChunkMetadata; +import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; import org.apache.tsfile.file.metadata.IDeviceID; @@ -622,9 +622,9 @@ private void validateSingleTsFileWithAlignedSeries(TsFileSequenceReader reader) throws IOException { Map compressionTypeMap = new HashMap<>(); for (IDeviceID device : reader.getAllDevices()) { - List alignedChunkMetadataList = + List alignedChunkMetadataList = reader.getAlignedChunkMetadata(device, true); - for (AlignedChunkMetadata alignedChunkMetadata : alignedChunkMetadataList) { + for (AbstractAlignedChunkMetadata alignedChunkMetadata : alignedChunkMetadataList) { IChunkMetadata timeChunkMetadata = alignedChunkMetadata.getTimeChunkMetadata(); List valueChunkMetadataList = alignedChunkMetadata.getValueChunkMetadataList(); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/FastInnerCompactionPerformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/FastInnerCompactionPerformerTest.java index 8c9088f5cebc..f2537e9bfddf 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/FastInnerCompactionPerformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/FastInnerCompactionPerformerTest.java @@ -39,7 +39,7 @@ import org.apache.tsfile.common.conf.TSFileDescriptor; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.exception.write.WriteProcessException; -import org.apache.tsfile.file.metadata.AlignedChunkMetadata; +import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; import org.apache.tsfile.file.metadata.IDeviceID; @@ -2047,10 +2047,10 @@ public void testMergeAlignedSeriesTimeValuePairFromDifferentFiles() TsFileResource targetResource = tsFileManager.getTsFileList(false).get(0); try (TsFileSequenceReader reader = new TsFileSequenceReader(targetResource.getTsFilePath())) { - List chunkMetadataList = + List chunkMetadataList = reader.getAlignedChunkMetadata( IDeviceID.Factory.DEFAULT_FACTORY.create("root.testsg.d1"), true); - for (AlignedChunkMetadata alignedChunkMetadata : chunkMetadataList) { + for (AbstractAlignedChunkMetadata alignedChunkMetadata : chunkMetadataList) { ChunkMetadata timeChunkMetadata = (ChunkMetadata) alignedChunkMetadata.getTimeChunkMetadata(); Chunk timeChunk = reader.readMemChunk(timeChunkMetadata); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairUnsortedFileCompactionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairUnsortedFileCompactionTest.java index a9c559a00a52..f2e39d8f8668 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairUnsortedFileCompactionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairUnsortedFileCompactionTest.java @@ -46,7 +46,7 @@ import org.apache.iotdb.db.storageengine.dataregion.utils.TsFileResourceUtils; import org.apache.tsfile.exception.write.WriteProcessException; -import org.apache.tsfile.file.metadata.AlignedChunkMetadata; +import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; import org.apache.tsfile.file.metadata.IDeviceID; @@ -682,10 +682,10 @@ public void testMergeAlignedSeriesPointWithSameTimestamp() throws IOException { Assert.assertTrue(task.start()); TsFileResource target = tsFileManager.getTsFileList(false).get(0); try (TsFileSequenceReader reader = new TsFileSequenceReader(target.getTsFilePath())) { - List chunkMetadataList = + List chunkMetadataList = reader.getAlignedChunkMetadata( IDeviceID.Factory.DEFAULT_FACTORY.create("root.testsg.d1"), true); - for (AlignedChunkMetadata alignedChunkMetadata : chunkMetadataList) { + for (AbstractAlignedChunkMetadata alignedChunkMetadata : chunkMetadataList) { ChunkMetadata timeChunkMetadata = (ChunkMetadata) alignedChunkMetadata.getTimeChunkMetadata(); Chunk timeChunk = reader.readMemChunk(timeChunkMetadata); @@ -796,7 +796,7 @@ public void testSplitChunk() throws IOException { Assert.assertTrue(task.start()); TsFileResource target = tsFileManager.getTsFileList(false).get(0); try (TsFileSequenceReader reader = new TsFileSequenceReader(target.getTsFilePath())) { - List chunkMetadataList = + List chunkMetadataList = reader.getAlignedChunkMetadata( IDeviceID.Factory.DEFAULT_FACTORY.create("root.testsg.d1"), true); Assert.assertEquals(3, chunkMetadataList.size()); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/CompactionTableModelTestFileWriter.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/CompactionTableModelTestFileWriter.java index 0e5fa29a41af..9bc597083739 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/CompactionTableModelTestFileWriter.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/CompactionTableModelTestFileWriter.java @@ -54,7 +54,7 @@ public void registerTableSchema(String tableName, List idColumnNames) { for (String idColumnName : idColumnNames) { measurementSchemas.add( new MeasurementSchema( - idColumnName, TSDataType.TEXT, TSEncoding.PLAIN, CompressionType.UNCOMPRESSED)); + idColumnName, TSDataType.STRING, TSEncoding.PLAIN, CompressionType.UNCOMPRESSED)); columnTypes.add(ColumnCategory.TAG); } tableSchema.merge(new TableSchema(tableName, measurementSchemas, columnTypes)); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/BatchCompactionUtilsTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/BatchCompactionUtilsTest.java index ef804437361b..c257ee4ec203 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/BatchCompactionUtilsTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/BatchCompactionUtilsTest.java @@ -34,6 +34,7 @@ import org.apache.tsfile.exception.write.PageException; import org.apache.tsfile.exception.write.WriteProcessException; import org.apache.tsfile.file.header.PageHeader; +import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; import org.apache.tsfile.file.metadata.AlignedChunkMetadata; import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; @@ -107,7 +108,7 @@ public void testBatchCompactionPointReader() throws IOException { true); try (TsFileSequenceReader reader = new TsFileSequenceReader(seqResource1.getTsFile().getAbsolutePath())) { - AlignedChunkMetadata alignedChunkMetadata = + AbstractAlignedChunkMetadata alignedChunkMetadata = reader .getAlignedChunkMetadata( IDeviceID.Factory.DEFAULT_FACTORY.create("root.testsg.d0"), true) @@ -311,14 +312,14 @@ public void testMapAlignedChunkMetadata1() { new MeasurementSchema("s1", TSDataType.INT32), new MeasurementSchema("s2", TSDataType.INT32), new MeasurementSchema("s4", TSDataType.INT32)); - AlignedChunkMetadata newAlignedChunkMetadata = + AbstractAlignedChunkMetadata newAlignedChunkMetadata = AlignedSeriesBatchCompactionUtils.fillAlignedChunkMetadataBySchemaList( alignedChunkMetadata, measurementSchemas); Assert.assertEquals( + Arrays.asList("s0", "s1", "s2", "s4"), newAlignedChunkMetadata.getValueChunkMetadataList().stream() .map(IChunkMetadata::getMeasurementUid) - .collect(Collectors.toList()), - Arrays.asList("s0", "s1", "s2", "s4")); + .collect(Collectors.toList())); } @Test @@ -333,14 +334,14 @@ public void testMapAlignedChunkMetadata2() { Arrays.asList( new MeasurementSchema("s0", TSDataType.INT32), new MeasurementSchema("s4", TSDataType.INT32)); - AlignedChunkMetadata newAlignedChunkMetadata = + AbstractAlignedChunkMetadata newAlignedChunkMetadata = AlignedSeriesBatchCompactionUtils.fillAlignedChunkMetadataBySchemaList( alignedChunkMetadata, measurementSchemas); Assert.assertEquals( + Arrays.asList(null, "s4"), newAlignedChunkMetadata.getValueChunkMetadataList().stream() .map(chunkMetadata -> chunkMetadata == null ? null : chunkMetadata.getMeasurementUid()) - .collect(Collectors.toList()), - Arrays.asList(null, "s4")); + .collect(Collectors.toList())); } @Test @@ -368,23 +369,23 @@ public void testMapAlignedChunkMetadata3() { new MeasurementSchema("s2", TSDataType.INT32), new MeasurementSchema("s3", TSDataType.INT32), new MeasurementSchema("s4", TSDataType.INT32)); - AlignedChunkMetadata newAlignedChunkMetadata1 = + AbstractAlignedChunkMetadata newAlignedChunkMetadata1 = AlignedSeriesBatchCompactionUtils.fillAlignedChunkMetadataBySchemaList( alignedChunkMetadata1, measurementSchemas); Assert.assertEquals( + Arrays.asList("s0", "s1", "s2", null, null), newAlignedChunkMetadata1.getValueChunkMetadataList().stream() .map(chunkMetadata -> chunkMetadata == null ? null : chunkMetadata.getMeasurementUid()) - .collect(Collectors.toList()), - Arrays.asList("s0", "s1", "s2", null, null)); + .collect(Collectors.toList())); - AlignedChunkMetadata newAlignedChunkMetadata2 = + AbstractAlignedChunkMetadata newAlignedChunkMetadata2 = AlignedSeriesBatchCompactionUtils.fillAlignedChunkMetadataBySchemaList( alignedChunkMetadata2, measurementSchemas); Assert.assertEquals( + Arrays.asList(null, null, null, "s3", "s4"), newAlignedChunkMetadata2.getValueChunkMetadataList().stream() .map(chunkMetadata -> chunkMetadata == null ? null : chunkMetadata.getMeasurementUid()) - .collect(Collectors.toList()), - Arrays.asList(null, null, null, "s3", "s4")); + .collect(Collectors.toList())); } @Test @@ -411,22 +412,22 @@ public void testMapAlignedChunkMetadata4() { Arrays.asList( new MeasurementSchema("s2", TSDataType.INT32), new MeasurementSchema("s4", TSDataType.INT32)); - AlignedChunkMetadata newAlignedChunkMetadata1 = + AbstractAlignedChunkMetadata newAlignedChunkMetadata1 = AlignedSeriesBatchCompactionUtils.fillAlignedChunkMetadataBySchemaList( alignedChunkMetadata1, measurementSchemas); Assert.assertEquals( + Arrays.asList("s2", null), newAlignedChunkMetadata1.getValueChunkMetadataList().stream() .map(chunkMetadata -> chunkMetadata == null ? null : chunkMetadata.getMeasurementUid()) - .collect(Collectors.toList()), - Arrays.asList("s2", null)); + .collect(Collectors.toList())); - AlignedChunkMetadata newAlignedChunkMetadata2 = + AbstractAlignedChunkMetadata newAlignedChunkMetadata2 = AlignedSeriesBatchCompactionUtils.fillAlignedChunkMetadataBySchemaList( alignedChunkMetadata2, measurementSchemas); Assert.assertEquals( + Arrays.asList(null, "s4"), newAlignedChunkMetadata2.getValueChunkMetadataList().stream() .map(chunkMetadata -> chunkMetadata == null ? null : chunkMetadata.getMeasurementUid()) - .collect(Collectors.toList()), - Arrays.asList(null, "s4")); + .collect(Collectors.toList())); } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/TimestampPrecisionUtilsTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/TimestampPrecisionUtilsTest.java index 87585cc663f9..7c7893e49af8 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/TimestampPrecisionUtilsTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/TimestampPrecisionUtilsTest.java @@ -45,6 +45,11 @@ public void testCheckIllegalMsTimestampPrecision() { fail(); } catch (Exception ignored) { } + try { + TimestampPrecisionUtils.checkTimestampPrecision(-1694689856546000L); + fail(); + } catch (Exception ignored) { + } } @Test @@ -68,6 +73,11 @@ public void testCheckIllegalUsTimestampPrecision() { fail(); } catch (Exception ignored) { } + try { + TimestampPrecisionUtils.checkTimestampPrecision(-1694689856546000000L); + fail(); + } catch (Exception ignored) { + } } @Test diff --git a/iotdb-core/metrics/core/pom.xml b/iotdb-core/metrics/core/pom.xml index f68d953adb56..9c763dd8aa3a 100644 --- a/iotdb-core/metrics/core/pom.xml +++ b/iotdb-core/metrics/core/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-metrics - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT metrics-core IoTDB: Core: Metrics: API Impl @@ -32,7 +32,7 @@ org.apache.iotdb metrics-interface - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT io.micrometer diff --git a/iotdb-core/metrics/interface/pom.xml b/iotdb-core/metrics/interface/pom.xml index 695fbc879d15..8708e74d4b44 100644 --- a/iotdb-core/metrics/interface/pom.xml +++ b/iotdb-core/metrics/interface/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-metrics - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT metrics-interface IoTDB: Core: Metrics: Metrics API @@ -33,17 +33,17 @@ org.apache.iotdb iotdb-session - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.apache.iotdb isession - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.apache.iotdb service-rpc - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.apache.tsfile diff --git a/iotdb-core/metrics/pom.xml b/iotdb-core/metrics/pom.xml index 2b62015bb9fd..7ea896c0f2d4 100644 --- a/iotdb-core/metrics/pom.xml +++ b/iotdb-core/metrics/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-core - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT iotdb-metrics pom diff --git a/iotdb-core/node-commons/pom.xml b/iotdb-core/node-commons/pom.xml index 8700c4de34b6..ccc104f539a1 100644 --- a/iotdb-core/node-commons/pom.xml +++ b/iotdb-core/node-commons/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-core - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT node-commons IoTDB: Core: Node Commons @@ -38,7 +38,7 @@ org.apache.iotdb service-rpc - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.apache.tsfile @@ -48,37 +48,37 @@ org.apache.iotdb udf-api - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.apache.iotdb trigger-api - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.apache.iotdb pipe-api - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.apache.iotdb iotdb-thrift-confignode - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.apache.iotdb iotdb-thrift - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.apache.iotdb iotdb-thrift-commons - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.apache.iotdb iotdb-thrift-consensus - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.apache.tsfile @@ -88,12 +88,12 @@ org.apache.iotdb metrics-interface - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.apache.iotdb metrics-core - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT io.dropwizard.metrics @@ -193,7 +193,7 @@ org.apache.iotdb iotdb-thrift-ainode - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT compile diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template index 20a9677ce443..4f13c3b8e20b 100644 --- a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template +++ b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template @@ -1730,10 +1730,10 @@ continuous_query_min_every_interval_in_ms=1000 pipe_lib_dir=ext/pipe # The maximum number of threads that can be used to execute the pipe subtasks in PipeSubtaskExecutor. -# The actual value will be min(pipe_subtask_executor_max_thread_num, max(1, CPU core number / 2)). +# When <= 0, use max(5, CPU core number). # effectiveMode: restart # Datatype: int -pipe_subtask_executor_max_thread_num=5 +pipe_subtask_executor_max_thread_num=0 # The connection timeout (in milliseconds) for the thrift client. # effectiveMode: restart @@ -1742,14 +1742,16 @@ pipe_sink_timeout_ms=900000 # The maximum number of selectors that can be used in the sink. # Recommend to set this value to less than or equal to pipe_sink_max_client_number. +# When <= 0, use max(4, CPU core number). # effectiveMode: restart # Datatype: int -pipe_sink_selector_number=4 +pipe_sink_selector_number=0 # The maximum number of clients that can be used in the sink. +# When <= 0, use max(16, CPU core number). # effectiveMode: restart # Datatype: int -pipe_sink_max_client_number=16 +pipe_sink_max_client_number=0 # Whether to enable receiving pipe data through air gap. # The receiver can only return 0 or 1 in tcp mode to indicate whether the data is received successfully. diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/authorizer/BasicAuthorizer.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/authorizer/BasicAuthorizer.java index 1867a207d088..d4fe42d8055d 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/authorizer/BasicAuthorizer.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/authorizer/BasicAuthorizer.java @@ -158,6 +158,17 @@ public void revokePrivilegeFromUser(String username, PrivilegeUnion union) throw userManager.revokePrivilegeFromEntity(username, union); } + @Override + public void revokeAllPrivilegeFromUser(String userName) throws AuthException { + checkAdmin(userName, "Invalid operation, administrator cannot revoke privileges"); + User user = userManager.getEntity(userName); + if (user == null) { + throw new AuthException( + TSStatusCode.USER_NOT_EXIST, String.format("User %s does not exist", userName)); + } + user.revokeAllRelationalPrivileges(); + } + @Override public void createRole(String roleName) throws AuthException { AuthUtils.validateRolename(roleName); @@ -201,6 +212,16 @@ public void revokePrivilegeFromRole(String roleName, PrivilegeUnion union) throw roleManager.revokePrivilegeFromEntity(roleName, union); } + @Override + public void revokeAllPrivilegeFromRole(String roleName) throws AuthException { + Role role = roleManager.getEntity(roleName); + if (role == null) { + throw new AuthException( + TSStatusCode.ROLE_NOT_EXIST, String.format("Role %s does not exist", roleName)); + } + role.revokeAllRelationalPrivileges(); + } + @Override public void grantRoleToUser(String roleName, String userName) throws AuthException { checkAdmin(userName, "Invalid operation, cannot grant role to administrator"); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/authorizer/IAuthorizer.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/authorizer/IAuthorizer.java index d7f8cd26770b..dbe9bf8550b5 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/authorizer/IAuthorizer.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/authorizer/IAuthorizer.java @@ -81,6 +81,8 @@ public interface IAuthorizer extends SnapshotProcessor { */ void revokePrivilegeFromUser(String userName, PrivilegeUnion union) throws AuthException; + void revokeAllPrivilegeFromUser(String userName) throws AuthException; + /** * Create a role. * @@ -115,6 +117,8 @@ public interface IAuthorizer extends SnapshotProcessor { */ void revokePrivilegeFromRole(String roleName, PrivilegeUnion union) throws AuthException; + void revokeAllPrivilegeFromRole(String roleName) throws AuthException; + /** * Grant a role to a user. * diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/entity/Role.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/entity/Role.java index 32ecdd9437af..4e5a026f4ada 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/entity/Role.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/entity/Role.java @@ -365,6 +365,14 @@ public void revokeAnyScopePrivilegeGrantOption(PrivilegeType priv) { anyScopePrivilegeGrantOptSet.remove(priv); } + public void revokeAllRelationalPrivileges() { + objectPrivilegeMap = new HashMap<>(); + sysPrivilegeSet.removeIf(PrivilegeType::forRelationalSys); + sysPriGrantOpt.removeIf(PrivilegeType::forRelationalSys); + anyScopePrivilegeSet = new HashSet<>(); + anyScopePrivilegeGrantOptSet = new HashSet<>(); + } + public void grantDBPrivilege(String dbName, PrivilegeType priv, boolean grantOption) { DatabasePrivilege databasePrivilege = getObjectPrivilege(dbName); databasePrivilege.grantDBPrivilege(priv); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java index b2eeaf25f2cb..566a37e82ebc 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java @@ -207,7 +207,7 @@ public class CommonConfig { /** The maximum number of threads that can be used to execute subtasks in PipeSubtaskExecutor. */ private int pipeSubtaskExecutorMaxThreadNum = - Math.min(5, Math.max(1, Runtime.getRuntime().availableProcessors() / 2)); + Math.max(5, Runtime.getRuntime().availableProcessors()); private int pipeNonForwardingEventsProgressReportInterval = 100; @@ -232,8 +232,10 @@ public class CommonConfig { private long pipeConnectorRetryIntervalMs = 1000L; private boolean pipeConnectorRPCThriftCompressionEnabled = false; - private int pipeAsyncConnectorSelectorNumber = 4; - private int pipeAsyncConnectorMaxClientNumber = 16; + private int pipeAsyncConnectorSelectorNumber = + Math.max(4, Runtime.getRuntime().availableProcessors()); + private int pipeAsyncConnectorMaxClientNumber = + Math.max(16, Runtime.getRuntime().availableProcessors()); private double pipeAllSinksRateLimitBytesPerSecond = -1; private int rateLimiterHotReloadCheckIntervalMs = 1000; @@ -259,7 +261,7 @@ public class CommonConfig { private long pipeMaxAllowedLinkedTsFileCount = 100; private float pipeMaxAllowedLinkedDeletedTsFileDiskUsagePercentage = 0.1F; private long pipeStuckRestartIntervalSeconds = 120; - private long pipeStuckRestartMinIntervalMs = 30 * 60 * 1000L; // 30 minutes + private long pipeStuckRestartMinIntervalMs = 5 * 60 * 1000L; // 5 minutes private int pipeMetaReportMaxLogNumPerRound = 10; private int pipeMetaReportMaxLogIntervalRounds = 36; @@ -940,10 +942,7 @@ public int getPipeSubtaskExecutorMaxThreadNum() { } public void setPipeSubtaskExecutorMaxThreadNum(int pipeSubtaskExecutorMaxThreadNum) { - this.pipeSubtaskExecutorMaxThreadNum = - Math.min( - pipeSubtaskExecutorMaxThreadNum, - Math.max(1, Runtime.getRuntime().availableProcessors() / 2)); + this.pipeSubtaskExecutorMaxThreadNum = pipeSubtaskExecutorMaxThreadNum; } public long getPipeSubtaskExecutorPendingQueueMaxBlockingTimeMs() { diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonDescriptor.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonDescriptor.java index 529d1053c422..3661feefa6ee 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonDescriptor.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonDescriptor.java @@ -306,13 +306,13 @@ private void loadPipeProps(TrimProperties properties) { "pipe_realtime_queue_poll_history_threshold", Integer.toString(config.getPipeRealTimeQueuePollHistoryThreshold())))); - config.setPipeSubtaskExecutorMaxThreadNum( + int pipeSubtaskExecutorMaxThreadNum = Integer.parseInt( properties.getProperty( "pipe_subtask_executor_max_thread_num", - Integer.toString(config.getPipeSubtaskExecutorMaxThreadNum())))); - if (config.getPipeSubtaskExecutorMaxThreadNum() <= 0) { - config.setPipeSubtaskExecutorMaxThreadNum(5); + Integer.toString(config.getPipeSubtaskExecutorMaxThreadNum()))); + if (pipeSubtaskExecutorMaxThreadNum > 0) { + config.setPipeSubtaskExecutorMaxThreadNum(pipeSubtaskExecutorMaxThreadNum); } config.setPipeSubtaskExecutorBasicCheckPointIntervalByConsumedEventCount( Integer.parseInt( @@ -405,22 +405,26 @@ private void loadPipeProps(TrimProperties properties) { properties.getProperty( "pipe_connector_rpc_thrift_compression_enabled", String.valueOf(config.isPipeConnectorRPCThriftCompressionEnabled()))))); - - config.setPipeAsyncConnectorSelectorNumber( + int pipeAsyncConnectorSelectorNumber = Integer.parseInt( Optional.ofNullable(properties.getProperty("pipe_sink_selector_number")) .orElse( properties.getProperty( "pipe_async_connector_selector_number", - String.valueOf(config.getPipeAsyncConnectorSelectorNumber()))))); - config.setPipeAsyncConnectorMaxClientNumber( + String.valueOf(config.getPipeAsyncConnectorSelectorNumber())))); + if (pipeAsyncConnectorSelectorNumber > 0) { + config.setPipeAsyncConnectorSelectorNumber(pipeAsyncConnectorSelectorNumber); + } + int pipeAsyncConnectorMaxClientNumber = Integer.parseInt( Optional.ofNullable(properties.getProperty("pipe_sink_max_client_number")) .orElse( properties.getProperty( "pipe_async_connector_max_client_number", - String.valueOf(config.getPipeAsyncConnectorMaxClientNumber()))))); - + String.valueOf(config.getPipeAsyncConnectorMaxClientNumber())))); + if (pipeAsyncConnectorMaxClientNumber > 0) { + config.setPipeAsyncConnectorMaxClientNumber(pipeAsyncConnectorMaxClientNumber); + } config.setPipeAllSinksRateLimitBytesPerSecond( Double.parseDouble( properties.getProperty( diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/connection/BlockingPendingQueue.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/connection/BlockingPendingQueue.java index 83ad7f04727f..82518bda14d9 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/connection/BlockingPendingQueue.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/connection/BlockingPendingQueue.java @@ -29,6 +29,7 @@ import java.util.concurrent.BlockingQueue; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Consumer; public abstract class BlockingPendingQueue { @@ -42,6 +43,8 @@ public abstract class BlockingPendingQueue { protected final PipeEventCounter eventCounter; + protected final AtomicBoolean isClosed = new AtomicBoolean(false); + protected BlockingPendingQueue( final BlockingQueue pendingQueue, final PipeEventCounter eventCounter) { this.pendingQueue = pendingQueue; @@ -49,6 +52,7 @@ protected BlockingPendingQueue( } public boolean waitedOffer(final E event) { + checkBeforeOffer(event); try { final boolean offered = pendingQueue.offer(event, MAX_BLOCKING_TIME_MS, TimeUnit.MILLISECONDS); @@ -64,6 +68,7 @@ public boolean waitedOffer(final E event) { } public boolean directOffer(final E event) { + checkBeforeOffer(event); final boolean offered = pendingQueue.offer(event); if (offered) { eventCounter.increaseEventCount(event); @@ -72,6 +77,7 @@ public boolean directOffer(final E event) { } public boolean put(final E event) { + checkBeforeOffer(event); try { pendingQueue.put(event); eventCounter.increaseEventCount(event); @@ -102,6 +108,7 @@ public E waitedPoll() { } public void clear() { + isClosed.set(true); pendingQueue.clear(); eventCounter.reset(); } @@ -112,6 +119,7 @@ public void forEach(final Consumer action) { } public void discardAllEvents() { + isClosed.set(true); pendingQueue.removeIf( event -> { if (event instanceof EnrichedEvent) { @@ -158,4 +166,10 @@ public int getTsFileInsertionEventCount() { public int getPipeHeartbeatEventCount() { return eventCounter.getPipeHeartbeatEventCount(); } + + protected void checkBeforeOffer(final E event) { + if (isClosed.get() && event instanceof EnrichedEvent) { + ((EnrichedEvent) event).clearReferenceCount(BlockingPendingQueue.class.getName()); + } + } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/options/PipeInclusionOptions.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/options/PipeInclusionOptions.java index a4a91f504a0a..61478ed88b16 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/options/PipeInclusionOptions.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/options/PipeInclusionOptions.java @@ -30,6 +30,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.atomic.AtomicReference; @@ -45,10 +46,16 @@ public class PipeInclusionOptions { private static final Logger LOGGER = LoggerFactory.getLogger(PipeInclusionOptions.class); - public static final PartialPath treeOnlySyncPrefix = - new PartialPath(new String[] {"schema", "timeseries"}); - public static final PartialPath tableOnlySyncPrefix = - new PartialPath(new String[] {"schema", "table"}); + public static final List treeOnlySyncPrefixes = + Collections.singletonList(new PartialPath(new String[] {"schema", "timeseries"})); + public static final List tableOnlySyncPrefixes = + Collections.unmodifiableList( + Arrays.asList( + new PartialPath(new String[] {"schema", "table"}), + new PartialPath(new String[] {"auth", "role", "grant", "table"}), + new PartialPath(new String[] {"auth", "role", "revoke", "table"}), + new PartialPath(new String[] {"auth", "user", "grant", "table"}), + new PartialPath(new String[] {"auth", "user", "revoke", "table"}))); public static final String ALL = "all"; private static final Set OPTIONS = new HashSet<>(); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/column/ColumnHeaderConstant.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/column/ColumnHeaderConstant.java index 2bafd0e5678e..7b7d01ca55f4 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/column/ColumnHeaderConstant.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/column/ColumnHeaderConstant.java @@ -213,6 +213,36 @@ private ColumnHeaderConstant() { public static final String SCHEMA_REGION_GROUP_NUM_TABLE_MODEL = "schema_region_group_num"; public static final String DATA_REGION_GROUP_NUM_TABLE_MODEL = "data_region_group_num"; + public static final String REGION_ID_TABLE_MODEL = "region_id"; + public static final String DATANODE_ID_TABLE_MODEL = "datanode_id"; + public static final String SERIES_SLOT_NUM_TABLE_MODEL = "series_slot_num"; + public static final String TIME_SLOT_NUM_TABLE_MODEL = "time_slot_num"; + public static final String RPC_ADDRESS_TABLE_MODEL = "rpc_address"; + public static final String RPC_PORT_TABLE_MODEL = "rpc_port"; + public static final String INTERNAL_ADDRESS_TABLE_MODEL = "internal_address"; + public static final String CREATE_TIME_TABLE_MODEL = "create_time"; + public static final String TS_FILE_SIZE_BYTES_TABLE_MODEL = "tsfile_size_bytes"; + + public static final String CREATION_TIME_TABLE_MODEL = "creation_time"; + public static final String PIPE_SOURCE_TABLE_MODEL = "pipe_source"; + public static final String PIPE_PROCESSOR_TABLE_MODEL = "pipe_processor"; + public static final String PIPE_SINK_TABLE_MODEL = "pipe_sink"; + public static final String EXCEPTION_MESSAGE_TABLE_MODEL = "exception_message"; + public static final String REMAINING_EVENT_COUNT_TABLE_MODEL = "remaining_event_count"; + public static final String ESTIMATED_REMAINING_SECONDS_TABLE_MODEL = + "estimated_remaining_seconds"; + + public static final String PLUGIN_NAME_TABLE_MODEL = "plugin_name"; + public static final String PLUGIN_TYPE_TABLE_MODEL = "plugin_type"; + public static final String CLASS_NAME_TABLE_MODEL = "class_name"; + public static final String PLUGIN_JAR_TABLE_MODEL = "plugin_jar"; + + public static final String TOPIC_NAME_TABLE_MODEL = "topic_name"; + public static final String TOPIC_CONFIGS_TABLE_MODEL = "topic_configs"; + + public static final String CONSUMER_GROUP_NAME_TABLE_MODEL = "consumer_group_name"; + public static final String SUBSCRIBED_CONSUMERS_TABLE_MODEL = "subscribed_consumers"; + // column names for show space quota public static final String QUOTA_TYPE = "QuotaType"; public static final String LIMIT = "Limit"; diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/InformationSchema.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/InformationSchema.java index f2f4e548d4ce..6fdf456ab76a 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/InformationSchema.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/InformationSchema.java @@ -37,6 +37,11 @@ public class InformationSchema { public static final String DATABASES = "databases"; public static final String TABLES = "tables"; public static final String COLUMNS = "columns"; + public static final String REGIONS = "regions"; + public static final String PIPES = "pipes"; + public static final String PIPE_PLUGINS = "pipe_plugins"; + public static final String TOPICS = "topics"; + public static final String SUBSCRIPTIONS = "subscriptions"; static { final TsTable queriesTable = new TsTable(QUERIES); @@ -52,6 +57,9 @@ public class InformationSchema { queriesTable.addColumnSchema( new AttributeColumnSchema( ColumnHeaderConstant.STATEMENT.toLowerCase(Locale.ENGLISH), TSDataType.STRING)); + queriesTable.addColumnSchema( + new AttributeColumnSchema( + ColumnHeaderConstant.USER.toLowerCase(Locale.ENGLISH), TSDataType.STRING)); queriesTable.removeColumnSchema(TsTable.TIME_COLUMN_NAME); schemaTables.put(QUERIES, queriesTable); @@ -114,6 +122,107 @@ public class InformationSchema { ColumnHeaderConstant.STATUS.toLowerCase(Locale.ENGLISH), TSDataType.STRING)); columnTable.removeColumnSchema(TsTable.TIME_COLUMN_NAME); schemaTables.put(COLUMNS, columnTable); + + final TsTable regionTable = new TsTable(REGIONS); + regionTable.addColumnSchema( + new TagColumnSchema(ColumnHeaderConstant.REGION_ID_TABLE_MODEL, TSDataType.INT32)); + regionTable.addColumnSchema( + new TagColumnSchema(ColumnHeaderConstant.DATANODE_ID_TABLE_MODEL, TSDataType.INT32)); + regionTable.addColumnSchema( + new AttributeColumnSchema( + ColumnHeaderConstant.TYPE.toLowerCase(Locale.ENGLISH), TSDataType.STRING)); + regionTable.addColumnSchema( + new AttributeColumnSchema( + ColumnHeaderConstant.STATUS.toLowerCase(Locale.ENGLISH), TSDataType.STRING)); + regionTable.addColumnSchema( + new AttributeColumnSchema( + ColumnHeaderConstant.DATABASE.toLowerCase(Locale.ENGLISH), TSDataType.STRING)); + regionTable.addColumnSchema( + new AttributeColumnSchema( + ColumnHeaderConstant.SERIES_SLOT_NUM_TABLE_MODEL, TSDataType.INT32)); + regionTable.addColumnSchema( + new AttributeColumnSchema( + ColumnHeaderConstant.TIME_SLOT_NUM_TABLE_MODEL, TSDataType.INT64)); + regionTable.addColumnSchema( + new AttributeColumnSchema(ColumnHeaderConstant.RPC_ADDRESS_TABLE_MODEL, TSDataType.STRING)); + regionTable.addColumnSchema( + new AttributeColumnSchema(ColumnHeaderConstant.RPC_PORT_TABLE_MODEL, TSDataType.INT32)); + regionTable.addColumnSchema( + new AttributeColumnSchema( + ColumnHeaderConstant.INTERNAL_ADDRESS_TABLE_MODEL, TSDataType.STRING)); + regionTable.addColumnSchema( + new AttributeColumnSchema( + ColumnHeaderConstant.ROLE.toLowerCase(Locale.ENGLISH), TSDataType.STRING)); + regionTable.addColumnSchema( + new AttributeColumnSchema( + ColumnHeaderConstant.CREATE_TIME_TABLE_MODEL, TSDataType.TIMESTAMP)); + regionTable.addColumnSchema( + new AttributeColumnSchema( + ColumnHeaderConstant.TS_FILE_SIZE_BYTES_TABLE_MODEL, TSDataType.INT64)); + regionTable.removeColumnSchema(TsTable.TIME_COLUMN_NAME); + schemaTables.put(REGIONS, regionTable); + + final TsTable pipeTable = new TsTable(PIPES); + pipeTable.addColumnSchema( + new TagColumnSchema( + ColumnHeaderConstant.ID.toLowerCase(Locale.ENGLISH), TSDataType.STRING)); + pipeTable.addColumnSchema( + new AttributeColumnSchema( + ColumnHeaderConstant.CREATION_TIME_TABLE_MODEL, TSDataType.TIMESTAMP)); + pipeTable.addColumnSchema( + new AttributeColumnSchema( + ColumnHeaderConstant.STATE.toLowerCase(Locale.ENGLISH), TSDataType.STRING)); + pipeTable.addColumnSchema( + new AttributeColumnSchema(ColumnHeaderConstant.PIPE_SOURCE_TABLE_MODEL, TSDataType.STRING)); + pipeTable.addColumnSchema( + new AttributeColumnSchema( + ColumnHeaderConstant.PIPE_PROCESSOR_TABLE_MODEL, TSDataType.STRING)); + pipeTable.addColumnSchema( + new AttributeColumnSchema(ColumnHeaderConstant.PIPE_SINK_TABLE_MODEL, TSDataType.STRING)); + pipeTable.addColumnSchema( + new AttributeColumnSchema( + ColumnHeaderConstant.EXCEPTION_MESSAGE_TABLE_MODEL, TSDataType.STRING)); + pipeTable.addColumnSchema( + new AttributeColumnSchema( + ColumnHeaderConstant.REMAINING_EVENT_COUNT_TABLE_MODEL, TSDataType.INT64)); + pipeTable.addColumnSchema( + new AttributeColumnSchema( + ColumnHeaderConstant.ESTIMATED_REMAINING_SECONDS_TABLE_MODEL, TSDataType.DOUBLE)); + pipeTable.removeColumnSchema(TsTable.TIME_COLUMN_NAME); + schemaTables.put(PIPES, pipeTable); + + final TsTable pipePluginTable = new TsTable(PIPE_PLUGINS); + pipePluginTable.addColumnSchema( + new TagColumnSchema(ColumnHeaderConstant.PLUGIN_NAME_TABLE_MODEL, TSDataType.STRING)); + pipePluginTable.addColumnSchema( + new AttributeColumnSchema(ColumnHeaderConstant.PLUGIN_TYPE_TABLE_MODEL, TSDataType.STRING)); + pipePluginTable.addColumnSchema( + new AttributeColumnSchema(ColumnHeaderConstant.CLASS_NAME_TABLE_MODEL, TSDataType.STRING)); + pipePluginTable.addColumnSchema( + new AttributeColumnSchema(ColumnHeaderConstant.PLUGIN_JAR_TABLE_MODEL, TSDataType.STRING)); + pipePluginTable.removeColumnSchema(TsTable.TIME_COLUMN_NAME); + schemaTables.put(PIPE_PLUGINS, pipePluginTable); + + final TsTable topicTable = new TsTable(TOPICS); + topicTable.addColumnSchema( + new TagColumnSchema(ColumnHeaderConstant.TOPIC_NAME_TABLE_MODEL, TSDataType.STRING)); + topicTable.addColumnSchema( + new AttributeColumnSchema( + ColumnHeaderConstant.TOPIC_CONFIGS_TABLE_MODEL, TSDataType.STRING)); + topicTable.removeColumnSchema(TsTable.TIME_COLUMN_NAME); + schemaTables.put(TOPICS, topicTable); + + final TsTable subscriptionTable = new TsTable(SUBSCRIPTIONS); + subscriptionTable.addColumnSchema( + new TagColumnSchema(ColumnHeaderConstant.TOPIC_NAME_TABLE_MODEL, TSDataType.STRING)); + subscriptionTable.addColumnSchema( + new TagColumnSchema( + ColumnHeaderConstant.CONSUMER_GROUP_NAME_TABLE_MODEL, TSDataType.STRING)); + subscriptionTable.addColumnSchema( + new AttributeColumnSchema( + ColumnHeaderConstant.SUBSCRIBED_CONSUMERS_TABLE_MODEL, TSDataType.STRING)); + subscriptionTable.removeColumnSchema(TsTable.TIME_COLUMN_NAME); + schemaTables.put(SUBSCRIPTIONS, subscriptionTable); } public static Map getSchemaTables() { diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java index 906c9f05968e..65ecf6280b55 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java @@ -98,6 +98,7 @@ public enum Metric { MEM("mem"), CACHE("cache"), CACHE_HIT_RATE("cache_hit"), + CACHE_ENTRIES_NUM("cache_entries_num"), QUANTITY("quantity"), LEADER_QUANTITY("leader_quantity"), SCHEMA_REGION("schema_region"), diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/TableBuiltinScalarFunction.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/TableBuiltinScalarFunction.java index ee9b3d043b42..6d5b1ef4721a 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/TableBuiltinScalarFunction.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/TableBuiltinScalarFunction.java @@ -65,6 +65,8 @@ public enum TableBuiltinScalarFunction { E("e"), DATE_BIN("date_bin"), FORMAT("format"), + GREATEST("greatest"), + LEAST("least"), ; private final String functionName; diff --git a/iotdb-core/pom.xml b/iotdb-core/pom.xml index 77bec8db8895..71e5457cde61 100644 --- a/iotdb-core/pom.xml +++ b/iotdb-core/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-parent - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT iotdb-core pom diff --git a/iotdb-core/relational-grammar/pom.xml b/iotdb-core/relational-grammar/pom.xml index a27124cb36d9..df149c0441e9 100644 --- a/iotdb-core/relational-grammar/pom.xml +++ b/iotdb-core/relational-grammar/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-core - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT iotdb-relational-grammar IoTDB: Core: Relational-Antlr-Parser diff --git a/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 b/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 index 20180597005e..473e64713131 100644 --- a/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 +++ b/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 @@ -108,6 +108,9 @@ statement | countTimeSlotListStatement | showSeriesSlotListStatement | migrateRegionStatement + | reconstructRegionStatement + | extendRegionStatement + | removeRegionStatement | removeDataNodeStatement // Admin Statement @@ -123,6 +126,7 @@ statement | loadConfigurationStatement | setConfigurationStatement | showCurrentSqlDialectStatement + | setSqlDialectStatement | showCurrentUserStatement | showCurrentDatabaseStatement | showCurrentTimestampStatement @@ -479,6 +483,18 @@ migrateRegionStatement : MIGRATE REGION regionId=INTEGER_VALUE FROM fromId=INTEGER_VALUE TO toId=INTEGER_VALUE ; +reconstructRegionStatement + : RECONSTRUCT REGION regionIds+=INTEGER_VALUE (COMMA regionIds+=INTEGER_VALUE)* ON targetDataNodeId=INTEGER_VALUE + ; + +extendRegionStatement + : EXTEND REGION regionId=INTEGER_VALUE TO targetDataNodeId=INTEGER_VALUE + ; + +removeRegionStatement + : REMOVE REGION regionId=INTEGER_VALUE FROM targetDataNodeId=INTEGER_VALUE + ; + removeDataNodeStatement : REMOVE DATANODE dataNodeId=INTEGER_VALUE (',' dataNodeId=INTEGER_VALUE)* ; @@ -547,6 +563,10 @@ showCurrentSqlDialectStatement : SHOW CURRENT_SQL_DIALECT ; +setSqlDialectStatement + : SET SQL_DIALECT EQ (TABLE | TREE) + ; + showCurrentUserStatement : SHOW CURRENT_USER ; @@ -604,11 +624,11 @@ listRolePrivilegeStatement ; listUserStatement - : LIST USER + : LIST USER (OF ROLE roleName=identifier)? ; listRoleStatement - : LIST ROLE + : LIST ROLE (OF USER userName=identifier)? ; @@ -619,7 +639,7 @@ revokeStatement privilegeObjectScope : systemPrivileges | objectPrivileges ON objectType objectName=identifier - | objectPrivileges ON objectScope + | objectPrivileges ON (TABLE)? objectScope | objectPrivileges ON ANY | ALL ; @@ -630,6 +650,7 @@ systemPrivileges objectPrivileges : objectPrivilege (',' objectPrivilege)* + | ALL ; objectScope @@ -1188,6 +1209,7 @@ EXCLUDING: 'EXCLUDING'; EXECUTE: 'EXECUTE'; EXISTS: 'EXISTS'; EXPLAIN: 'EXPLAIN'; +EXTEND: 'EXTEND'; EXTRACT: 'EXTRACT'; EXTRACTOR: 'EXTRACTOR'; FALSE: 'FALSE'; @@ -1389,6 +1411,7 @@ SINK: 'SINK'; SKIP_TOKEN: 'SKIP'; SOME: 'SOME'; SOURCE: 'SOURCE'; +SQL_DIALECT: 'SQL_DIALECT'; START: 'START'; STATS: 'STATS'; STOP: 'STOP'; @@ -1416,6 +1439,7 @@ TOPIC: 'TOPIC'; TOPICS: 'TOPICS'; TRAILING: 'TRAILING'; TRANSACTION: 'TRANSACTION'; +TREE: 'TREE'; TRIM: 'TRIM'; TRUE: 'TRUE'; TRUNCATE: 'TRUNCATE'; diff --git a/iotdb-protocol/openapi/pom.xml b/iotdb-protocol/openapi/pom.xml index c82eebcc0cf1..27c35db316ec 100644 --- a/iotdb-protocol/openapi/pom.xml +++ b/iotdb-protocol/openapi/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-protocol - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT openapi IoTDB: Protocol: OpenAPI diff --git a/iotdb-protocol/pom.xml b/iotdb-protocol/pom.xml index 523ea7f37dae..84bdca026a6a 100644 --- a/iotdb-protocol/pom.xml +++ b/iotdb-protocol/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-parent - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT iotdb-protocol pom diff --git a/iotdb-protocol/thrift-ainode/pom.xml b/iotdb-protocol/thrift-ainode/pom.xml index 3ed313f9ab39..4ac2efd8ce51 100644 --- a/iotdb-protocol/thrift-ainode/pom.xml +++ b/iotdb-protocol/thrift-ainode/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-protocol - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT iotdb-thrift-ainode IoTDB: Protocol: Thrift AI Node @@ -41,7 +41,7 @@ org.apache.iotdb iotdb-thrift-commons - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT diff --git a/iotdb-protocol/thrift-commons/pom.xml b/iotdb-protocol/thrift-commons/pom.xml index a9ebdba2fe22..b2c14fc62b42 100644 --- a/iotdb-protocol/thrift-commons/pom.xml +++ b/iotdb-protocol/thrift-commons/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-protocol - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT iotdb-thrift-commons IoTDB: Protocol: Thrift Commons diff --git a/iotdb-protocol/thrift-confignode/pom.xml b/iotdb-protocol/thrift-confignode/pom.xml index 7d7f00db430a..2816a1a21a9f 100644 --- a/iotdb-protocol/thrift-confignode/pom.xml +++ b/iotdb-protocol/thrift-confignode/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-protocol - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT iotdb-thrift-confignode IoTDB: Protocol: Thrift Config Node @@ -41,7 +41,7 @@ org.apache.iotdb iotdb-thrift-commons - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT diff --git a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift index 7385fafd2e08..c649566fe071 100644 --- a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift +++ b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift @@ -310,21 +310,25 @@ struct TMigrateRegionReq { 1: required i32 regionId 2: required i32 fromId 3: required i32 toId + 4: required common.Model model } struct TReconstructRegionReq { 1: required list regionIds 2: required i32 dataNodeId + 3: required common.Model model } struct TExtendRegionReq { 1: required i32 regionId 2: required i32 dataNodeId + 3: required common.Model model } struct TRemoveRegionReq { 1: required i32 regionId 2: required i32 dataNodeId + 3: required common.Model model } // Authorize diff --git a/iotdb-protocol/thrift-consensus/pom.xml b/iotdb-protocol/thrift-consensus/pom.xml index 6c4fcb3fc153..1297b6ba431d 100644 --- a/iotdb-protocol/thrift-consensus/pom.xml +++ b/iotdb-protocol/thrift-consensus/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-protocol - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT iotdb-thrift-consensus IoTDB: Protocol: Thrift Consensus @@ -33,7 +33,7 @@ org.apache.iotdb iotdb-thrift-commons - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.slf4j diff --git a/iotdb-protocol/thrift-datanode/pom.xml b/iotdb-protocol/thrift-datanode/pom.xml index 3204b36c956c..8fae19237f8f 100644 --- a/iotdb-protocol/thrift-datanode/pom.xml +++ b/iotdb-protocol/thrift-datanode/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-protocol - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT iotdb-thrift IoTDB: Protocol: Thrift Data Node @@ -33,7 +33,7 @@ org.apache.iotdb iotdb-thrift-commons - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.apache.thrift diff --git a/library-udf/pom.xml b/library-udf/pom.xml index 48af935e19ae..ac695ec9a319 100644 --- a/library-udf/pom.xml +++ b/library-udf/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-parent - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT library-udf IoTDB: UDF @@ -41,7 +41,7 @@ org.apache.iotdb udf-api - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT org.slf4j diff --git a/pom.xml b/pom.xml index c11760ef8788..62d393eb2044 100644 --- a/pom.xml +++ b/pom.xml @@ -28,7 +28,7 @@ org.apache.iotdb iotdb-parent - 2.0.0-SNAPSHOT + 2.0.2-SNAPSHOT pom Apache IoTDB Project Parent POM This is the top level project that builds, packages the iotdb engine, client, and integration libs. @@ -106,7 +106,7 @@ 0.11.5 3.26.2 5.14.0 - 2.5.0 + 2.5.2 3.1 4.13.2 @@ -168,7 +168,7 @@ 0.14.1 1.9 1.5.6-3 - 2.0.0-250118-SNAPSHOT + 2.1.0-250207-SNAPSHOT + ${project.parent.build.directory}/generated-sources-csharp + + From 27985e72ee6a116ca42cdab39241e868a5fadd6a Mon Sep 17 00:00:00 2001 From: 0xB <76414902+YC27@users.noreply.github.com> Date: Thu, 20 Feb 2025 14:55:14 +0800 Subject: [PATCH 10/10] Revert "Collector Node: Fix compile error & dependency issue" (#14908) * Revert "Collector Node: Fix compile error & dependency issue (#14906)" This reverts commit 959b57527ac61a0399e2f115b6d2cf355ea153df. * Collector Node: Fix compile error & dependency issue --- .github/workflows/cluster-it-1c1d.yml | 9 +- .github/workflows/cluster-it-1c1d1a.yml | 9 +- .github/workflows/cluster-it-1c3d.yml | 9 +- .github/workflows/compile-check.yml | 11 +- .github/workflows/dependency-check.yml | 1 + .github/workflows/multi-language-client.yml | 1 + .../{pipe-it.yml => pipe-it-2cluster.yml} | 314 +----- .github/workflows/sonar-codecov.yml | 1 + .github/workflows/table-cluster-it-1c1d.yml | 9 +- .github/workflows/table-cluster-it-1c3d.yml | 9 +- .github/workflows/unit-test.yml | 1 + code-coverage/pom.xml | 2 +- distribution/pom.xml | 12 +- docker/src/main/DockerCompose/entrypoint.sh | 5 +- .../DockerCompose/replace-conf-from-env.sh | 33 +- example/client-cpp-example/pom.xml | 2 +- example/jdbc/pom.xml | 2 +- example/mqtt-customize/pom.xml | 2 +- example/mqtt/pom.xml | 2 +- example/pipe-count-point-processor/pom.xml | 2 +- example/pipe-opc-ua-sink/pom.xml | 2 +- example/pom.xml | 2 +- example/rest-java-example/pom.xml | 2 +- example/schema/pom.xml | 2 +- example/session/pom.xml | 2 +- example/trigger/pom.xml | 2 +- example/udf/pom.xml | 2 +- integration-test/pom.xml | 74 +- .../iotdb/it/env/cluster/env/AbstractEnv.java | 12 - .../it/env/remote/env/RemoteServerEnv.java | 10 - ...a => MultiClusterIT2AutoCreateSchema.java} | 2 +- ...ultiClusterIT2DualTableManualEnhanced.java | 22 - .../MultiClusterIT2DualTreeAutoBasic.java | 22 - ...=> MultiClusterIT2ManualCreateSchema.java} | 2 +- ...al.java => MultiClusterIT2TableModel.java} | 2 +- .../org/apache/iotdb/itbase/env/BaseEnv.java | 3 - .../itbase/runtime/ClusterTestStatement.java | 5 +- .../iotdb/db/it/IoTDBInsertWithQueryIT.java | 1 - .../apache/iotdb/db/it/IoTDBLoadTsFileIT.java | 73 -- .../apache/iotdb/db/it/auth/IoTDBAuthIT.java | 38 - .../db/it/auth/IoTDBRelationalAuthIT.java | 183 --- .../db/it/auth/IoTDBSystemPermissionIT.java | 2 +- .../iotdb/db/it/query/IoTDBQueryDemoIT.java | 25 - .../apache/iotdb/db/it/utils/TestUtils.java | 227 +--- .../AbstractPipeDualAutoIT.java} | 4 +- .../IoTDBPipeAlterIT.java | 9 +- .../IoTDBPipeAutoConflictIT.java | 9 +- .../IoTDBPipeAutoDropIT.java | 9 +- .../IoTDBPipeClusterIT.java | 11 +- .../IoTDBPipeConditionalOperationsIT.java | 9 +- .../IoTDBPipeConnectorCompressionIT.java | 9 +- .../IoTDBPipeConnectorParallelIT.java | 9 +- .../IoTDBPipeDataSinkIT.java | 9 +- .../IoTDBPipeExtractorIT.java | 9 +- .../IoTDBPipeIdempotentIT.java | 100 +- .../IoTDBPipeLifeCycleIT.java | 22 +- .../IoTDBPipeNullValueIT.java | 9 +- .../IoTDBPipeProcessorIT.java | 9 +- .../IoTDBPipeProtocolIT.java | 9 +- .../IoTDBPipeSwitchStatusIT.java | 9 +- .../IoTDBPipeSyntaxIT.java | 9 +- .../IoTDBPipeWithLoadIT.java | 9 +- .../IoTDBTreePatternFormatIT.java | 9 +- .../PipeNowFunctionIT.java | 9 +- .../enhanced/IoTDBPipeIdempotentIT.java | 197 ---- .../AbstractPipeDualManualIT.java} | 4 +- .../manual/IoTDBPipeInclusionIT.java | 8 +- .../manual/IoTDBPipeManualConflictIT.java | 8 +- .../manual/IoTDBPipeMetaHistoricalIT.java | 8 +- .../manual/IoTDBPipeMetaLeaderChangeIT.java | 8 +- .../manual/IoTDBPipeMetaRestartIT.java | 8 +- .../manual/IoTDBPipeMultiSchemaRegionIT.java | 8 +- .../manual/IoTDBPipePermissionIT.java | 8 +- .../manual/IoTDBPipeReqAutoSliceIT.java | 8 +- .../IoTDBPipeTableManualIT.java} | 87 +- .../IoTDBPipeTypeConversionISessionIT.java | 8 +- .../manual/IoTDBPipeTypeConversionIT.java | 8 +- .../pipe/it/single/IoTDBPipeOPCUAIT.java | 2 +- .../AbstractPipeTableModelTestIT.java} | 4 +- .../IoTDBPipeAlterIT.java | 10 +- .../IoTDBPipeAutoConflictIT.java | 10 +- .../IoTDBPipeAutoDropIT.java | 10 +- .../IoTDBPipeClusterIT.java | 10 +- .../IoTDBPipeConnectorCompressionIT.java | 10 +- .../IoTDBPipeConnectorParallelIT.java | 10 +- .../IoTDBPipeDataSinkIT.java | 14 +- .../IoTDBPipeDoubleLivingIT.java | 13 +- .../IoTDBPipeExtractorIT.java | 10 +- .../IoTDBPipeIsolationIT.java | 10 +- .../IoTDBPipeLifeCycleIT.java | 157 +-- .../IoTDBPipeNullValueIT.java | 10 +- .../IoTDBPipeProtocolIT.java | 10 +- .../IoTDBPipeSwitchStatusIT.java | 10 +- .../IoTDBPipeSyntaxIT.java | 9 +- .../IoTDBPipeTypeConversionISessionIT.java | 18 +- .../IoTDBPipeTypeConversionIT.java | 18 +- .../IoTDBPipeWithLoadIT.java | 10 +- .../IoTDBTablePatternFormatIT.java | 10 +- .../tablemodel/TableModelUtils.java | 2 +- .../it/db/it/IoTDBAuthenticationTableIT.java | 1001 ----------------- .../it/db/it/IoTDBDeletionTableIT.java | 44 +- .../it/db/it/IoTDBInsertTableIT.java | 11 - .../db/it/IoTDBLoadConfigurationTableIT.java | 85 -- .../it/db/it/IoTDBSetSystemStatusTableIT.java | 104 -- .../old/query/IoTDBGreatestLeastTableIT.java | 305 ----- .../it/query/recent/IoTDBMaintainAuthIT.java | 289 ----- .../it/query/recent/IoTDBNullIdQueryIT.java | 33 - .../it/query/recent/IoTDBNullValueIT.java | 75 -- .../it/query/recent/IoTDBQueryAuthIT.java | 320 ------ .../relational/it/schema/IoTDBDatabaseIT.java | 145 +-- .../relational/it/schema/IoTDBTableIT.java | 57 - .../session/it/IoTDBSessionInsertNullIT.java | 58 - .../it/local/IoTDBSubscriptionBasicIT.java | 67 -- iotdb-api/external-api/pom.xml | 2 +- iotdb-api/pipe-api/pom.xml | 2 +- iotdb-api/pom.xml | 2 +- iotdb-api/trigger-api/pom.xml | 2 +- iotdb-api/udf-api/pom.xml | 2 +- iotdb-client/cli/pom.xml | 22 +- .../tool/common/ImportTsFileOperation.java | 59 - .../iotdb/tool/data/AbstractDataTool.java | 6 +- .../apache/iotdb/tool/data/ImportData.java | 22 +- .../iotdb/tool/tsfile/ImportTsFile.java | 57 +- .../iotdb/tool/tsfile/ImportTsFileBase.java | 9 +- .../iotdb/tool/ImportTsFileOperationTest.java | 49 - iotdb-client/client-cpp/pom.xml | 4 +- iotdb-client/client-py/pom.xml | 6 +- iotdb-client/isession/pom.xml | 8 +- iotdb-client/jdbc/pom.xml | 8 +- .../java/org/apache/iotdb/jdbc/Constant.java | 3 - .../apache/iotdb/jdbc/IoTDBConnection.java | 4 - .../iotdb/jdbc/IoTDBConnectionParams.java | 4 +- .../org/apache/iotdb/jdbc/IoTDBStatement.java | 7 - iotdb-client/pom.xml | 2 +- iotdb-client/service-rpc/pom.xml | 6 +- .../java/org/apache/iotdb/rpc/RpcUtils.java | 14 - .../org/apache/iotdb/rpc/RpcUtilsTest.java | 10 - iotdb-client/session/pom.xml | 10 +- .../org/apache/iotdb/session/Session.java | 25 +- .../iotdb/session/SessionConnection.java | 12 +- .../iotdb/session/TableSessionBuilder.java | 4 +- .../iotdb/session/pool/SessionPool.java | 5 +- .../session/pool/TableSessionWrapper.java | 15 - .../base/AbstractSubscriptionConsumer.java | 4 +- .../base/AbstractSubscriptionProviders.java | 48 +- iotdb-collector/collector-core/pom.xml | 14 +- iotdb-collector/collector-openapi/pom.xml | 2 +- iotdb-collector/pom.xml | 2 +- iotdb-core/ainode/pom.xml | 10 +- iotdb-core/ainode/pyproject.toml | 14 +- .../ainode/resources/conf/ainode-env.sh | 2 +- iotdb-core/antlr/pom.xml | 2 +- .../iotdb/db/qp/sql/IdentifierParser.g4 | 5 - .../apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 14 +- .../org/apache/iotdb/db/qp/sql/SqlLexer.g4 | 20 - iotdb-core/confignode/pom.xml | 26 +- .../request/ConfigPhysicalPlanVisitor.java | 221 +--- .../confignode/manager/PermissionManager.java | 7 +- .../confignode/manager/ProcedureManager.java | 36 +- .../event/PipeConfigRegionSnapshotEvent.java | 14 +- .../ConfigRegionListeningFilter.java | 84 +- .../extractor/IoTDBConfigRegionExtractor.java | 23 +- ...gPhysicalPlanTablePatternParseVisitor.java | 62 - ...figPhysicalPlanTableScopeParseVisitor.java | 86 -- ...igPhysicalPlanTreePatternParseVisitor.java | 30 +- ...nfigPhysicalPlanTreeScopeParseVisitor.java | 80 -- .../protocol/IoTDBConfigNodeReceiver.java | 162 +-- ...PipeConfigPhysicalPlanTSStatusVisitor.java | 207 +--- .../confignode/persistence/AuthorInfo.java | 70 +- ...nfigPhysicalPlanScopeParseVisitorTest.java | 116 -- ...sicalPlanTablePatternParseVisitorTest.java | 25 - iotdb-core/consensus/pom.xml | 14 +- iotdb-core/datanode/pom.xml | 40 +- .../iotdb/db/auth/AuthorityChecker.java | 3 - .../PipeRealtimePriorityBlockingQueue.java | 2 - .../processor/PipeProcessorSubtask.java | 5 +- .../processor/PipeProcessorSubtaskWorker.java | 13 +- .../tsfile/PipeTsFileInsertionEvent.java | 49 +- .../SchemaRegionListeningFilter.java | 8 +- .../thrift/impl/ClientRPCServiceImpl.java | 27 +- .../exchange/SharedTsBlockQueue.java | 8 +- .../execution/exchange/sink/SinkChannel.java | 11 +- .../exchange/source/LocalSourceHandle.java | 4 +- .../executor/RegionWriteExecutor.java | 33 +- .../fragment/FragmentInstanceContext.java | 49 +- .../execution/operator/AbstractOperator.java | 2 +- .../process/AbstractSortOperator.java | 2 +- .../operator/process/SortOperator.java | 2 +- .../process/TableStreamSortOperator.java | 2 +- .../operator/source/FileLoaderUtils.java | 4 +- ...formationSchemaContentSupplierFactory.java | 341 +----- .../grouped/HashAggregationOperator.java | 4 +- .../grouped/StreamingAggregationOperator.java | 9 +- .../StreamingHashAggregationOperator.java | 4 +- .../relational/ColumnTransformerBuilder.java | 15 - .../execution/schedule/DriverScheduler.java | 8 + .../db/queryengine/plan/Coordinator.java | 16 +- .../plan/analyze/AnalyzeUtils.java | 9 +- .../plan/analyze/AnalyzeVisitor.java | 4 +- .../dualkeycache/IDualKeyCacheStats.java | 6 - .../schema/dualkeycache/impl/CacheStats.java | 24 +- .../dualkeycache/impl/DualKeyCacheImpl.java | 122 +- .../plan/analyze/load/LoadTsFileAnalyzer.java | 7 - .../plan/analyze/schema/SchemaValidator.java | 13 +- .../plan/execution/IQueryExecution.java | 2 - .../plan/execution/QueryExecution.java | 5 - .../execution/config/ConfigExecution.java | 9 +- .../config/TableConfigTaskVisitor.java | 76 -- .../config/TreeConfigTaskVisitor.java | 23 - .../executor/ClusterConfigTaskExecutor.java | 69 +- .../config/executor/IConfigTaskExecutor.java | 19 +- .../config/metadata/ShowPipePluginsTask.java | 4 +- .../metadata/region/ExtendRegionTask.java | 20 +- .../metadata/region/MigrateRegionTask.java | 21 +- .../region/ReconstructRegionTask.java | 25 +- .../metadata/region/RemoveRegionTask.java | 20 +- .../config/session/SetSqlDialectTask.java | 41 - .../queryengine/plan/parser/ASTVisitor.java | 31 +- .../plan/planner/TableOperatorGenerator.java | 9 +- .../plan/node/load/LoadTsFilePieceNode.java | 6 +- .../analyzer/StatementAnalyzer.java | 11 +- .../metadata/TableMetadataImpl.java | 20 - .../fetcher/cache/TableDeviceSchemaCache.java | 19 +- .../cache/TableDeviceSchemaCacheMetrics.java | 34 - .../DataNodeLocationSupplierFactory.java | 22 +- .../relational/security/AccessControl.java | 8 - .../security/AccessControlImpl.java | 62 +- .../security/AllowAllAccessControl.java | 5 - .../plan/relational/sql/ast/AstVisitor.java | 28 - .../plan/relational/sql/ast/ExtendRegion.java | 77 -- .../plan/relational/sql/ast/InsertRows.java | 2 +- .../relational/sql/ast/LoadConfiguration.java | 34 - .../relational/sql/ast/MigrateRegion.java | 90 -- .../relational/sql/ast/ReconstructRegion.java | 76 -- .../plan/relational/sql/ast/RemoveRegion.java | 77 -- .../relational/sql/ast/SetSqlDialect.java | 79 -- .../relational/sql/ast/SetSystemStatus.java | 34 - .../relational/sql/parser/AstBuilder.java | 162 +-- .../scheduler/load/LoadTsFileScheduler.java | 47 +- .../plan/statement/StatementVisitor.java | 16 - .../statement/sys/SetSqlDialectStatement.java | 76 -- .../sys/ShowCurrentSqlDialectStatement.java | 45 - .../sys/ShowCurrentUserStatement.java | 45 - ...bstractGreatestLeastColumnTransformer.java | 109 -- .../BinaryGreatestColumnTransformer.java | 53 - .../multi/BinaryLeastColumnTransformer.java | 53 - .../BooleanGreatestColumnTransformer.java | 52 - .../multi/BooleanLeastColumnTransformer.java | 49 - .../DoubleGreatestColumnTransformer.java | 52 - .../multi/DoubleLeastColumnTransformer.java | 52 - .../multi/FloatGreatestColumnTransformer.java | 52 - .../multi/FloatLeastColumnTransformer.java | 52 - .../multi/Int32GreatestColumnTransformer.java | 52 - .../multi/Int32LeastColumnTransformer.java | 52 - .../multi/Int64GreatestColumnTransformer.java | 52 - .../multi/Int64LeastColumnTransformer.java | 52 - .../utils/ResourceByPathUtils.java | 31 +- .../iotdb/db/storageengine/StorageEngine.java | 23 +- .../impl/ReadChunkCompactionPerformer.java | 11 +- .../utils/MultiTsFileDeviceIterator.java | 14 +- ...edFastAlignedSeriesCompactionExecutor.java | 26 +- ...dChunkAlignedSeriesCompactionExecutor.java | 40 +- .../AlignedSeriesBatchCompactionUtils.java | 28 +- .../FastAlignedSeriesCompactionExecutor.java | 23 +- .../fast/SeriesCompactionExecutor.java | 10 +- ...dChunkAlignedSeriesCompactionExecutor.java | 31 +- .../writer/FastCrossCompactionWriter.java | 10 +- .../writer/FastInnerCompactionWriter.java | 10 +- .../io/CompactionDiskTSMIterator.java | 43 - .../compaction/io/CompactionTsFileInput.java | 122 +- .../compaction/io/CompactionTsFileReader.java | 60 +- .../compaction/io/CompactionTsFileWriter.java | 9 - .../memtable/AlignedReadOnlyMemChunk.java | 10 +- .../dataregion/memtable/TsFileProcessor.java | 7 +- .../reader/chunk/DiskAlignedChunkLoader.java | 5 +- .../reader/chunk/MemAlignedChunkReader.java | 4 +- .../reader/chunk/MemAlignedPageReader.java | 6 +- .../DiskAlignedChunkMetadataLoader.java | 4 +- .../storageengine/load/LoadTsFileManager.java | 5 +- ...tementDataTypeConvertExecutionVisitor.java | 4 - ...tementDataTypeConvertExecutionVisitor.java | 4 - .../broker/SubscriptionPrefetchingQueue.java | 11 - .../SubscriptionEventTabletResponse.java | 5 + .../TsFileRewriteOverPrecisedI64Scan.java | 223 ---- .../utils/TsFileRewriteSmallRangeI64Scan.java | 230 ---- .../db/tools/utils/TsFileSequenceScan.java | 34 +- .../db/tools/utils/TsFileStatisticScan.java | 266 ----- .../iotdb/db/utils/ErrorHandlingUtils.java | 6 +- .../iotdb/db/utils/ModificationUtils.java | 9 +- .../db/utils/TimestampPrecisionUtils.java | 4 +- .../iotdb/db/utils/constant/SqlConstant.java | 2 +- .../queryengine/execution/exchange/Utils.java | 3 - .../operator/MergeTreeSortOperatorTest.java | 6 - .../operator/OperatorMemoryTest.java | 2 - .../plan/parser/StatementGeneratorTest.java | 3 - .../relational/analyzer/ShowQueriesTest.java | 7 +- ...sistentCompressionTypeAndEncodingTest.java | 6 +- .../FastInnerCompactionPerformerTest.java | 6 +- .../RepairUnsortedFileCompactionTest.java | 8 +- .../CompactionTableModelTestFileWriter.java | 2 +- .../utils/BatchCompactionUtilsTest.java | 39 +- .../db/utils/TimestampPrecisionUtilsTest.java | 10 - iotdb-core/metrics/core/pom.xml | 4 +- iotdb-core/metrics/interface/pom.xml | 8 +- iotdb-core/metrics/pom.xml | 2 +- iotdb-core/node-commons/pom.xml | 24 +- .../conf/iotdb-system.properties.template | 10 +- .../auth/authorizer/BasicAuthorizer.java | 21 - .../commons/auth/authorizer/IAuthorizer.java | 4 - .../iotdb/commons/auth/entity/Role.java | 8 - .../iotdb/commons/conf/CommonConfig.java | 15 +- .../iotdb/commons/conf/CommonDescriptor.java | 24 +- .../task/connection/BlockingPendingQueue.java | 14 - .../options/PipeInclusionOptions.java | 15 +- .../schema/column/ColumnHeaderConstant.java | 30 - .../schema/table/InformationSchema.java | 109 -- .../commons/service/metric/enums/Metric.java | 1 - .../TableBuiltinScalarFunction.java | 2 - iotdb-core/pom.xml | 2 +- iotdb-core/relational-grammar/pom.xml | 2 +- .../relational/grammar/sql/RelationalSql.g4 | 30 +- iotdb-protocol/openapi/pom.xml | 2 +- iotdb-protocol/pom.xml | 2 +- iotdb-protocol/thrift-ainode/pom.xml | 4 +- iotdb-protocol/thrift-commons/pom.xml | 2 +- iotdb-protocol/thrift-confignode/pom.xml | 4 +- .../src/main/thrift/confignode.thrift | 4 - iotdb-protocol/thrift-consensus/pom.xml | 4 +- iotdb-protocol/thrift-datanode/pom.xml | 4 +- library-udf/pom.xml | 4 +- pom.xml | 19 +- 331 files changed, 1441 insertions(+), 9926 deletions(-) rename .github/workflows/{pipe-it.yml => pipe-it-2cluster.yml} (62%) rename integration-test/src/main/java/org/apache/iotdb/itbase/category/{MultiClusterIT2DualTreeAutoEnhanced.java => MultiClusterIT2AutoCreateSchema.java} (94%) delete mode 100644 integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2DualTableManualEnhanced.java delete mode 100644 integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2DualTreeAutoBasic.java rename integration-test/src/main/java/org/apache/iotdb/itbase/category/{MultiClusterIT2DualTableManualBasic.java => MultiClusterIT2ManualCreateSchema.java} (94%) rename integration-test/src/main/java/org/apache/iotdb/itbase/category/{MultiClusterIT2DualTreeManual.java => MultiClusterIT2TableModel.java} (94%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{dual/treemodel/auto/AbstractPipeDualTreeModelAutoIT.java => autocreate/AbstractPipeDualAutoIT.java} (97%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{dual/treemodel/auto/basic => autocreate}/IoTDBPipeAlterIT.java (98%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{dual/treemodel/auto/enhanced => autocreate}/IoTDBPipeAutoConflictIT.java (98%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{dual/treemodel/auto/enhanced => autocreate}/IoTDBPipeAutoDropIT.java (95%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{dual/treemodel/auto/enhanced => autocreate}/IoTDBPipeClusterIT.java (98%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{dual/treemodel/auto/enhanced => autocreate}/IoTDBPipeConditionalOperationsIT.java (96%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{dual/treemodel/auto/enhanced => autocreate}/IoTDBPipeConnectorCompressionIT.java (97%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{dual/treemodel/auto/basic => autocreate}/IoTDBPipeConnectorParallelIT.java (92%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{dual/treemodel/auto/basic => autocreate}/IoTDBPipeDataSinkIT.java (98%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{dual/treemodel/auto/basic => autocreate}/IoTDBPipeExtractorIT.java (99%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{dual/treemodel/auto/enhanced => autocreate}/IoTDBPipeIdempotentIT.java (80%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{dual/treemodel/auto/basic => autocreate}/IoTDBPipeLifeCycleIT.java (97%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{dual/treemodel/auto/enhanced => autocreate}/IoTDBPipeNullValueIT.java (97%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{dual/treemodel/auto/basic => autocreate}/IoTDBPipeProcessorIT.java (94%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{dual/treemodel/auto/basic => autocreate}/IoTDBPipeProtocolIT.java (98%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{dual/treemodel/auto/basic => autocreate}/IoTDBPipeSwitchStatusIT.java (97%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{dual/treemodel/auto/basic => autocreate}/IoTDBPipeSyntaxIT.java (98%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{dual/treemodel/auto/enhanced => autocreate}/IoTDBPipeWithLoadIT.java (93%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{dual/treemodel/auto/basic => autocreate}/IoTDBTreePatternFormatIT.java (95%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{dual/treemodel/auto/enhanced => autocreate}/PipeNowFunctionIT.java (97%) delete mode 100644 integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeIdempotentIT.java rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{dual/treemodel/manual/AbstractPipeDualTreeModelManualIT.java => manual/AbstractPipeDualManualIT.java} (97%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{dual/treemodel => }/manual/IoTDBPipeInclusionIT.java (97%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{dual/treemodel => }/manual/IoTDBPipeManualConflictIT.java (97%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{dual/treemodel => }/manual/IoTDBPipeMetaHistoricalIT.java (98%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{dual/treemodel => }/manual/IoTDBPipeMetaLeaderChangeIT.java (97%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{dual/treemodel => }/manual/IoTDBPipeMetaRestartIT.java (96%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{dual/treemodel => }/manual/IoTDBPipeMultiSchemaRegionIT.java (95%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{dual/treemodel => }/manual/IoTDBPipePermissionIT.java (97%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{dual/treemodel => }/manual/IoTDBPipeReqAutoSliceIT.java (98%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{dual/tablemodel/manual/enhanced/IoTDBPipeMetaIT.java => manual/IoTDBPipeTableManualIT.java} (77%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{dual/treemodel => }/manual/IoTDBPipeTypeConversionISessionIT.java (99%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{dual/treemodel => }/manual/IoTDBPipeTypeConversionIT.java (98%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{dual/tablemodel/manual/AbstractPipeTableModelDualManualIT.java => tablemodel/AbstractPipeTableModelTestIT.java} (95%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{dual/tablemodel/manual/basic => tablemodel}/IoTDBPipeAlterIT.java (98%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{dual/tablemodel/manual/enhanced => tablemodel}/IoTDBPipeAutoConflictIT.java (96%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{dual/tablemodel/manual/enhanced => tablemodel}/IoTDBPipeAutoDropIT.java (94%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{dual/tablemodel/manual/enhanced => tablemodel}/IoTDBPipeClusterIT.java (98%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{dual/tablemodel/manual/enhanced => tablemodel}/IoTDBPipeConnectorCompressionIT.java (97%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{dual/tablemodel/manual/enhanced => tablemodel}/IoTDBPipeConnectorParallelIT.java (91%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{dual/tablemodel/manual/basic => tablemodel}/IoTDBPipeDataSinkIT.java (98%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{dual/tablemodel/manual/enhanced => tablemodel}/IoTDBPipeDoubleLivingIT.java (96%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{dual/tablemodel/manual/basic => tablemodel}/IoTDBPipeExtractorIT.java (99%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{dual/tablemodel/manual/basic => tablemodel}/IoTDBPipeIsolationIT.java (97%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{dual/tablemodel/manual/basic => tablemodel}/IoTDBPipeLifeCycleIT.java (83%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{dual/tablemodel/manual/enhanced => tablemodel}/IoTDBPipeNullValueIT.java (94%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{dual/tablemodel/manual/basic => tablemodel}/IoTDBPipeProtocolIT.java (97%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{dual/tablemodel/manual/basic => tablemodel}/IoTDBPipeSwitchStatusIT.java (97%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{dual/tablemodel/manual/basic => tablemodel}/IoTDBPipeSyntaxIT.java (98%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{dual/tablemodel/manual/enhanced => tablemodel}/IoTDBPipeTypeConversionISessionIT.java (96%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{dual/tablemodel/manual/enhanced => tablemodel}/IoTDBPipeTypeConversionIT.java (97%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{dual/tablemodel/manual/basic => tablemodel}/IoTDBPipeWithLoadIT.java (97%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{dual/tablemodel/manual/basic => tablemodel}/IoTDBTablePatternFormatIT.java (98%) rename integration-test/src/test/java/org/apache/iotdb/pipe/it/{dual => }/tablemodel/TableModelUtils.java (99%) delete mode 100644 integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBAuthenticationTableIT.java delete mode 100644 integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLoadConfigurationTableIT.java delete mode 100644 integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBSetSystemStatusTableIT.java delete mode 100644 integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/query/IoTDBGreatestLeastTableIT.java delete mode 100644 integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBMaintainAuthIT.java delete mode 100644 integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBNullValueIT.java delete mode 100644 integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBQueryAuthIT.java delete mode 100644 iotdb-client/cli/src/main/java/org/apache/iotdb/tool/common/ImportTsFileOperation.java delete mode 100644 iotdb-client/cli/src/test/java/org/apache/iotdb/tool/ImportTsFileOperationTest.java delete mode 100644 iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/extractor/PipeConfigPhysicalPlanTableScopeParseVisitor.java delete mode 100644 iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/extractor/PipeConfigPhysicalPlanTreeScopeParseVisitor.java delete mode 100644 iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/extractor/PipeConfigPhysicalPlanScopeParseVisitorTest.java delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/session/SetSqlDialectTask.java delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ExtendRegion.java delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/LoadConfiguration.java delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/MigrateRegion.java delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ReconstructRegion.java delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/RemoveRegion.java delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SetSqlDialect.java delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SetSystemStatus.java delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/sys/SetSqlDialectStatement.java delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/sys/ShowCurrentSqlDialectStatement.java delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/sys/ShowCurrentUserStatement.java delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/AbstractGreatestLeastColumnTransformer.java delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/BinaryGreatestColumnTransformer.java delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/BinaryLeastColumnTransformer.java delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/BooleanGreatestColumnTransformer.java delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/BooleanLeastColumnTransformer.java delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/DoubleGreatestColumnTransformer.java delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/DoubleLeastColumnTransformer.java delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/FloatGreatestColumnTransformer.java delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/FloatLeastColumnTransformer.java delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/Int32GreatestColumnTransformer.java delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/Int32LeastColumnTransformer.java delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/Int64GreatestColumnTransformer.java delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/Int64LeastColumnTransformer.java delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/io/CompactionDiskTSMIterator.java delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/utils/TsFileRewriteOverPrecisedI64Scan.java delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/utils/TsFileRewriteSmallRangeI64Scan.java delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/utils/TsFileStatisticScan.java diff --git a/.github/workflows/cluster-it-1c1d.yml b/.github/workflows/cluster-it-1c1d.yml index 3fc0e6360dc9..40bbec867760 100644 --- a/.github/workflows/cluster-it-1c1d.yml +++ b/.github/workflows/cluster-it-1c1d.yml @@ -4,16 +4,17 @@ on: push: branches: - master - - 'rel/*' - - 'rc/*' + - 'rel/1.*' + - 'rc/1.*' + - 'force_ci/**' paths-ignore: - 'docs/**' - 'site/**' pull_request: branches: - master - - 'rel/*' - - 'rc/*' + - 'rel/1.*' + - 'rc/1.*' paths-ignore: - 'docs/**' - 'site/**' diff --git a/.github/workflows/cluster-it-1c1d1a.yml b/.github/workflows/cluster-it-1c1d1a.yml index 7801c13a26a4..42196570a57d 100644 --- a/.github/workflows/cluster-it-1c1d1a.yml +++ b/.github/workflows/cluster-it-1c1d1a.yml @@ -4,16 +4,17 @@ on: push: branches: - master - - 'rel/*' - - 'rc/*' + - 'rel/1.*' + - 'rc/1.*' + - 'force_ci/**' paths-ignore: - 'docs/**' - 'site/**' pull_request: branches: - master - - 'rel/*' - - 'rc/*' + - 'rel/1.*' + - 'rc/1.*' - 'force_ci/**' paths-ignore: - 'docs/**' diff --git a/.github/workflows/cluster-it-1c3d.yml b/.github/workflows/cluster-it-1c3d.yml index e889d3c98d20..fbc15aef5c09 100644 --- a/.github/workflows/cluster-it-1c3d.yml +++ b/.github/workflows/cluster-it-1c3d.yml @@ -4,16 +4,17 @@ on: push: branches: - master - - 'rel/*' - - 'rc/*' + - 'rel/1.*' + - 'rc/1.*' + - 'force_ci/**' paths-ignore: - 'docs/**' - 'site/**' pull_request: branches: - master - - 'rel/*' - - 'rc/*' + - 'rel/1.*' + - 'rc/1.*' - 'force_ci/**' paths-ignore: - 'docs/**' diff --git a/.github/workflows/compile-check.yml b/.github/workflows/compile-check.yml index 92522f48970d..d70cb70219f0 100644 --- a/.github/workflows/compile-check.yml +++ b/.github/workflows/compile-check.yml @@ -6,16 +6,17 @@ on: push: branches: - master - - 'rel/*' - - 'rc/*' + - 'rel/1.*' + - 'rc/1.*' + - 'force_ci/**' paths-ignore: - 'docs/**' - 'site/**' pull_request: branches: - master - - 'rel/*' - - 'rc/*' + - 'rel/1.*' + - 'rc/1.*' - 'force_ci/**' paths-ignore: - 'docs/**' @@ -50,4 +51,4 @@ jobs: - name: Compiler Test shell: bash run: | - mvn clean package -P with-integration-tests -DskipTests -ntp + mvn clean compile -P with-integration-tests -ntp diff --git a/.github/workflows/dependency-check.yml b/.github/workflows/dependency-check.yml index 30f29f16090c..954e9c7e52c4 100644 --- a/.github/workflows/dependency-check.yml +++ b/.github/workflows/dependency-check.yml @@ -8,6 +8,7 @@ on: - master - 'rel/*' - "rc/*" + - 'force_ci/**' paths-ignore: - 'docs/**' - 'site/**' diff --git a/.github/workflows/multi-language-client.yml b/.github/workflows/multi-language-client.yml index 095f22557d66..f3efb776b15c 100644 --- a/.github/workflows/multi-language-client.yml +++ b/.github/workflows/multi-language-client.yml @@ -4,6 +4,7 @@ on: branches: - master - "rc/*" + - 'force_ci/**' paths: - 'pom.xml' - 'iotdb-client/pom.xml' diff --git a/.github/workflows/pipe-it.yml b/.github/workflows/pipe-it-2cluster.yml similarity index 62% rename from .github/workflows/pipe-it.yml rename to .github/workflows/pipe-it-2cluster.yml index 9e7854073af1..9be00874afbd 100644 --- a/.github/workflows/pipe-it.yml +++ b/.github/workflows/pipe-it-2cluster.yml @@ -4,8 +4,9 @@ on: push: branches: - master - - 'rel/*' - - 'rc/*' + - 'rel/1.*' + - 'rc/1.*' + - 'force_ci/**' paths-ignore: - 'docs/**' - 'site/**' @@ -13,8 +14,8 @@ on: pull_request: branches: - master - - 'rel/*' - - 'rc/*' + - 'rel/1.*' + - 'rc/1.*' - 'force_ci/**' paths-ignore: - 'docs/**' @@ -33,110 +34,7 @@ env: DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} jobs: - single: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - java: [17] - # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet. - cluster1: [LightWeightStandaloneMode, ScalableSingleNodeMode, HighPerformanceMode, PipeConsensusBatchMode, PipeConsensusStreamMode] - cluster2: [LightWeightStandaloneMode, ScalableSingleNodeMode, HighPerformanceMode] - os: [ ubuntu-latest ] - exclude: - - cluster1: LightWeightStandaloneMode - cluster2: LightWeightStandaloneMode - - cluster1: LightWeightStandaloneMode - cluster2: ScalableSingleNodeMode - - cluster1: ScalableSingleNodeMode - cluster2: LightWeightStandaloneMode - - cluster1: ScalableSingleNodeMode - cluster2: HighPerformanceMode - - cluster1: HighPerformanceMode - cluster2: LightWeightStandaloneMode - - cluster1: HighPerformanceMode - cluster2: HighPerformanceMode - - cluster1: PipeConsensusBatchMode - cluster2: LightWeightStandaloneMode - - cluster1: PipeConsensusBatchMode - cluster2: HighPerformanceMode - - cluster1: PipeConsensusStreamMode - cluster2: LightWeightStandaloneMode - - cluster1: PipeConsensusStreamMode - cluster2: HighPerformanceMode - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v4 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v4 - with: - distribution: liberica - java-version: ${{ matrix.java }} - - name: Cache Maven packages - uses: actions/cache@v4 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Sleep for a random duration between 0 and 10000 milliseconds - run: | - sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) - - name: IT Test - shell: bash - # we do not compile client-cpp for saving time, it is tested in client.yml - # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml - run: | - retry() { - local -i max_attempts=3 - local -i attempt=1 - local -i retry_sleep=5 - local test_output - - while [ $attempt -le $max_attempts ]; do - mvn clean verify \ - -P with-integration-tests \ - -DskipUTs \ - -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ - -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \ - -pl integration-test \ - -am -PMultiClusterIT1 \ - -ntp >> ~/run-tests-$attempt.log && return 0 - test_output=$(cat ~/run-tests-$attempt.log) - - mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ - - echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" - echo "$test_output" - echo "==================== END: ~/run-tests-$attempt.log ======================" - - if echo "$test_output" | grep -q "Could not transfer artifact"; then - if [ $attempt -lt $max_attempts ]; then - echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." - sleep $retry_sleep - attempt=$((attempt + 1)) - else - echo "Test failed after $max_attempts attempts due to artifact transfer issue." - echo "Treating this as a success because the issue is likely transient." - return 0 - fi - elif [ $? -ne 0 ]; then - echo "Test failed with a different error." - return 1 - else - echo "Tests passed" - return 0 - fi - done - } - retry - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v4 - with: - name: cluster-log-single-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }} - path: integration-test/target/cluster-logs - retention-days: 30 - dual-tree-auto-basic: + auto-create-schema: strategy: fail-fast: false max-parallel: 15 @@ -180,7 +78,7 @@ jobs: -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ -DClusterConfigurations=${{ matrix.cluster }},${{ matrix.cluster }} \ -pl integration-test \ - -am -PMultiClusterIT2DualTreeAutoBasic \ + -am -PMultiClusterIT2AutoCreateSchema \ -ntp >> ~/run-tests-$attempt.log && return 0 test_output=$(cat ~/run-tests-$attempt.log) @@ -214,113 +112,10 @@ jobs: if: failure() uses: actions/upload-artifact@v4 with: - name: cluster-log-dual-tree-auto-basic-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster }}-${{ matrix.cluster }} - path: integration-test/target/cluster-logs - retention-days: 30 - dual-tree-auto-enhanced: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - java: [17] - # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet. - cluster1: [LightWeightStandaloneMode, ScalableSingleNodeMode, HighPerformanceMode, PipeConsensusBatchMode, PipeConsensusStreamMode] - cluster2: [LightWeightStandaloneMode, ScalableSingleNodeMode, HighPerformanceMode] - os: [ ubuntu-latest ] - exclude: - - cluster1: LightWeightStandaloneMode - cluster2: LightWeightStandaloneMode - - cluster1: LightWeightStandaloneMode - cluster2: ScalableSingleNodeMode - - cluster1: ScalableSingleNodeMode - cluster2: LightWeightStandaloneMode - - cluster1: ScalableSingleNodeMode - cluster2: HighPerformanceMode - - cluster1: HighPerformanceMode - cluster2: LightWeightStandaloneMode - - cluster1: HighPerformanceMode - cluster2: HighPerformanceMode - - cluster1: PipeConsensusBatchMode - cluster2: LightWeightStandaloneMode - - cluster1: PipeConsensusBatchMode - cluster2: HighPerformanceMode - - cluster1: PipeConsensusStreamMode - cluster2: LightWeightStandaloneMode - - cluster1: PipeConsensusStreamMode - cluster2: HighPerformanceMode - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v4 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v4 - with: - distribution: liberica - java-version: ${{ matrix.java }} - - name: Cache Maven packages - uses: actions/cache@v4 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Sleep for a random duration between 0 and 10000 milliseconds - run: | - sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) - - name: IT Test - shell: bash - # we do not compile client-cpp for saving time, it is tested in client.yml - # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml - run: | - retry() { - local -i max_attempts=3 - local -i attempt=1 - local -i retry_sleep=5 - local test_output - - while [ $attempt -le $max_attempts ]; do - mvn clean verify \ - -P with-integration-tests \ - -DskipUTs \ - -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ - -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \ - -pl integration-test \ - -am -PMultiClusterIT2DualTreeAutoEnhanced \ - -ntp >> ~/run-tests-$attempt.log && return 0 - test_output=$(cat ~/run-tests-$attempt.log) - - mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ - - echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" - echo "$test_output" - echo "==================== END: ~/run-tests-$attempt.log ======================" - - if echo "$test_output" | grep -q "Could not transfer artifact"; then - if [ $attempt -lt $max_attempts ]; then - echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." - sleep $retry_sleep - attempt=$((attempt + 1)) - else - echo "Test failed after $max_attempts attempts due to artifact transfer issue." - echo "Treating this as a success because the issue is likely transient." - return 0 - fi - elif [ $? -ne 0 ]; then - echo "Test failed with a different error." - return 1 - else - echo "Tests passed" - return 0 - fi - done - } - retry - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v4 - with: - name: cluster-log-dual-tree-auto-enhanced-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }} + name: cluster-log-auto-create-schema-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster }}-${{ matrix.cluster }} path: integration-test/target/cluster-logs retention-days: 30 - dual-tree-manual: + manual-create-schema: strategy: fail-fast: false max-parallel: 15 @@ -386,7 +181,7 @@ jobs: -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \ -pl integration-test \ - -am -PMultiClusterIT2DualTreeManual \ + -am -PMultiClusterIT2ManualCreateSchema \ -ntp >> ~/run-tests-$attempt.log && return 0 test_output=$(cat ~/run-tests-$attempt.log) @@ -420,7 +215,7 @@ jobs: if: failure() uses: actions/upload-artifact@v4 with: - name: cluster-log-dual-tree-manual-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }} + name: cluster-log-manual-create-schema-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }} path: integration-test/target/cluster-logs retention-days: 30 subscription-arch-verification: @@ -669,7 +464,7 @@ jobs: name: cluster-log-subscription-regression-misc-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }} path: integration-test/target/cluster-logs retention-days: 30 - dual-table-manual-basic: + table-model: strategy: fail-fast: false max-parallel: 15 @@ -713,7 +508,7 @@ jobs: -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ -DClusterConfigurations=${{ matrix.cluster }},${{ matrix.cluster }} \ -pl integration-test \ - -am -PMultiClusterIT2DualTableManualBasic \ + -am -PMultiClusterIT2TableModel \ -ntp >> ~/run-tests-$attempt.log && return 0 test_output=$(cat ~/run-tests-$attempt.log) @@ -747,87 +542,6 @@ jobs: if: failure() uses: actions/upload-artifact@v4 with: - name: cluster-log-dual-table-manual-basic-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster }}-${{ matrix.cluster }} + name: cluster-log-table-model-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster }}-${{ matrix.cluster }} path: integration-test/target/cluster-logs retention-days: 30 - dual-table-manual-enhanced: - strategy: - fail-fast: false - max-parallel: 15 - matrix: - java: [17] - # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet. - cluster: [LightWeightStandaloneMode, ScalableSingleNodeMode, HighPerformanceMode, PipeConsensusBatchMode, PipeConsensusStreamMode] - os: [ ubuntu-latest ] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v4 - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v4 - with: - distribution: liberica - java-version: ${{ matrix.java }} - - name: Cache Maven packages - uses: actions/cache@v4 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2- - - name: Sleep for a random duration between 0 and 10000 milliseconds - run: | - sleep $(( $(( RANDOM % 10000 + 1 )) / 1000)) - - name: IT Test - shell: bash - # we do not compile client-cpp for saving time, it is tested in client.yml - # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml - run: | - retry() { - local -i max_attempts=3 - local -i attempt=1 - local -i retry_sleep=5 - local test_output - - while [ $attempt -le $max_attempts ]; do - mvn clean verify \ - -P with-integration-tests \ - -DskipUTs \ - -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \ - -DClusterConfigurations=${{ matrix.cluster }},${{ matrix.cluster }} \ - -pl integration-test \ - -am -PMultiClusterIT2DualTableManualEnhanced \ - -ntp >> ~/run-tests-$attempt.log && return 0 - test_output=$(cat ~/run-tests-$attempt.log) - - mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ - - echo "==================== BEGIN: ~/run-tests-$attempt.log ====================" - echo "$test_output" - echo "==================== END: ~/run-tests-$attempt.log ======================" - - if echo "$test_output" | grep -q "Could not transfer artifact"; then - if [ $attempt -lt $max_attempts ]; then - echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..." - sleep $retry_sleep - attempt=$((attempt + 1)) - else - echo "Test failed after $max_attempts attempts due to artifact transfer issue." - echo "Treating this as a success because the issue is likely transient." - return 0 - fi - elif [ $? -ne 0 ]; then - echo "Test failed with a different error." - return 1 - else - echo "Tests passed" - return 0 - fi - done - } - retry - - name: Upload Artifact - if: failure() - uses: actions/upload-artifact@v4 - with: - name: cluster-log-dual-table-manual-enhanced-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster }}-${{ matrix.cluster }} - path: integration-test/target/cluster-logs - retention-days: 30 \ No newline at end of file diff --git a/.github/workflows/sonar-codecov.yml b/.github/workflows/sonar-codecov.yml index 638bf34a3c53..5818a0ec7c25 100644 --- a/.github/workflows/sonar-codecov.yml +++ b/.github/workflows/sonar-codecov.yml @@ -9,6 +9,7 @@ on: - master - "rel/*" - "rc/*" + - 'force_ci/**' paths-ignore: - "docs/**" - 'site/**' diff --git a/.github/workflows/table-cluster-it-1c1d.yml b/.github/workflows/table-cluster-it-1c1d.yml index b04101c6479b..d34b893834da 100644 --- a/.github/workflows/table-cluster-it-1c1d.yml +++ b/.github/workflows/table-cluster-it-1c1d.yml @@ -4,16 +4,17 @@ on: push: branches: - master - - 'rel/*' - - 'rc/*' + - 'rel/1.*' + - 'rc/1.*' + - 'force_ci/**' paths-ignore: - 'docs/**' - 'site/**' pull_request: branches: - master - - 'rel/*' - - 'rc/*' + - 'rel/1.*' + - 'rc/1.*' - 'force_ci/**' paths-ignore: - 'docs/**' diff --git a/.github/workflows/table-cluster-it-1c3d.yml b/.github/workflows/table-cluster-it-1c3d.yml index a2603dd79138..09e604478964 100644 --- a/.github/workflows/table-cluster-it-1c3d.yml +++ b/.github/workflows/table-cluster-it-1c3d.yml @@ -4,16 +4,17 @@ on: push: branches: - master - - 'rel/*' - - 'rc/*' + - 'rel/1.*' + - 'rc/1.*' + - 'force_ci/**' paths-ignore: - 'docs/**' - 'site/**' pull_request: branches: - master - - 'rel/*' - - 'rc/*' + - 'rel/1.*' + - 'rc/1.*' - 'force_ci/**' paths-ignore: - 'docs/**' diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml index 34fb0262f995..565ce10dd0f8 100644 --- a/.github/workflows/unit-test.yml +++ b/.github/workflows/unit-test.yml @@ -9,6 +9,7 @@ on: - master - 'rel/*' - "rc/*" + - 'force_ci/**' paths-ignore: - 'docs/**' - 'site/**' diff --git a/code-coverage/pom.xml b/code-coverage/pom.xml index 02f7d8f3f09d..f7a1f82d5f16 100644 --- a/code-coverage/pom.xml +++ b/code-coverage/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-parent - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT iotdb-code-coverage pom diff --git a/distribution/pom.xml b/distribution/pom.xml index 22f3af9c6fe8..87a457dc2a61 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-parent - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT iotdb-distribution pom @@ -33,25 +33,25 @@ org.apache.iotdb iotdb-server - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT zip org.apache.iotdb iotdb-cli - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT zip org.apache.iotdb iotdb-confignode - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT zip org.apache.iotdb library-udf - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT @@ -174,7 +174,7 @@ org.apache.iotdb iotdb-ainode - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT diff --git a/docker/src/main/DockerCompose/entrypoint.sh b/docker/src/main/DockerCompose/entrypoint.sh index 68df9db78b77..72beea010eeb 100755 --- a/docker/src/main/DockerCompose/entrypoint.sh +++ b/docker/src/main/DockerCompose/entrypoint.sh @@ -26,8 +26,11 @@ function on_stop(){ if [[ "$start_what" != "confignode" ]]; then echo "###### manually flush ######"; start-cli.sh -e "flush;" || true + stop-datanode.sh + echo "##### done ######"; + else + stop-confignode.sh; fi - stop-standalone.sh } trap 'on_stop' SIGTERM SIGKILL SIGQUIT diff --git a/docker/src/main/DockerCompose/replace-conf-from-env.sh b/docker/src/main/DockerCompose/replace-conf-from-env.sh index eb183033119c..dc3d6cf2cf11 100755 --- a/docker/src/main/DockerCompose/replace-conf-from-env.sh +++ b/docker/src/main/DockerCompose/replace-conf-from-env.sh @@ -21,37 +21,36 @@ conf_path=${IOTDB_HOME}/conf target_files="iotdb-system.properties" -function process_single(){ - local key_value="$1" - local filename=$2 - local key=$(echo $key_value|cut -d = -f1) - local line=$(grep -ni "${key}=" ${filename}) - #echo "line=$line" - if [[ -n "${line}" ]]; then +function process_single() { + local key_value="$1" + local filename=$2 + local key=$(echo $key_value | cut -d = -f1) + local line=$(grep -ni "${key}=" ${filename}) + #echo "line=$line" + if [[ -n "${line}" ]]; then echo "update $key $filename" - local line_no=$(echo $line|cut -d : -f1) - local content=$(echo $line|cut -d : -f2) + local line_no=$(echo $line | cut -d : -f1) + local content=$(echo $line | cut -d : -f2) if [[ "${content:0:1}" != "#" ]]; then sed -i "${line_no}d" ${filename} fi sed -i "${line_no} i${key_value}" ${filename} else - echo "append $key $filename" - line_no=$(wc -l $filename) - sed -i "${line_no} a${key_value}" ${filename} - fi + echo "append $key to $filename" + + echo "${key_value}" >>"${filename}" + fi } -function replace_configs(){ +function replace_configs() { for v in $(env); do if [[ "${v}" =~ "=" && "${v}" =~ "_" && ! "${v}" =~ "JAVA_" ]]; then -# echo "###### $v ####" + # echo "###### $v ####" for f in ${target_files}; do - process_single $v ${conf_path}/$f + process_single $v ${conf_path}/$f done fi done } replace_configs - diff --git a/example/client-cpp-example/pom.xml b/example/client-cpp-example/pom.xml index b19e84c13dca..4832533b9aab 100644 --- a/example/client-cpp-example/pom.xml +++ b/example/client-cpp-example/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-examples - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT client-cpp-example IoTDB: Example: CPP Client diff --git a/example/jdbc/pom.xml b/example/jdbc/pom.xml index d8fa0a05e22f..73833335f0ee 100644 --- a/example/jdbc/pom.xml +++ b/example/jdbc/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-examples - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT jdbc-example IoTDB: Example: JDBC diff --git a/example/mqtt-customize/pom.xml b/example/mqtt-customize/pom.xml index 4d05d14dc57a..bd0c9e6a7c14 100644 --- a/example/mqtt-customize/pom.xml +++ b/example/mqtt-customize/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-examples - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT customize-mqtt-example IoTDB: Example: Customized MQTT diff --git a/example/mqtt/pom.xml b/example/mqtt/pom.xml index 18d2e83a0729..892fa11c5456 100644 --- a/example/mqtt/pom.xml +++ b/example/mqtt/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-examples - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT mqtt-example IoTDB: Example: MQTT diff --git a/example/pipe-count-point-processor/pom.xml b/example/pipe-count-point-processor/pom.xml index 935a5bca588d..4ba2d8c51217 100644 --- a/example/pipe-count-point-processor/pom.xml +++ b/example/pipe-count-point-processor/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-examples - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT pipe-count-point-processor-example IoTDB: Example: Pipe: Count Point Processor diff --git a/example/pipe-opc-ua-sink/pom.xml b/example/pipe-opc-ua-sink/pom.xml index 3b3ef78c7cef..7846ab5e24ab 100644 --- a/example/pipe-opc-ua-sink/pom.xml +++ b/example/pipe-opc-ua-sink/pom.xml @@ -23,7 +23,7 @@ org.apache.iotdb iotdb-examples - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT 4.0.0 pipe-opc-ua-sink-example diff --git a/example/pom.xml b/example/pom.xml index 9912b7760909..c8eaeb7cd90d 100644 --- a/example/pom.xml +++ b/example/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-parent - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT iotdb-examples pom diff --git a/example/rest-java-example/pom.xml b/example/rest-java-example/pom.xml index 8af4b94104cf..2fc64e72b60b 100644 --- a/example/rest-java-example/pom.xml +++ b/example/rest-java-example/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-examples - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT rest-java-example IoTDB: Example: Java Rest diff --git a/example/schema/pom.xml b/example/schema/pom.xml index aef252aede13..b2e927742c26 100644 --- a/example/schema/pom.xml +++ b/example/schema/pom.xml @@ -24,7 +24,7 @@ iotdb-examples org.apache.iotdb - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT schema-example IoTDB: Example: Schema diff --git a/example/session/pom.xml b/example/session/pom.xml index cd5f40dc913e..a7bdd7aa1241 100644 --- a/example/session/pom.xml +++ b/example/session/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-examples - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT client-example IoTDB: Example: Session Client diff --git a/example/trigger/pom.xml b/example/trigger/pom.xml index c9c57dbf652f..efe4342b58a9 100644 --- a/example/trigger/pom.xml +++ b/example/trigger/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-examples - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT trigger-example IoTDB: Example: Trigger diff --git a/example/udf/pom.xml b/example/udf/pom.xml index 9da91d7b3de9..3449e7258b7f 100644 --- a/example/udf/pom.xml +++ b/example/udf/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-examples - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT udf-example IoTDB: Example: UDF diff --git a/integration-test/pom.xml b/integration-test/pom.xml index 6438c36e3d5c..43e30ace4ea9 100644 --- a/integration-test/pom.xml +++ b/integration-test/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-parent - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT integration-test IoTDB: Integration-Test @@ -87,47 +87,47 @@ org.apache.iotdb iotdb-server - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.apache.iotdb iotdb-session - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.apache.iotdb iotdb-jdbc - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.apache.iotdb trigger-api - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.apache.iotdb isession - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.apache.iotdb service-rpc - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.apache.iotdb iotdb-thrift-confignode - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.apache.iotdb node-commons - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.apache.iotdb iotdb-thrift-commons - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.apache.tsfile @@ -137,7 +137,7 @@ org.apache.iotdb udf-api - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.apache.iotdb @@ -147,7 +147,7 @@ org.apache.iotdb iotdb-consensus - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.slf4j @@ -176,17 +176,17 @@ org.apache.iotdb iotdb-confignode - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.apache.iotdb iotdb-thrift - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.apache.iotdb iotdb-cli - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT commons-codec @@ -216,7 +216,7 @@ org.apache.iotdb iotdb-server - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT test-jar test @@ -445,7 +445,7 @@ org.apache.iotdb.itbase.category.ManualIT - org.apache.iotdb.itbase.category.MultiClusterIT1,org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoBasic,org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoEnhanced,org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeManual,org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionArchVerification,org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegressionConsumer,org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegressionMisc,org.apache.iotdb.itbase.category.MultiClusterIT3,org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualBasic,org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualEnhanced + org.apache.iotdb.itbase.category.MultiClusterIT1,org.apache.iotdb.itbase.category.MultiClusterIT2AutoCreateSchema,org.apache.iotdb.itbase.category.MultiClusterIT2ManualCreateSchema,org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionArchVerification,org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegressionConsumer,org.apache.iotdb.itbase.category.MultiClusterIT2SubscriptionRegressionMisc,org.apache.iotdb.itbase.category.MultiClusterIT3,org.apache.iotdb.itbase.category.MultiClusterIT2TableModel false true true @@ -467,13 +467,13 @@ - MultiClusterIT2DualTreeAutoBasic + MultiClusterIT2AutoCreateSchema false org.apache.iotdb.itbase.category.ManualIT - org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoBasic + org.apache.iotdb.itbase.category.MultiClusterIT2AutoCreateSchema false true true @@ -481,13 +481,13 @@ - MultiClusterIT2DualTreeAutoEnhanced + MultiClusterIT2ManualCreateSchema false org.apache.iotdb.itbase.category.ManualIT - org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoEnhanced + org.apache.iotdb.itbase.category.MultiClusterIT2ManualCreateSchema false true true @@ -495,41 +495,13 @@ - MultiClusterIT2DualTableManualBasic + MultiClusterIT2TableModel false org.apache.iotdb.itbase.category.ManualIT - org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualBasic - false - true - true - MultiCluster - - - - MultiClusterIT2DualTableManualEnhanced - - false - - - org.apache.iotdb.itbase.category.ManualIT - org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualEnhanced - false - true - true - MultiCluster - - - - MultiClusterIT2DualTreeManual - - false - - - org.apache.iotdb.itbase.category.ManualIT - org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeManual + org.apache.iotdb.itbase.category.MultiClusterIT2TableModel false true true diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/env/AbstractEnv.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/env/AbstractEnv.java index a255dde22911..acaf76e4514d 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/env/AbstractEnv.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/env/AbstractEnv.java @@ -565,18 +565,6 @@ public ITableSession getTableSessionConnection() throws IoTDBConnectionException .build(); } - @Override - public ITableSession getTableSessionConnection(String userName, String password) - throws IoTDBConnectionException { - final DataNodeWrapper dataNode = - this.dataNodeWrapperList.get(rand.nextInt(this.dataNodeWrapperList.size())); - return new TableSessionBuilder() - .nodeUrls(Collections.singletonList(dataNode.getIpAndPortString())) - .username(userName) - .password(password) - .build(); - } - @Override public ITableSession getTableSessionConnectionWithDB(final String database) throws IoTDBConnectionException { diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/remote/env/RemoteServerEnv.java b/integration-test/src/main/java/org/apache/iotdb/it/env/remote/env/RemoteServerEnv.java index f0daa951e829..b29c5b96315b 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/remote/env/RemoteServerEnv.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/remote/env/RemoteServerEnv.java @@ -320,16 +320,6 @@ public ITableSession getTableSessionConnection(List nodeUrls) .build(); } - @Override - public ITableSession getTableSessionConnection(String userName, String password) - throws IoTDBConnectionException { - return new TableSessionBuilder() - .nodeUrls(Collections.singletonList(ip_addr + ":" + port)) - .username(userName) - .password(password) - .build(); - } - @Override public ISession getSessionConnection(String userName, String password) throws IoTDBConnectionException { diff --git a/integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2DualTreeAutoEnhanced.java b/integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2AutoCreateSchema.java similarity index 94% rename from integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2DualTreeAutoEnhanced.java rename to integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2AutoCreateSchema.java index 7ce45f738104..25000e3012ae 100644 --- a/integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2DualTreeAutoEnhanced.java +++ b/integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2AutoCreateSchema.java @@ -19,4 +19,4 @@ package org.apache.iotdb.itbase.category; -public class MultiClusterIT2DualTreeAutoEnhanced {} +public interface MultiClusterIT2AutoCreateSchema {} diff --git a/integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2DualTableManualEnhanced.java b/integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2DualTableManualEnhanced.java deleted file mode 100644 index 701401c26854..000000000000 --- a/integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2DualTableManualEnhanced.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.itbase.category; - -public class MultiClusterIT2DualTableManualEnhanced {} diff --git a/integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2DualTreeAutoBasic.java b/integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2DualTreeAutoBasic.java deleted file mode 100644 index 477a606584b6..000000000000 --- a/integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2DualTreeAutoBasic.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.itbase.category; - -public class MultiClusterIT2DualTreeAutoBasic {} diff --git a/integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2DualTableManualBasic.java b/integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2ManualCreateSchema.java similarity index 94% rename from integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2DualTableManualBasic.java rename to integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2ManualCreateSchema.java index 07eb001fcde3..2fb36dd22920 100644 --- a/integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2DualTableManualBasic.java +++ b/integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2ManualCreateSchema.java @@ -19,4 +19,4 @@ package org.apache.iotdb.itbase.category; -public class MultiClusterIT2DualTableManualBasic {} +public interface MultiClusterIT2ManualCreateSchema {} diff --git a/integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2DualTreeManual.java b/integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2TableModel.java similarity index 94% rename from integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2DualTreeManual.java rename to integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2TableModel.java index b9e3e2f99d12..c9280cd3fc60 100644 --- a/integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2DualTreeManual.java +++ b/integration-test/src/main/java/org/apache/iotdb/itbase/category/MultiClusterIT2TableModel.java @@ -19,4 +19,4 @@ package org.apache.iotdb.itbase.category; -public class MultiClusterIT2DualTreeManual {} +public class MultiClusterIT2TableModel {} diff --git a/integration-test/src/main/java/org/apache/iotdb/itbase/env/BaseEnv.java b/integration-test/src/main/java/org/apache/iotdb/itbase/env/BaseEnv.java index 0cae6d1fa254..ea044ddfd9de 100644 --- a/integration-test/src/main/java/org/apache/iotdb/itbase/env/BaseEnv.java +++ b/integration-test/src/main/java/org/apache/iotdb/itbase/env/BaseEnv.java @@ -210,9 +210,6 @@ IConfigNodeRPCService.Iface getLeaderConfigNodeConnection() ITableSession getTableSessionConnection(List nodeUrls) throws IoTDBConnectionException; - ITableSession getTableSessionConnection(String userName, String password) - throws IoTDBConnectionException; - /** * Get the index of the first dataNode with a SchemaRegion leader. * diff --git a/integration-test/src/main/java/org/apache/iotdb/itbase/runtime/ClusterTestStatement.java b/integration-test/src/main/java/org/apache/iotdb/itbase/runtime/ClusterTestStatement.java index f5dc5289d126..d99ea24717e3 100644 --- a/integration-test/src/main/java/org/apache/iotdb/itbase/runtime/ClusterTestStatement.java +++ b/integration-test/src/main/java/org/apache/iotdb/itbase/runtime/ClusterTestStatement.java @@ -31,7 +31,6 @@ import java.util.ArrayList; import java.util.List; -import static org.apache.iotdb.rpc.RpcUtils.isSetSqlDialect; import static org.apache.iotdb.rpc.RpcUtils.isUseDatabase; /** The implementation of {@link ClusterTestStatement} in cluster test. */ @@ -192,8 +191,8 @@ public void setCursorName(String name) { public boolean execute(String sql) throws SQLException { sql = sql.trim(); boolean result = writeStatement.execute(sql); - // if 'use XXXX' or 'set sql_dialect', sendRequest to all statements - if (isUseDatabase(sql) || isSetSqlDialect(sql)) { + // if use XXXX, sendRequest to all statements + if (isUseDatabase(sql)) { for (Statement readStatement : readStatements) { boolean tmp = readStatement.execute(sql); result = result && tmp; diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBInsertWithQueryIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBInsertWithQueryIT.java index a5032580f225..e9823c505b13 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBInsertWithQueryIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBInsertWithQueryIT.java @@ -52,7 +52,6 @@ public class IoTDBInsertWithQueryIT { @Before public void setUp() throws Exception { - EnvFactory.getEnv().getConfig().getCommonConfig().setTimestampPrecisionCheckEnabled(false); EnvFactory.getEnv().initClusterEnvironment(); } diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java index 3e090f99fa56..b8e431356f55 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java @@ -21,8 +21,6 @@ import org.apache.iotdb.commons.auth.entity.PrivilegeType; import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant; -import org.apache.iotdb.db.storageengine.dataregion.compaction.tablemodel.CompactionTableModelTestFileWriter; -import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.it.utils.TsFileGenerator; @@ -33,10 +31,8 @@ import org.apache.iotdb.jdbc.IoTDBSQLException; import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.read.common.Path; -import org.apache.tsfile.read.common.TimeRange; import org.apache.tsfile.utils.Pair; import org.apache.tsfile.write.record.Tablet; import org.apache.tsfile.write.schema.IMeasurementSchema; @@ -500,75 +496,6 @@ public void testAuth() throws Exception { String.format("load \"%s\" sgLevel=2", tmpDir.getAbsolutePath()), "test", "test123"); } - @Test - public void testTableAuth() throws Exception { - createUser("test", "test123"); - - final TsFileResource resource4 = new TsFileResource(new File(tmpDir, "test1-0-0-0.tsfile")); - try (final CompactionTableModelTestFileWriter writer = - new CompactionTableModelTestFileWriter(resource4)) { - writer.registerTableSchema("t2", Arrays.asList("id1", "id2", "id3")); - writer.startChunkGroup("t2", Arrays.asList("id_field1", "id_field2", "id_field3")); - writer.generateSimpleNonAlignedSeriesToCurrentDevice( - "s1", - new TimeRange[][][] {new TimeRange[][] {new TimeRange[] {new TimeRange(20, 22)}}}, - TSEncoding.PLAIN, - CompressionType.LZ4); - writer.generateSimpleNonAlignedSeriesToCurrentDevice( - "s2", - new TimeRange[][][] {new TimeRange[][] {new TimeRange[] {new TimeRange(20, 22)}}}, - TSEncoding.PLAIN, - CompressionType.LZ4); - writer.endChunkGroup(); - writer.endFile(); - } - - try (final Connection userCon = - EnvFactory.getEnv().getConnection("test", "test123", BaseEnv.TABLE_SQL_DIALECT); - final Statement userStmt = userCon.createStatement()) { - Assert.assertThrows( - SQLException.class, - () -> { - userStmt.execute( - String.format( - "load '%s' with ('database-level'='2', 'database-name'='test')", - tmpDir.getAbsolutePath())); - }); - } - - try (Connection adminCon = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); - Statement adminStmt = adminCon.createStatement()) { - adminStmt.execute("grant create on database test to user test"); - } - - try (final Connection userCon = - EnvFactory.getEnv().getConnection("test", "test123", BaseEnv.TABLE_SQL_DIALECT); - final Statement userStmt = userCon.createStatement()) { - Assert.assertThrows( - SQLException.class, - () -> { - userStmt.execute( - String.format( - "load '%s' with ('database-level'='2', 'database-name'='test')", - tmpDir.getAbsolutePath())); - }); - } - - try (final Connection adminCon = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); - final Statement adminStmt = adminCon.createStatement()) { - adminStmt.execute("grant insert on any to user test"); - } - - try (final Connection userCon = - EnvFactory.getEnv().getConnection("test", "test123", BaseEnv.TABLE_SQL_DIALECT); - final Statement userStmt = userCon.createStatement()) { - userStmt.execute( - String.format( - "load '%s' with ('database-level'='2', 'database-name'='test')", - tmpDir.getAbsolutePath())); - } - } - @Test public void testLoadWithOnSuccess() throws Exception { final File file1 = new File(tmpDir, "1-0-0-0.tsfile"); diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/auth/IoTDBAuthIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/auth/IoTDBAuthIT.java index 301de48bedae..277dda1fa10d 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/auth/IoTDBAuthIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/auth/IoTDBAuthIT.java @@ -21,7 +21,6 @@ import org.apache.iotdb.commons.auth.entity.PrivilegeType; import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant; -import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.ClusterIT; @@ -47,8 +46,6 @@ import java.util.List; import java.util.Set; -import static org.apache.iotdb.db.it.utils.TestUtils.createUser; -import static org.apache.iotdb.db.it.utils.TestUtils.resultSetEqualTest; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -158,30 +155,6 @@ public void allPrivilegesTest() throws SQLException { Assert.assertThrows( SQLException.class, () -> userStmt.execute("GRANT USER tempuser PRIVILEGES WRITE_SCHEMA ON root.a")); - - adminStmt.execute("GRANT ALL ON root.** TO USER tempuser WITH GRANT OPTION"); - userStmt.execute("CREATE USER testuser 'password'"); - userStmt.execute("GRANT ALL ON root.** TO USER testuser WITH GRANT OPTION"); - ResultSet dataSet = userStmt.executeQuery("LIST PRIVILEGES OF USER testuser"); - - Set ansSet = - new HashSet<>( - Arrays.asList( - ",,MANAGE_USER,true,", - ",,MANAGE_ROLE,true,", - ",,USE_TRIGGER,true,", - ",,USE_UDF,true,", - ",,USE_CQ,true,", - ",,USE_PIPE,true,", - ",,USE_MODEL,true,", - ",,EXTEND_TEMPLATE,true,", - ",,MANAGE_DATABASE,true,", - ",,MAINTAIN,true,", - ",root.**,READ_DATA,true,", - ",root.**,WRITE_DATA,true,", - ",root.**,READ_SCHEMA,true,", - ",root.**,WRITE_SCHEMA,true,")); - TestUtils.assertResultSetEqual(dataSet, "Role,Scope,Privileges,GrantOption,", ansSet); } } } @@ -1280,15 +1253,4 @@ public void testCreateRoleIdentifierName() throws SQLException { adminStmt.execute("create role tail"); adminStmt.execute("create user tail 'password'"); } - - @Test - public void noNeedPrivilegeTest() { - createUser("tempuser", "temppw"); - String[] expectedHeader = new String[] {"CurrentUser"}; - String[] retArray = - new String[] { - "tempuser,", - }; - resultSetEqualTest("show current_user", expectedHeader, retArray, "tempuser", "temppw"); - } } diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/auth/IoTDBRelationalAuthIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/auth/IoTDBRelationalAuthIT.java index 2454ec95c7b3..dfad35cb6963 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/auth/IoTDBRelationalAuthIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/auth/IoTDBRelationalAuthIT.java @@ -106,12 +106,6 @@ public void listUserPrivileges() throws SQLException { ",testdb.tb,INSERT,true,", ",testdb.tb,DROP,true,")); TestUtils.assertResultSetEqual(rs, "Role,Scope,Privileges,GrantOption,", ans); - adminStmt.execute("create role testrole"); - adminStmt.execute("GRANT ROLE testrole to testuser"); - rs = adminStmt.executeQuery("LIST USER OF ROLE testrole"); - TestUtils.assertResultSetEqual(rs, "User,", Collections.singleton("testuser,")); - rs = adminStmt.executeQuery("LIST ROLE OF USER testuser"); - TestUtils.assertResultSetEqual(rs, "Role,", Collections.singleton("testrole,")); } } @@ -293,168 +287,6 @@ public void checkAuthorStatementPrivilegeCheck() throws SQLException { @Test public void checkGrantRevokeAllPrivileges() throws SQLException { - // In this IT: - // grant - // 1. grant all on table tb1 with grant option - // 2. grant all on database testdb - // 3. grant all on any - // revoke - // 1. revoke grant option for all on table tb1 - // 2. revoke all on table tb1 - // 3. revoke all on database testdb - // 4. revoke all on any - // grant and revoke - // 1. grant all on user/role - // 2. revoke all on any - // 3. revoke all on user/role - - for (boolean isUser : new boolean[] {true, false}) { - try (Connection adminCon = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); - Statement adminStmt = adminCon.createStatement()) { - adminStmt.execute("create database testdb"); - adminStmt.execute(isUser ? "create user test 'password'" : "create role test"); - adminStmt.execute("use testdb"); - - // 1. grant all on table tb1 with grant option - adminStmt.execute( - "grant all on table tb1 to " - + (isUser ? "user test" : "role test") - + " with grant option"); - Set listPrivilegeResult = new HashSet<>(); - for (PrivilegeType privilegeType : PrivilegeType.values()) { - if (privilegeType.isRelationalPrivilege()) { - listPrivilegeResult.add( - (isUser ? "," : "test,") + "testdb.tb1," + privilegeType + ",true,"); - } - } - ResultSet resultSet = - adminStmt.executeQuery("List privileges of " + (isUser ? "user test" : "role test")); - TestUtils.assertResultSetEqual( - resultSet, "Role,Scope,Privileges,GrantOption,", listPrivilegeResult); - - // 2. grant all on database testdb - adminStmt.execute( - "grant all on database testdb to " + (isUser ? "user test" : "role test")); - for (PrivilegeType privilegeType : PrivilegeType.values()) { - if (privilegeType.isRelationalPrivilege()) { - listPrivilegeResult.add( - (isUser ? "," : "test,") + "testdb.*," + privilegeType + ",false,"); - } - } - resultSet = - adminStmt.executeQuery("List privileges of " + (isUser ? "user test" : "role test")); - TestUtils.assertResultSetEqual( - resultSet, "Role,Scope,Privileges,GrantOption,", listPrivilegeResult); - - // 3. grant all on any - adminStmt.execute("grant all on any to " + (isUser ? "user test" : "role test")); - for (PrivilegeType privilegeType : PrivilegeType.values()) { - if (privilegeType.isRelationalPrivilege()) { - listPrivilegeResult.add((isUser ? "," : "test,") + "*.*," + privilegeType + ",false,"); - } - } - resultSet = - adminStmt.executeQuery("List privileges of " + (isUser ? "user test" : "role test")); - TestUtils.assertResultSetEqual( - resultSet, "Role,Scope,Privileges,GrantOption,", listPrivilegeResult); - - // 1. revoke grant option for all on table tb1 - adminStmt.execute( - "revoke grant option for all on table tb1 from " - + (isUser ? "user test" : "role test")); - for (PrivilegeType privilegeType : PrivilegeType.values()) { - if (privilegeType.isRelationalPrivilege()) { - listPrivilegeResult.remove( - (isUser ? "," : "test,") + "testdb.tb1," + privilegeType + ",true,"); - listPrivilegeResult.add( - (isUser ? "," : "test,") + "testdb.tb1," + privilegeType + ",false,"); - } - } - resultSet = - adminStmt.executeQuery("List privileges of " + (isUser ? "user test" : "role test")); - TestUtils.assertResultSetEqual( - resultSet, "Role,Scope,Privileges,GrantOption,", listPrivilegeResult); - - // 2. revoke all on table tb1 - adminStmt.execute("revoke all on table tb1 from " + (isUser ? "user test" : "role test")); - for (PrivilegeType privilegeType : PrivilegeType.values()) { - if (privilegeType.isRelationalPrivilege()) { - listPrivilegeResult.remove( - (isUser ? "," : "test,") + "testdb.tb1," + privilegeType + ",false,"); - } - } - resultSet = - adminStmt.executeQuery("List privileges of " + (isUser ? "user test" : "role test")); - TestUtils.assertResultSetEqual( - resultSet, "Role,Scope,Privileges,GrantOption,", listPrivilegeResult); - - // 3. revoke all on database testdb - adminStmt.execute( - "revoke all on database testdb from " + (isUser ? "user test" : "role test")); - for (PrivilegeType privilegeType : PrivilegeType.values()) { - if (privilegeType.isRelationalPrivilege()) { - listPrivilegeResult.remove( - (isUser ? "," : "test,") + "testdb.*," + privilegeType + ",false,"); - } - } - resultSet = - adminStmt.executeQuery("List privileges of " + (isUser ? "user test" : "role test")); - TestUtils.assertResultSetEqual( - resultSet, "Role,Scope,Privileges,GrantOption,", listPrivilegeResult); - - // 4. revoke all on any - adminStmt.execute("revoke all on any from " + (isUser ? "user test" : "role test")); - for (PrivilegeType privilegeType : PrivilegeType.values()) { - if (privilegeType.isRelationalPrivilege()) { - listPrivilegeResult.remove( - (isUser ? "," : "test,") + "*.*," + privilegeType + ",false,"); - } - } - Assert.assertTrue(listPrivilegeResult.isEmpty()); - resultSet = - adminStmt.executeQuery("List privileges of " + (isUser ? "user test" : "role test")); - TestUtils.assertResultSetEqual( - resultSet, "Role,Scope,Privileges,GrantOption,", listPrivilegeResult); - - // 1. grant all on user/role - adminStmt.execute("grant all to " + (isUser ? "user test" : "role test")); - for (PrivilegeType privilegeType : PrivilegeType.values()) { - if (privilegeType.isRelationalPrivilege()) { - listPrivilegeResult.add((isUser ? "," : "test,") + "*.*," + privilegeType + ",false,"); - } else if (privilegeType.forRelationalSys()) { - listPrivilegeResult.add((isUser ? "," : "test,") + "," + privilegeType + ",false,"); - } - } - resultSet = - adminStmt.executeQuery("List privileges of " + (isUser ? "user test" : "role test")); - TestUtils.assertResultSetEqual( - resultSet, "Role,Scope,Privileges,GrantOption,", listPrivilegeResult); - - // 2. revoke all on any - adminStmt.execute("revoke all on any from " + (isUser ? "user test" : "role test")); - for (PrivilegeType privilegeType : PrivilegeType.values()) { - if (privilegeType.isRelationalPrivilege()) { - listPrivilegeResult.remove( - (isUser ? "," : "test,") + "*.*," + privilegeType + ",false,"); - } - } - resultSet = - adminStmt.executeQuery("List privileges of " + (isUser ? "user test" : "role test")); - TestUtils.assertResultSetEqual( - resultSet, "Role,Scope,Privileges,GrantOption,", listPrivilegeResult); - - // 3. revoke all on user/role - adminStmt.execute("revoke all from " + (isUser ? "user test" : "role test")); - listPrivilegeResult.clear(); - resultSet = - adminStmt.executeQuery("List privileges of " + (isUser ? "user test" : "role test")); - TestUtils.assertResultSetEqual( - resultSet, "Role,Scope,Privileges,GrantOption,", listPrivilegeResult); - adminStmt.execute("drop database testdb"); - adminStmt.execute(isUser ? "drop user test" : "drop role test"); - } - } - try (Connection adminCon = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); Statement adminStmt = adminCon.createStatement()) { adminStmt.execute("create user test 'password'"); @@ -513,21 +345,6 @@ public void checkGrantRevokeAllPrivileges() throws SQLException { ResultSet resultSet = adminStmt.executeQuery("List privileges of user test"); TestUtils.assertResultSetEqual( resultSet, "Role,Scope,Privileges,GrantOption,", Collections.emptySet()); - adminStmt.execute("GRANT ALL ON db1.test TO USER test"); - adminStmt.execute("GRANT ALL ON DATABASE db2 TO USER test with grant option"); - resultSet = adminStmt.executeQuery("List privileges of user test"); - Set resultSetALL = new HashSet<>(); - for (PrivilegeType privilegeType : PrivilegeType.values()) { - if (privilegeType.isRelationalPrivilege()) { - resultSetALL.add(",db2.*," + privilegeType + ",true,"); - resultSetALL.add(",db1.test," + privilegeType + ",false,"); - } - } - TestUtils.assertResultSetEqual(resultSet, "Role,Scope,Privileges,GrantOption,", resultSetALL); - adminStmt.execute("REVOKE ALL FROM USER test"); - resultSet = adminStmt.executeQuery("List privileges of user test"); - TestUtils.assertResultSetEqual( - resultSet, "Role,Scope,Privileges,GrantOption,", Collections.emptySet()); } } } diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/auth/IoTDBSystemPermissionIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/auth/IoTDBSystemPermissionIT.java index 2b04fcc64b19..4b3cebd68f1b 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/auth/IoTDBSystemPermissionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/auth/IoTDBSystemPermissionIT.java @@ -219,7 +219,7 @@ public void maintainOperationsTest() { executeNonQuery("show queries", "test", "test123"); assertNonQueryTestFail( "kill query 'test'", - "701: Please ensure your input is correct", + "305: Please ensure your input is correct", "test", "test123"); executeNonQuery("show cluster", "test", "test123"); diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBQueryDemoIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBQueryDemoIT.java index 2397e49c1dc0..9fc2cb74b44b 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBQueryDemoIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBQueryDemoIT.java @@ -18,13 +18,11 @@ */ package org.apache.iotdb.db.it.query; -import org.apache.iotdb.isession.ISession; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.ClusterIT; import org.apache.iotdb.itbase.category.LocalStandaloneIT; -import com.google.common.collect.ImmutableList; import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; @@ -43,7 +41,6 @@ import java.util.List; import java.util.Map; -import static org.apache.iotdb.db.it.utils.TestUtils.assertTestFail; import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) @@ -671,26 +668,4 @@ public void RegexpNonExistTest() { fail(e.getMessage()); } } - - @Test - public void selectWithTimeTest() { - try (ISession session = EnvFactory.getEnv().getSessionConnection()) { - session.executeRawDataQuery( - ImmutableList.of("root.ln.wf01.wt01.time", "root.ln.wf01.wt01.temperature"), 0, 100); - - fail(); - } catch (Exception e) { - e.getMessage().contains("509: root.ln.wf01.wt01.time is not a legal path"); - } - - String expectedErrMsg = - "701: Time column is no need to appear in SELECT Clause explicitly, it will always be returned if possible"; - assertTestFail("select time from root.ln.wf01.wt01", expectedErrMsg); - assertTestFail("select time, temperature from root.ln.wf01.wt01", expectedErrMsg); - assertTestFail("select time from root.ln.wf01.wt01 where temperature > 1", expectedErrMsg); - // parse error when process 'wt01.time' - assertTestFail( - "select wt01.time, wt01.temperature from root.ln.wf01", - "700: Error occurred while parsing SQL to physical plan"); - } } diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/utils/TestUtils.java b/integration-test/src/test/java/org/apache/iotdb/db/it/utils/TestUtils.java index 9e407bba7d54..5cd9c63d5272 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/utils/TestUtils.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/utils/TestUtils.java @@ -268,75 +268,6 @@ public static void tableResultSetEqualTest( } } - public static void tableExecuteTest(String sql, String userName, String password) { - try (Connection connection = - EnvFactory.getEnv().getConnection(userName, password, BaseEnv.TABLE_SQL_DIALECT)) { - connection.setClientInfo("time_zone", "+00:00"); - try (Statement statement = connection.createStatement()) { - statement.execute(sql); - } - } catch (SQLException e) { - e.printStackTrace(); - fail(e.getMessage()); - } - } - - public static void tableQueryNoVerifyResultTest( - String sql, String[] expectedHeader, String userName, String password) { - try (Connection connection = - EnvFactory.getEnv().getConnection(userName, password, BaseEnv.TABLE_SQL_DIALECT)) { - connection.setClientInfo("time_zone", "+00:00"); - try (Statement statement = connection.createStatement()) { - try (ResultSet resultSet = statement.executeQuery(sql)) { - ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); - for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { - assertEquals(expectedHeader[i - 1], resultSetMetaData.getColumnName(i)); - } - assertEquals(expectedHeader.length, resultSetMetaData.getColumnCount()); - } - } - } catch (SQLException e) { - e.printStackTrace(); - fail(e.getMessage()); - } - } - - public static void tableResultSetEqualTest( - String sql, - String[] expectedHeader, - String[] expectedRetArray, - String userName, - String password) { - try (Connection connection = - EnvFactory.getEnv().getConnection(userName, password, BaseEnv.TABLE_SQL_DIALECT)) { - connection.setClientInfo("time_zone", "+00:00"); - try (Statement statement = connection.createStatement()) { - try (ResultSet resultSet = statement.executeQuery(sql)) { - ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); - for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { - assertEquals(expectedHeader[i - 1], resultSetMetaData.getColumnName(i)); - } - assertEquals(expectedHeader.length, resultSetMetaData.getColumnCount()); - - int cnt = 0; - while (resultSet.next()) { - StringBuilder builder = new StringBuilder(); - for (int i = 1; i <= expectedHeader.length; i++) { - builder.append(resultSet.getString(i)).append(","); - } - assertEquals(expectedRetArray[cnt], builder.toString()); - // System.out.println(String.format("\"%s\",", builder.toString())); - cnt++; - } - assertEquals(expectedRetArray.length, cnt); - } - } - } catch (SQLException e) { - e.printStackTrace(); - fail(e.getMessage()); - } - } - public static void tableResultSetFuzzyTest( String sql, String[] expectedHeader, int expectedCount, String database) { tableResultSetFuzzyTest( @@ -398,18 +329,6 @@ public static void tableAssertTestFail( } } - public static void tableAssertTestFail( - String sql, String errMsg, String userName, String password) { - try (Connection connection = - EnvFactory.getEnv().getConnection(userName, password, BaseEnv.TABLE_SQL_DIALECT); - Statement statement = connection.createStatement()) { - statement.executeQuery(sql); - fail("No exception!"); - } catch (SQLException e) { - Assert.assertTrue(e.getMessage(), e.getMessage().contains(errMsg)); - } - } - public static void resultSetEqualTest( String sql, String[] expectedHeader, @@ -536,26 +455,6 @@ public static void assertTestFail( } } - public static void assertTableTestFail( - final BaseEnv env, - final String sql, - final String errMsg, - final String userName, - final String password, - final String db) { - try (final Connection connection = - env.getConnection(userName, password, BaseEnv.TABLE_SQL_DIALECT); - final Statement statement = connection.createStatement()) { - if (Objects.nonNull(db)) { - statement.execute("use " + "\"" + db + "\""); - } - statement.executeQuery(sql); - fail("No exception!"); - } catch (final SQLException e) { - Assert.assertTrue(e.getMessage(), e.getMessage().contains(errMsg)); - } - } - public static void assertNonQueryTestFail(String sql, String errMsg) { assertNonQueryTestFail(sql, errMsg, SessionConfig.DEFAULT_USER, SessionConfig.DEFAULT_PASSWORD); } @@ -590,9 +489,7 @@ public static void assertTableNonQueryTestFail( BaseEnv env, String sql, String errMsg, String userName, String password, String db) { try (Connection connection = env.getConnection(userName, password, BaseEnv.TABLE_SQL_DIALECT); Statement statement = connection.createStatement()) { - if (Objects.nonNull(db)) { - statement.execute("use " + "\"" + db + "\""); - } + statement.execute("use " + "\"" + db + "\""); statement.execute(sql); fail("No exception!"); } catch (SQLException e) { @@ -749,57 +646,13 @@ public static void executeNonQuery(String sql, String userName, String password) } } - public static void executeNonQueryWithRetry(final BaseEnv env, final String sql) { - executeNonQueryWithRetry(env, sql, SessionConfig.DEFAULT_USER, SessionConfig.DEFAULT_PASSWORD); - } - - public static void executeNonQueryWithRetry( - final BaseEnv env, final String sql, final String userName, final String password) { - executeNonQueryWithRetry(env, sql, userName, password, null, TREE_SQL_DIALECT); - } - - public static void executeNonQueryWithRetry( - final BaseEnv env, - final String sql, - final String userName, - final String password, - final String database, - final String sqlDialect) { - executeNonQueriesWithRetry( - env, Collections.singletonList(sql), userName, password, database, sqlDialect); - } - - public static void executeNonQueriesWithRetry( - final BaseEnv env, final List sqlList, final String userName, final String password) { - executeNonQueriesWithRetry(env, sqlList, userName, password, "", TREE_SQL_DIALECT); - } - - public static void executeNonQueriesWithRetry( - final BaseEnv env, - final List sqlList, - final String userName, - final String password, - final String database, - final String sqlDialect) { - int lastIndex = 0; + public static void executeNonQueryWithRetry(BaseEnv env, String sql) { for (int retryCountLeft = 10; retryCountLeft >= 0; retryCountLeft--) { - try (final Connection connection = - env.getConnection( - userName, - password, - BaseEnv.TABLE_SQL_DIALECT.equals(sqlDialect) - ? BaseEnv.TABLE_SQL_DIALECT - : TREE_SQL_DIALECT); - final Statement statement = connection.createStatement()) { - if (BaseEnv.TABLE_SQL_DIALECT.equals(sqlDialect) && database != null) { - statement.execute("use " + database); - } - for (int i = lastIndex; i < sqlList.size(); ++i) { - lastIndex = i; - statement.execute(sqlList.get(i)); - } - return; - } catch (final SQLException e) { + try (Connection connection = env.getConnection(); + Statement statement = connection.createStatement()) { + statement.execute(sql); + break; + } catch (SQLException e) { if (retryCountLeft > 0) { try { Thread.sleep(10000); @@ -1019,57 +872,27 @@ public static void executeQuery(String sql, String userName, String password) { } public static void executeQueryWithRetry( - final BaseEnv env, final String sql, final String userName, final String password) { - executeQueryWithRetry(env, sql, userName, password, null, TREE_SQL_DIALECT); - } - - public static void executeQueryWithRetry( - final BaseEnv env, - final String sql, - final String userName, - final String password, - final String database, - final String sqlDialect) { - executeQueriesWithRetry( - env, Collections.singletonList(sql), userName, password, database, sqlDialect); - } - - public static void executeQueriesWithRetry( - final BaseEnv env, - final List sqlList, - final String userName, - final String password, - final String database, - final String sqlDialect) { - int lastIndex = 0; - for (int retryCountLeft = 10; retryCountLeft >= 0; retryCountLeft--) { - try (final Connection connection = - env.getConnection( - userName, - password, - BaseEnv.TABLE_SQL_DIALECT.equals(sqlDialect) - ? BaseEnv.TABLE_SQL_DIALECT - : TREE_SQL_DIALECT); - final Statement statement = connection.createStatement()) { - if (BaseEnv.TABLE_SQL_DIALECT.equals(sqlDialect) && database != null) { - statement.execute("use " + database); - } - for (int i = lastIndex; i < sqlList.size(); ++i) { - lastIndex = i; - statement.executeQuery(sqlList.get(i)); - } - return; - } catch (final SQLException e) { - if (retryCountLeft > 0) { - try { - Thread.sleep(10000); - } catch (InterruptedException ignored) { + BaseEnv env, String sql, String userName, String password) { + try (Connection connection = env.getConnection(userName, password); + Statement statement = connection.createStatement()) { + for (int retryCountLeft = 10; retryCountLeft >= 0; retryCountLeft--) { + try { + statement.executeQuery(sql); + } catch (SQLException e) { + if (retryCountLeft > 0) { + try { + Thread.sleep(10000); + } catch (InterruptedException ignored) { + } + } else { + e.printStackTrace(); + fail(e.getMessage()); } - } else { - e.printStackTrace(); - fail(e.getMessage()); } } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); } } diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/AbstractPipeDualTreeModelAutoIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/AbstractPipeDualAutoIT.java similarity index 97% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/AbstractPipeDualTreeModelAutoIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/AbstractPipeDualAutoIT.java index 4aa06e7acf31..3ad55920d294 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/AbstractPipeDualTreeModelAutoIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/AbstractPipeDualAutoIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.dual.treemodel.auto; +package org.apache.iotdb.pipe.it.autocreate; import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.consensus.ConsensusFactory; @@ -34,7 +34,7 @@ import java.util.Objects; import java.util.concurrent.TimeUnit; -public abstract class AbstractPipeDualTreeModelAutoIT { +abstract class AbstractPipeDualAutoIT { protected BaseEnv senderEnv; protected BaseEnv receiverEnv; diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeAlterIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAlterIT.java similarity index 98% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeAlterIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAlterIT.java index b1c8c4cf995b..a2f5c65cb73e 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeAlterIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAlterIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.dual.treemodel.auto.basic; +package org.apache.iotdb.pipe.it.autocreate; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; import org.apache.iotdb.confignode.rpc.thrift.TShowPipeInfo; @@ -25,8 +25,7 @@ import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoBasic; -import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeDualTreeModelAutoIT; +import org.apache.iotdb.itbase.category.MultiClusterIT2AutoCreateSchema; import org.junit.Assert; import org.junit.Test; @@ -45,8 +44,8 @@ import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2DualTreeAutoBasic.class}) -public class IoTDBPipeAlterIT extends AbstractPipeDualTreeModelAutoIT { +@Category({MultiClusterIT2AutoCreateSchema.class}) +public class IoTDBPipeAlterIT extends AbstractPipeDualAutoIT { @Test public void testBasicAlterPipe() throws Exception { diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeAutoConflictIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAutoConflictIT.java similarity index 98% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeAutoConflictIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAutoConflictIT.java index 65eb0d1ea140..65eb2458df77 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeAutoConflictIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAutoConflictIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.dual.treemodel.auto.enhanced; +package org.apache.iotdb.pipe.it.autocreate; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; @@ -27,8 +27,7 @@ import org.apache.iotdb.it.env.MultiEnvFactory; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoEnhanced; -import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeDualTreeModelAutoIT; +import org.apache.iotdb.itbase.category.MultiClusterIT2AutoCreateSchema; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -45,8 +44,8 @@ import java.util.Set; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2DualTreeAutoEnhanced.class}) -public class IoTDBPipeAutoConflictIT extends AbstractPipeDualTreeModelAutoIT { +@Category({MultiClusterIT2AutoCreateSchema.class}) +public class IoTDBPipeAutoConflictIT extends AbstractPipeDualAutoIT { @Before public void setUp() { MultiEnvFactory.createEnv(2); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeAutoDropIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAutoDropIT.java similarity index 95% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeAutoDropIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAutoDropIT.java index f1c3672e14a2..adff5e5f53da 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeAutoDropIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAutoDropIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.dual.treemodel.auto.enhanced; +package org.apache.iotdb.pipe.it.autocreate; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; @@ -27,8 +27,7 @@ import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoEnhanced; -import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeDualTreeModelAutoIT; +import org.apache.iotdb.itbase.category.MultiClusterIT2AutoCreateSchema; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -48,8 +47,8 @@ import static org.awaitility.Awaitility.await; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2DualTreeAutoEnhanced.class}) -public class IoTDBPipeAutoDropIT extends AbstractPipeDualTreeModelAutoIT { +@Category({MultiClusterIT2AutoCreateSchema.class}) +public class IoTDBPipeAutoDropIT extends AbstractPipeDualAutoIT { @Test public void testAutoDropInHistoricalTransfer() throws Exception { diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeClusterIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeClusterIT.java similarity index 98% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeClusterIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeClusterIT.java index ff827d6332b6..ad8c86ea054c 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeClusterIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeClusterIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.dual.treemodel.auto.enhanced; +package org.apache.iotdb.pipe.it.autocreate; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.exception.ClientManagerException; @@ -34,9 +34,8 @@ import org.apache.iotdb.it.env.cluster.env.AbstractEnv; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoEnhanced; -import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; -import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeDualTreeModelAutoIT; +import org.apache.iotdb.itbase.category.MultiClusterIT2AutoCreateSchema; +import org.apache.iotdb.pipe.it.tablemodel.TableModelUtils; import org.apache.iotdb.rpc.TSStatusCode; import org.apache.thrift.TException; @@ -60,8 +59,8 @@ import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2DualTreeAutoEnhanced.class}) -public class IoTDBPipeClusterIT extends AbstractPipeDualTreeModelAutoIT { +@Category({MultiClusterIT2AutoCreateSchema.class}) +public class IoTDBPipeClusterIT extends AbstractPipeDualAutoIT { @Override @Before diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeConditionalOperationsIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeConditionalOperationsIT.java similarity index 96% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeConditionalOperationsIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeConditionalOperationsIT.java index 36a12d5bcd6e..e6dfd37a307d 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeConditionalOperationsIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeConditionalOperationsIT.java @@ -17,15 +17,14 @@ * under the License. */ -package org.apache.iotdb.pipe.it.dual.treemodel.auto.enhanced; +package org.apache.iotdb.pipe.it.autocreate; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; import org.apache.iotdb.confignode.rpc.thrift.TShowPipeInfo; import org.apache.iotdb.confignode.rpc.thrift.TShowPipeReq; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoEnhanced; -import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeDualTreeModelAutoIT; +import org.apache.iotdb.itbase.category.MultiClusterIT2AutoCreateSchema; import org.junit.Assert; import org.junit.Test; @@ -40,8 +39,8 @@ import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2DualTreeAutoEnhanced.class}) -public class IoTDBPipeConditionalOperationsIT extends AbstractPipeDualTreeModelAutoIT { +@Category({MultiClusterIT2AutoCreateSchema.class}) +public class IoTDBPipeConditionalOperationsIT extends AbstractPipeDualAutoIT { @Test public void testBasicCreatePipeIfNotExists() throws Exception { diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeConnectorCompressionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeConnectorCompressionIT.java similarity index 97% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeConnectorCompressionIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeConnectorCompressionIT.java index 37a542e6aa9f..2a2c83522715 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeConnectorCompressionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeConnectorCompressionIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.dual.treemodel.auto.enhanced; +package org.apache.iotdb.pipe.it.autocreate; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; @@ -30,8 +30,7 @@ import org.apache.iotdb.it.env.MultiEnvFactory; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoEnhanced; -import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeDualTreeModelAutoIT; +import org.apache.iotdb.itbase.category.MultiClusterIT2AutoCreateSchema; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -52,8 +51,8 @@ import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2DualTreeAutoEnhanced.class}) -public class IoTDBPipeConnectorCompressionIT extends AbstractPipeDualTreeModelAutoIT { +@Category({MultiClusterIT2AutoCreateSchema.class}) +public class IoTDBPipeConnectorCompressionIT extends AbstractPipeDualAutoIT { @Override @Before diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeConnectorParallelIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeConnectorParallelIT.java similarity index 92% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeConnectorParallelIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeConnectorParallelIT.java index 8857f1942179..87bb4b465e56 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeConnectorParallelIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeConnectorParallelIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.dual.treemodel.auto.basic; +package org.apache.iotdb.pipe.it.autocreate; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; @@ -25,8 +25,7 @@ import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoBasic; -import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeDualTreeModelAutoIT; +import org.apache.iotdb.itbase.category.MultiClusterIT2AutoCreateSchema; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -41,8 +40,8 @@ import java.util.Set; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2DualTreeAutoBasic.class}) -public class IoTDBPipeConnectorParallelIT extends AbstractPipeDualTreeModelAutoIT { +@Category({MultiClusterIT2AutoCreateSchema.class}) +public class IoTDBPipeConnectorParallelIT extends AbstractPipeDualAutoIT { @Test public void testIoTConnectorParallel() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeDataSinkIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeDataSinkIT.java similarity index 98% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeDataSinkIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeDataSinkIT.java index 8855bf4e25f4..7facee0bbf06 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeDataSinkIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeDataSinkIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.dual.treemodel.auto.basic; +package org.apache.iotdb.pipe.it.autocreate; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; @@ -25,8 +25,7 @@ import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoBasic; -import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeDualTreeModelAutoIT; +import org.apache.iotdb.itbase.category.MultiClusterIT2AutoCreateSchema; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -41,8 +40,8 @@ import java.util.Map; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2DualTreeAutoBasic.class}) -public class IoTDBPipeDataSinkIT extends AbstractPipeDualTreeModelAutoIT { +@Category({MultiClusterIT2AutoCreateSchema.class}) +public class IoTDBPipeDataSinkIT extends AbstractPipeDualAutoIT { @Test public void testThriftConnectorWithRealtimeFirstDisabled() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeExtractorIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeExtractorIT.java similarity index 99% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeExtractorIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeExtractorIT.java index 32075c4040cc..0b4636c0d137 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeExtractorIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeExtractorIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.dual.treemodel.auto.basic; +package org.apache.iotdb.pipe.it.autocreate; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; @@ -29,9 +29,8 @@ import org.apache.iotdb.it.env.MultiEnvFactory; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoBasic; +import org.apache.iotdb.itbase.category.MultiClusterIT2AutoCreateSchema; import org.apache.iotdb.itbase.env.BaseEnv; -import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeDualTreeModelAutoIT; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -53,8 +52,8 @@ import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2DualTreeAutoBasic.class}) -public class IoTDBPipeExtractorIT extends AbstractPipeDualTreeModelAutoIT { +@Category({MultiClusterIT2AutoCreateSchema.class}) +public class IoTDBPipeExtractorIT extends AbstractPipeDualAutoIT { @Before public void setUp() { diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeIdempotentIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeIdempotentIT.java similarity index 80% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeIdempotentIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeIdempotentIT.java index e741ab82f6d9..cbe1ad065578 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeIdempotentIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeIdempotentIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.dual.treemodel.auto.enhanced; +package org.apache.iotdb.pipe.it.autocreate; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; @@ -27,8 +27,9 @@ import org.apache.iotdb.it.env.MultiEnvFactory; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoEnhanced; -import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeDualTreeModelAutoIT; +import org.apache.iotdb.itbase.category.MultiClusterIT2AutoCreateSchema; +import org.apache.iotdb.itbase.env.BaseEnv; +import org.apache.iotdb.pipe.it.tablemodel.TableModelUtils; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -45,8 +46,8 @@ import java.util.Set; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2DualTreeAutoEnhanced.class}) -public class IoTDBPipeIdempotentIT extends AbstractPipeDualTreeModelAutoIT { +@Category({MultiClusterIT2AutoCreateSchema.class}) +public class IoTDBPipeIdempotentIT extends AbstractPipeDualAutoIT { @Override @Before public void setUp() { @@ -395,6 +396,36 @@ public void testDropRoleIdempotent() throws Exception { // Table model + @Test + public void testCreateTableIdempotent() throws Exception { + testTableConfigIdempotent(Collections.emptyList(), "create table test()"); + } + + @Test + public void testAlterTableAddColumnIdempotent() throws Exception { + testTableConfigIdempotent( + Collections.singletonList("create table test()"), "alter table test add column a id"); + } + + @Test + public void testAlterTableSetPropertiesIdempotent() throws Exception { + testTableConfigIdempotent( + Collections.singletonList("create table test()"), + "alter table test set properties ttl=100"); + } + + @Test + public void testAlterTableDropColumnIdempotent() throws Exception { + testTableConfigIdempotent( + Collections.singletonList("create table test(a id, b attribute, c int32)"), + "alter table test drop column b"); + } + + @Test + public void testDropTableIdempotent() throws Exception { + testTableConfigIdempotent(Collections.singletonList("create table test()"), "drop table test"); + } + private void testIdempotent( final List beforeSqlList, final String testSql, @@ -452,4 +483,63 @@ private void testIdempotent( // Assume that the afterSql is executed on receiverEnv TestUtils.assertDataEventuallyOnEnv(receiverEnv, afterSqlQuery, expectedHeader, expectedResSet); } + + private void testTableConfigIdempotent(final List beforeSqlList, final String testSql) + throws Exception { + final String database = "test"; + TableModelUtils.createDatabase(senderEnv, database); + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + final String receiverIp = receiverDataNode.getIp(); + final int receiverPort = receiverDataNode.getPort(); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("extractor.inclusion", "all"); + extractorAttributes.put("extractor.inclusion.exclusion", ""); + extractorAttributes.put("extractor.forwarding-pipe-requests", "false"); + extractorAttributes.put("extractor.capture.table", "true"); + extractorAttributes.put("extractor.capture.tree", "false"); + + connectorAttributes.put("connector", "iotdb-thrift-connector"); + connectorAttributes.put("connector.ip", receiverIp); + connectorAttributes.put("connector.port", Integer.toString(receiverPort)); + connectorAttributes.put("connector.batch.enable", "false"); + connectorAttributes.put("connector.exception.conflict.resolve-strategy", "retry"); + connectorAttributes.put("connector.exception.conflict.retry-max-time-seconds", "-1"); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq("testPipe", connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + } + + if (!TestUtils.tryExecuteNonQueriesWithRetry( + database, BaseEnv.TABLE_SQL_DIALECT, senderEnv, beforeSqlList)) { + return; + } + + if (!TestUtils.tryExecuteNonQueryWithRetry( + database, BaseEnv.TABLE_SQL_DIALECT, receiverEnv, testSql)) { + return; + } + + // Create an idempotent conflict + if (!TestUtils.tryExecuteNonQueryWithRetry( + database, BaseEnv.TABLE_SQL_DIALECT, senderEnv, testSql)) { + return; + } + + TableModelUtils.createDatabase(senderEnv, "test2"); + + // Assume that the "database" is executed on receiverEnv + TestUtils.assertDataSizeEventuallyOnEnv(receiverEnv, "show databases", 3, null); + } } diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeLifeCycleIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeLifeCycleIT.java similarity index 97% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeLifeCycleIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeLifeCycleIT.java index 9e03b3d80320..f151463b396c 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeLifeCycleIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeLifeCycleIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.dual.treemodel.auto.basic; +package org.apache.iotdb.pipe.it.autocreate; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.auth.entity.PrivilegeType; @@ -26,8 +26,7 @@ import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoBasic; -import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeDualTreeModelAutoIT; +import org.apache.iotdb.itbase.category.MultiClusterIT2AutoCreateSchema; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -46,14 +45,14 @@ import static org.apache.iotdb.db.it.utils.TestUtils.assertNonQueryTestFail; import static org.apache.iotdb.db.it.utils.TestUtils.assertTestFail; import static org.apache.iotdb.db.it.utils.TestUtils.createUser; -import static org.apache.iotdb.db.it.utils.TestUtils.executeNonQueriesWithRetry; -import static org.apache.iotdb.db.it.utils.TestUtils.executeNonQueryWithRetry; import static org.apache.iotdb.db.it.utils.TestUtils.executeQueryWithRetry; import static org.apache.iotdb.db.it.utils.TestUtils.grantUserSystemPrivileges; +import static org.apache.iotdb.db.it.utils.TestUtils.tryExecuteNonQueriesWithRetry; +import static org.apache.iotdb.db.it.utils.TestUtils.tryExecuteNonQueryWithRetry; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2DualTreeAutoBasic.class}) -public class IoTDBPipeLifeCycleIT extends AbstractPipeDualTreeModelAutoIT { +@Category({MultiClusterIT2AutoCreateSchema.class}) +public class IoTDBPipeLifeCycleIT extends AbstractPipeDualAutoIT { @Test public void testLifeCycleWithHistoryEnabled() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); @@ -827,16 +826,18 @@ public void testPermission() { grantUserSystemPrivileges(senderEnv, "test", PrivilegeType.USE_PIPE); - executeNonQueryWithRetry( + tryExecuteNonQueryWithRetry( senderEnv, "create pipe testPipe\n" + "with connector (\n" - + " 'connector'='write-back-connector'\n" + + " 'connector'='iotdb-thrift-connector',\n" + + " 'connector.ip'='127.0.0.1',\n" + + " 'connector.port'='6668'\n" + ")", "test", "test123"); executeQueryWithRetry(senderEnv, "show pipes", "test", "test123"); - executeNonQueriesWithRetry( + tryExecuteNonQueriesWithRetry( senderEnv, Arrays.asList("start pipe testPipe", "stop pipe testPipe", "drop pipe testPipe"), "test", @@ -848,6 +849,7 @@ public void testPermission() { "701: Untrusted uri xxx", "test", "test123"); + tryExecuteNonQueryWithRetry(senderEnv, "drop pipePlugin TestProcessor", "test", "test123"); executeQueryWithRetry(senderEnv, "show pipe plugins", "test", "test123"); } } diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeNullValueIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeNullValueIT.java similarity index 97% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeNullValueIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeNullValueIT.java index 765104e1f523..3231fc523aab 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeNullValueIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeNullValueIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.dual.treemodel.auto.enhanced; +package org.apache.iotdb.pipe.it.autocreate; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; @@ -26,8 +26,7 @@ import org.apache.iotdb.isession.ISession; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoEnhanced; -import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeDualTreeModelAutoIT; +import org.apache.iotdb.itbase.category.MultiClusterIT2AutoCreateSchema; import org.apache.iotdb.rpc.IoTDBConnectionException; import org.apache.iotdb.rpc.StatementExecutionException; import org.apache.iotdb.rpc.TSStatusCode; @@ -52,8 +51,8 @@ import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2DualTreeAutoEnhanced.class}) -public class IoTDBPipeNullValueIT extends AbstractPipeDualTreeModelAutoIT { +@Category({MultiClusterIT2AutoCreateSchema.class}) +public class IoTDBPipeNullValueIT extends AbstractPipeDualAutoIT { // Test dimensions: // 1. is or not aligned diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeProcessorIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeProcessorIT.java similarity index 94% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeProcessorIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeProcessorIT.java index 4a17939065f4..27bc5be96599 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeProcessorIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeProcessorIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.dual.treemodel.auto.basic; +package org.apache.iotdb.pipe.it.autocreate; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; @@ -27,8 +27,7 @@ import org.apache.iotdb.it.env.MultiEnvFactory; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoBasic; -import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeDualTreeModelAutoIT; +import org.apache.iotdb.itbase.category.MultiClusterIT2AutoCreateSchema; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -44,8 +43,8 @@ import java.util.Set; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2DualTreeAutoBasic.class}) -public class IoTDBPipeProcessorIT extends AbstractPipeDualTreeModelAutoIT { +@Category({MultiClusterIT2AutoCreateSchema.class}) +public class IoTDBPipeProcessorIT extends AbstractPipeDualAutoIT { @Before public void setUp() { MultiEnvFactory.createEnv(2); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeProtocolIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeProtocolIT.java similarity index 98% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeProtocolIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeProtocolIT.java index 6035ed9c16a0..c5d41f001cb8 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeProtocolIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeProtocolIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.dual.treemodel.auto.basic; +package org.apache.iotdb.pipe.it.autocreate; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; @@ -28,8 +28,7 @@ import org.apache.iotdb.it.env.MultiEnvFactory; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoBasic; -import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeDualTreeModelAutoIT; +import org.apache.iotdb.itbase.category.MultiClusterIT2AutoCreateSchema; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -45,8 +44,8 @@ /** Test pipe's basic functionalities under multiple cluster and consensus protocol settings. */ @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2DualTreeAutoBasic.class}) -public class IoTDBPipeProtocolIT extends AbstractPipeDualTreeModelAutoIT { +@Category({MultiClusterIT2AutoCreateSchema.class}) +public class IoTDBPipeProtocolIT extends AbstractPipeDualAutoIT { @Override @Before public void setUp() { diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeSwitchStatusIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeSwitchStatusIT.java similarity index 97% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeSwitchStatusIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeSwitchStatusIT.java index ee70f1a3a3ba..03e52a982e61 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeSwitchStatusIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeSwitchStatusIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.dual.treemodel.auto.basic; +package org.apache.iotdb.pipe.it.autocreate; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; @@ -28,8 +28,7 @@ import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoBasic; -import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeDualTreeModelAutoIT; +import org.apache.iotdb.itbase.category.MultiClusterIT2AutoCreateSchema; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -42,8 +41,8 @@ import java.util.Map; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2DualTreeAutoBasic.class}) -public class IoTDBPipeSwitchStatusIT extends AbstractPipeDualTreeModelAutoIT { +@Category({MultiClusterIT2AutoCreateSchema.class}) +public class IoTDBPipeSwitchStatusIT extends AbstractPipeDualAutoIT { @Test public void testPipeSwitchStatus() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeSyntaxIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeSyntaxIT.java similarity index 98% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeSyntaxIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeSyntaxIT.java index a0450942d473..b9de28b71586 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeSyntaxIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeSyntaxIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.dual.treemodel.auto.basic; +package org.apache.iotdb.pipe.it.autocreate; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; @@ -26,8 +26,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TShowPipeReq; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoBasic; -import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeDualTreeModelAutoIT; +import org.apache.iotdb.itbase.category.MultiClusterIT2AutoCreateSchema; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -46,8 +45,8 @@ import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2DualTreeAutoBasic.class}) -public class IoTDBPipeSyntaxIT extends AbstractPipeDualTreeModelAutoIT { +@Category({MultiClusterIT2AutoCreateSchema.class}) +public class IoTDBPipeSyntaxIT extends AbstractPipeDualAutoIT { @Test public void testValidPipeName() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeWithLoadIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeWithLoadIT.java similarity index 93% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeWithLoadIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeWithLoadIT.java index 0b929e4daf88..563377c927ea 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/IoTDBPipeWithLoadIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeWithLoadIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.dual.treemodel.auto.enhanced; +package org.apache.iotdb.pipe.it.autocreate; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; @@ -27,8 +27,7 @@ import org.apache.iotdb.it.env.MultiEnvFactory; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoEnhanced; -import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeDualTreeModelAutoIT; +import org.apache.iotdb.itbase.category.MultiClusterIT2AutoCreateSchema; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -43,8 +42,8 @@ import java.util.Map; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2DualTreeAutoEnhanced.class}) -public class IoTDBPipeWithLoadIT extends AbstractPipeDualTreeModelAutoIT { +@Category({MultiClusterIT2AutoCreateSchema.class}) +public class IoTDBPipeWithLoadIT extends AbstractPipeDualAutoIT { @Before public void setUp() { diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBTreePatternFormatIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBTreePatternFormatIT.java similarity index 95% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBTreePatternFormatIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBTreePatternFormatIT.java index 28fa66213126..b62102f545df 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBTreePatternFormatIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBTreePatternFormatIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.dual.treemodel.auto.basic; +package org.apache.iotdb.pipe.it.autocreate; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; @@ -25,8 +25,7 @@ import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoBasic; -import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeDualTreeModelAutoIT; +import org.apache.iotdb.itbase.category.MultiClusterIT2AutoCreateSchema; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -41,8 +40,8 @@ import java.util.Set; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2DualTreeAutoBasic.class}) -public class IoTDBTreePatternFormatIT extends AbstractPipeDualTreeModelAutoIT { +@Category({MultiClusterIT2AutoCreateSchema.class}) +public class IoTDBTreePatternFormatIT extends AbstractPipeDualAutoIT { @Test public void testPrefixPattern() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/PipeNowFunctionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/PipeNowFunctionIT.java similarity index 97% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/PipeNowFunctionIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/PipeNowFunctionIT.java index 471a1e815948..6a2106b554d1 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/enhanced/PipeNowFunctionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/PipeNowFunctionIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.dual.treemodel.auto.enhanced; +package org.apache.iotdb.pipe.it.autocreate; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; @@ -27,9 +27,8 @@ import org.apache.iotdb.confignode.rpc.thrift.TShowPipeReq; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeAutoEnhanced; +import org.apache.iotdb.itbase.category.MultiClusterIT2AutoCreateSchema; import org.apache.iotdb.itbase.env.BaseEnv; -import org.apache.iotdb.pipe.it.dual.treemodel.auto.AbstractPipeDualTreeModelAutoIT; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -47,8 +46,8 @@ import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2DualTreeAutoEnhanced.class}) -public class PipeNowFunctionIT extends AbstractPipeDualTreeModelAutoIT { +@Category({MultiClusterIT2AutoCreateSchema.class}) +public class PipeNowFunctionIT extends AbstractPipeDualAutoIT { @Test public void testPipeNowFunction() throws Exception { diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeIdempotentIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeIdempotentIT.java deleted file mode 100644 index c80a67c4d4f8..000000000000 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeIdempotentIT.java +++ /dev/null @@ -1,197 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.pipe.it.dual.tablemodel.manual.enhanced; - -import org.apache.iotdb.common.rpc.thrift.TSStatus; -import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; -import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; -import org.apache.iotdb.db.it.utils.TestUtils; -import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; -import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualEnhanced; -import org.apache.iotdb.itbase.env.BaseEnv; -import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; -import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; -import org.apache.iotdb.rpc.TSStatusCode; - -import org.junit.Assert; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; - -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -@RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2DualTableManualEnhanced.class}) -public class IoTDBPipeIdempotentIT extends AbstractPipeTableModelDualManualIT { - @Test - public void testCreateTableIdempotent() throws Exception { - testTableConfigIdempotent(Collections.emptyList(), "create table test()"); - } - - @Test - public void testAlterTableAddColumnIdempotent() throws Exception { - testTableConfigIdempotent( - Collections.singletonList("create table test()"), "alter table test add column a id"); - } - - @Test - public void testAlterTableSetPropertiesIdempotent() throws Exception { - testTableConfigIdempotent( - Collections.singletonList("create table test()"), - "alter table test set properties ttl=100"); - } - - @Test - public void testAlterTableDropColumnIdempotent() throws Exception { - testTableConfigIdempotent( - Collections.singletonList("create table test(a id, b attribute, c int32)"), - "alter table test drop column b"); - } - - @Test - public void testDropTableIdempotent() throws Exception { - testTableConfigIdempotent(Collections.singletonList("create table test()"), "drop table test"); - } - - @Test - public void testTableCreateUserIdempotent() throws Exception { - testTableConfigIdempotent(Collections.emptyList(), "create user newUser 'password'"); - } - - @Test - public void testTableDropUserIdempotent() throws Exception { - testTableConfigIdempotent( - Collections.singletonList("create user newUser 'password'"), "drop user newUser"); - } - - @Test - public void testTableCreateRoleIdempotent() throws Exception { - testTableConfigIdempotent(Collections.emptyList(), "create role newRole"); - } - - @Test - public void testTableDropRoleIdempotent() throws Exception { - testTableConfigIdempotent( - Collections.singletonList("create role newRole"), "drop role newRole"); - } - - @Test - public void testTableAlterUserIdempotent3() throws Exception { - testTableConfigIdempotent( - Collections.singletonList("create user newUser 'password'"), - "alter user newUser set password 'passwd'"); - } - - @Test - public void testTableGrantRoleToUserIdempotent() throws Exception { - testTableConfigIdempotent( - Arrays.asList("create user newUser 'password'", "create role newRole"), - "grant role newRole to newUser"); - } - - @Test - public void testTableRevokeRoleFromUserIdempotent() throws Exception { - testTableConfigIdempotent( - Arrays.asList( - "create user newUser 'password'", - "create role newRole", - "grant role newRole to newUser"), - "revoke role newRole from newUser"); - } - - @Test - public void testTableGrantIdempotent() throws Exception { - testTableConfigIdempotent( - Collections.singletonList("create user newUser 'password'"), - "grant all to user newUser with grant option"); - } - - @Test - public void testTableRevokeIdempotent() throws Exception { - testTableConfigIdempotent( - Arrays.asList( - "create user newUser 'password'", "grant all to user newUser with grant option"), - "revoke all from user newUser"); - } - - private void testTableConfigIdempotent(final List beforeSqlList, final String testSql) - throws Exception { - final String database = "test"; - TableModelUtils.createDatabase(senderEnv, database); - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - extractorAttributes.put("extractor.inclusion", "all"); - extractorAttributes.put("extractor.inclusion.exclusion", ""); - extractorAttributes.put("extractor.forwarding-pipe-requests", "false"); - extractorAttributes.put("extractor.capture.table", "true"); - extractorAttributes.put("extractor.capture.tree", "false"); - - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - connectorAttributes.put("connector.batch.enable", "false"); - connectorAttributes.put("connector.exception.conflict.resolve-strategy", "retry"); - connectorAttributes.put("connector.exception.conflict.retry-max-time-seconds", "-1"); - - final TSStatus status = - client.createPipe( - new TCreatePipeReq("testPipe", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - } - - if (!TestUtils.tryExecuteNonQueriesWithRetry( - database, BaseEnv.TABLE_SQL_DIALECT, senderEnv, beforeSqlList)) { - return; - } - - if (!TestUtils.tryExecuteNonQueryWithRetry( - database, BaseEnv.TABLE_SQL_DIALECT, receiverEnv, testSql)) { - return; - } - - // Create an idempotent conflict - if (!TestUtils.tryExecuteNonQueryWithRetry( - database, BaseEnv.TABLE_SQL_DIALECT, senderEnv, testSql)) { - return; - } - - TableModelUtils.createDatabase(senderEnv, "test2"); - - // Assume that the "database" is executed on receiverEnv - TestUtils.assertDataSizeEventuallyOnEnv(receiverEnv, "show databases", 3, null); - } -} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/AbstractPipeDualTreeModelManualIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/AbstractPipeDualManualIT.java similarity index 97% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/AbstractPipeDualTreeModelManualIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/AbstractPipeDualManualIT.java index 581d363cdb5c..c93f80a513a4 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/AbstractPipeDualTreeModelManualIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/AbstractPipeDualManualIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.dual.treemodel.manual; +package org.apache.iotdb.pipe.it.manual; import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.consensus.ConsensusFactory; @@ -34,7 +34,7 @@ import java.util.Objects; import java.util.concurrent.TimeUnit; -public abstract class AbstractPipeDualTreeModelManualIT { +abstract class AbstractPipeDualManualIT { protected BaseEnv senderEnv; protected BaseEnv receiverEnv; diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeInclusionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeInclusionIT.java similarity index 97% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeInclusionIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeInclusionIT.java index 4bc4505c1b6f..76affe08850f 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeInclusionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeInclusionIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.dual.treemodel.manual; +package org.apache.iotdb.pipe.it.manual; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; @@ -25,7 +25,7 @@ import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeManual; +import org.apache.iotdb.itbase.category.MultiClusterIT2ManualCreateSchema; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -40,8 +40,8 @@ import java.util.Map; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2DualTreeManual.class}) -public class IoTDBPipeInclusionIT extends AbstractPipeDualTreeModelManualIT { +@Category({MultiClusterIT2ManualCreateSchema.class}) +public class IoTDBPipeInclusionIT extends AbstractPipeDualManualIT { @Test public void testPureSchemaInclusion() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeManualConflictIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeManualConflictIT.java similarity index 97% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeManualConflictIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeManualConflictIT.java index 63bef4ff759c..dbb181092286 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeManualConflictIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeManualConflictIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.dual.treemodel.manual; +package org.apache.iotdb.pipe.it.manual; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; @@ -25,7 +25,7 @@ import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeManual; +import org.apache.iotdb.itbase.category.MultiClusterIT2ManualCreateSchema; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -40,8 +40,8 @@ import java.util.Map; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2DualTreeManual.class}) -public class IoTDBPipeManualConflictIT extends AbstractPipeDualTreeModelManualIT { +@Category({MultiClusterIT2ManualCreateSchema.class}) +public class IoTDBPipeManualConflictIT extends AbstractPipeDualManualIT { @Test public void testDoubleLivingTimeseries() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeMetaHistoricalIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeMetaHistoricalIT.java similarity index 98% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeMetaHistoricalIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeMetaHistoricalIT.java index c351aad274c4..6c36de7b6966 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeMetaHistoricalIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeMetaHistoricalIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.dual.treemodel.manual; +package org.apache.iotdb.pipe.it.manual; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; @@ -28,7 +28,7 @@ import org.apache.iotdb.it.env.MultiEnvFactory; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeManual; +import org.apache.iotdb.itbase.category.MultiClusterIT2ManualCreateSchema; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -44,8 +44,8 @@ import java.util.Map; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2DualTreeManual.class}) -public class IoTDBPipeMetaHistoricalIT extends AbstractPipeDualTreeModelManualIT { +@Category({MultiClusterIT2ManualCreateSchema.class}) +public class IoTDBPipeMetaHistoricalIT extends AbstractPipeDualManualIT { @Override @Before public void setUp() { diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeMetaLeaderChangeIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeMetaLeaderChangeIT.java similarity index 97% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeMetaLeaderChangeIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeMetaLeaderChangeIT.java index 00281253abb0..c03f56bad4ee 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeMetaLeaderChangeIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeMetaLeaderChangeIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.dual.treemodel.manual; +package org.apache.iotdb.pipe.it.manual; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; @@ -27,7 +27,7 @@ import org.apache.iotdb.it.env.MultiEnvFactory; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeManual; +import org.apache.iotdb.itbase.category.MultiClusterIT2ManualCreateSchema; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -41,8 +41,8 @@ import java.util.Map; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2DualTreeManual.class}) -public class IoTDBPipeMetaLeaderChangeIT extends AbstractPipeDualTreeModelManualIT { +@Category({MultiClusterIT2ManualCreateSchema.class}) +public class IoTDBPipeMetaLeaderChangeIT extends AbstractPipeDualManualIT { @Override @Before public void setUp() { diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeMetaRestartIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeMetaRestartIT.java similarity index 96% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeMetaRestartIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeMetaRestartIT.java index ec9930d2642b..76cd4a90e8b8 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeMetaRestartIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeMetaRestartIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.dual.treemodel.manual; +package org.apache.iotdb.pipe.it.manual; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; @@ -25,7 +25,7 @@ import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeManual; +import org.apache.iotdb.itbase.category.MultiClusterIT2ManualCreateSchema; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -38,8 +38,8 @@ import java.util.Map; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2DualTreeManual.class}) -public class IoTDBPipeMetaRestartIT extends AbstractPipeDualTreeModelManualIT { +@Category({MultiClusterIT2ManualCreateSchema.class}) +public class IoTDBPipeMetaRestartIT extends AbstractPipeDualManualIT { @Test public void testAutoRestartSchemaTask() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeMultiSchemaRegionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeMultiSchemaRegionIT.java similarity index 95% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeMultiSchemaRegionIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeMultiSchemaRegionIT.java index 8c210f64638d..769b940c78d2 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeMultiSchemaRegionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeMultiSchemaRegionIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.dual.treemodel.manual; +package org.apache.iotdb.pipe.it.manual; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; @@ -25,7 +25,7 @@ import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeManual; +import org.apache.iotdb.itbase.category.MultiClusterIT2ManualCreateSchema; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -39,8 +39,8 @@ import java.util.Map; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2DualTreeManual.class}) -public class IoTDBPipeMultiSchemaRegionIT extends AbstractPipeDualTreeModelManualIT { +@Category({MultiClusterIT2ManualCreateSchema.class}) +public class IoTDBPipeMultiSchemaRegionIT extends AbstractPipeDualManualIT { @Test public void testMultiSchemaRegion() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipePermissionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipePermissionIT.java similarity index 97% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipePermissionIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipePermissionIT.java index 52320d52bd95..648899b13ea5 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipePermissionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipePermissionIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.dual.treemodel.manual; +package org.apache.iotdb.pipe.it.manual; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; @@ -27,7 +27,7 @@ import org.apache.iotdb.it.env.MultiEnvFactory; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeManual; +import org.apache.iotdb.itbase.category.MultiClusterIT2ManualCreateSchema; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -46,8 +46,8 @@ import static org.junit.jupiter.api.Assertions.fail; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2DualTreeManual.class}) -public class IoTDBPipePermissionIT extends AbstractPipeDualTreeModelManualIT { +@Category({MultiClusterIT2ManualCreateSchema.class}) +public class IoTDBPipePermissionIT extends AbstractPipeDualManualIT { @Override @Before public void setUp() { diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeReqAutoSliceIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeReqAutoSliceIT.java similarity index 98% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeReqAutoSliceIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeReqAutoSliceIT.java index 55b5f72314da..953f1faec33c 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeReqAutoSliceIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeReqAutoSliceIT.java @@ -17,13 +17,13 @@ * under the License. */ -package org.apache.iotdb.pipe.it.dual.treemodel.manual; +package org.apache.iotdb.pipe.it.manual; import org.apache.iotdb.commons.utils.function.CheckedTriConsumer; import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.isession.ISession; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeManual; +import org.apache.iotdb.itbase.category.MultiClusterIT2ManualCreateSchema; import org.apache.iotdb.rpc.IoTDBConnectionException; import org.apache.iotdb.rpc.StatementExecutionException; @@ -46,8 +46,8 @@ import java.util.Random; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2DualTreeManual.class}) -public class IoTDBPipeReqAutoSliceIT extends AbstractPipeDualTreeModelManualIT { +@Category({MultiClusterIT2ManualCreateSchema.class}) +public class IoTDBPipeReqAutoSliceIT extends AbstractPipeDualManualIT { private static final int generateDataSize = 10; @Override diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeMetaIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeTableManualIT.java similarity index 77% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeMetaIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeTableManualIT.java index 4e33f8369d62..f375bd35e81b 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeMetaIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeTableManualIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.dual.tablemodel.manual.enhanced; +package org.apache.iotdb.pipe.it.manual; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; @@ -25,10 +25,9 @@ import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualEnhanced; +import org.apache.iotdb.itbase.category.MultiClusterIT2ManualCreateSchema; import org.apache.iotdb.itbase.env.BaseEnv; -import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; -import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; +import org.apache.iotdb.pipe.it.tablemodel.TableModelUtils; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -43,8 +42,8 @@ import java.util.Map; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2DualTableManualEnhanced.class}) -public class IoTDBPipeMetaIT extends AbstractPipeTableModelDualManualIT { +@Category({MultiClusterIT2ManualCreateSchema.class}) +public class IoTDBPipeTableManualIT extends AbstractPipeDualManualIT { @Test public void testTableSync() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); @@ -291,80 +290,4 @@ public void testNoTable() throws Exception { dbName); } } - - @Test - public void testAuth() throws Exception { - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - final String receiverIp = receiverDataNode.getIp(); - final int receiverPort = receiverDataNode.getPort(); - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - - if (!TestUtils.tryExecuteNonQueriesWithRetry( - senderEnv, - Arrays.asList( - "create user testUser 'password'", "grant all on root.** to user testUser"))) { - return; - } - - final String dbName = "test"; - if (!TestUtils.tryExecuteNonQueriesWithRetry( - dbName, - BaseEnv.TABLE_SQL_DIALECT, - senderEnv, - Arrays.asList( - "grant create on db.tb to user testUser", - "grant drop on database test to user testUser"))) { - return; - } - - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - extractorAttributes.put("extractor.inclusion", "all"); - extractorAttributes.put("extractor.capture.tree", "false"); - extractorAttributes.put("extractor.capture.table", "true"); - extractorAttributes.put("extractor.database-name", "test"); - extractorAttributes.put("extractor.table-name", "t.*[0-9]"); - - connectorAttributes.put("connector", "iotdb-thrift-connector"); - connectorAttributes.put("connector.ip", receiverIp); - connectorAttributes.put("connector.port", Integer.toString(receiverPort)); - - final TSStatus status = - client.createPipe( - new TCreatePipeReq("testPipe", connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("testPipe").getCode()); - - if (!TestUtils.tryExecuteNonQueryWithRetry( - dbName, - BaseEnv.TABLE_SQL_DIALECT, - senderEnv, - "grant alter on any to user testUser with grant option")) { - return; - } - - TestUtils.assertDataAlwaysOnEnv( - receiverEnv, - "list privileges of user testUser", - "Role,Scope,Privileges,GrantOption,", - new HashSet<>( - Arrays.asList( - ",,MANAGE_USER,false,", - ",,MANAGE_ROLE,false,", - ",,MAINTAIN,false,", - ",*.*,ALTER,true,", - ",test.*,DROP,false,")), - dbName); - } - } } diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTypeConversionISessionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeTypeConversionISessionIT.java similarity index 99% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTypeConversionISessionIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeTypeConversionISessionIT.java index 23168924aed1..b94aea2f2371 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTypeConversionISessionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeTypeConversionISessionIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.dual.treemodel.manual; +package org.apache.iotdb.pipe.it.manual; import org.apache.iotdb.commons.utils.function.CheckedTriConsumer; import org.apache.iotdb.db.it.utils.TestUtils; @@ -25,7 +25,7 @@ import org.apache.iotdb.isession.ISession; import org.apache.iotdb.isession.SessionDataSet; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeManual; +import org.apache.iotdb.itbase.category.MultiClusterIT2ManualCreateSchema; import org.apache.iotdb.itbase.env.BaseEnv; import org.apache.iotdb.rpc.IoTDBConnectionException; import org.apache.iotdb.rpc.RpcUtils; @@ -62,8 +62,8 @@ import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2DualTreeManual.class}) -public class IoTDBPipeTypeConversionISessionIT extends AbstractPipeDualTreeModelManualIT { +@Category({MultiClusterIT2ManualCreateSchema.class}) +public class IoTDBPipeTypeConversionISessionIT extends AbstractPipeDualManualIT { private static final int generateDataSize = 100; @Test diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTypeConversionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeTypeConversionIT.java similarity index 98% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTypeConversionIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeTypeConversionIT.java index e87bda0f361d..f7cb595acbf0 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeTypeConversionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeTypeConversionIT.java @@ -17,12 +17,12 @@ * under the License. */ -package org.apache.iotdb.pipe.it.dual.treemodel.manual; +package org.apache.iotdb.pipe.it.manual; import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.db.pipe.receiver.transform.converter.ValueConverter; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2DualTreeManual; +import org.apache.iotdb.itbase.category.MultiClusterIT2ManualCreateSchema; import org.apache.iotdb.itbase.env.BaseEnv; import org.apache.iotdb.rpc.RpcUtils; @@ -46,8 +46,8 @@ import java.util.Set; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2DualTreeManual.class}) -public class IoTDBPipeTypeConversionIT extends AbstractPipeDualTreeModelManualIT { +@Category({MultiClusterIT2ManualCreateSchema.class}) +public class IoTDBPipeTypeConversionIT extends AbstractPipeDualManualIT { private static final int generateDataSize = 100; diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBPipeOPCUAIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBPipeOPCUAIT.java index 591881369ed4..9b8388659771 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBPipeOPCUAIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBPipeOPCUAIT.java @@ -25,7 +25,7 @@ import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.MultiClusterIT1; import org.apache.iotdb.itbase.env.BaseEnv; -import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; +import org.apache.iotdb.pipe.it.tablemodel.TableModelUtils; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/AbstractPipeTableModelDualManualIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/AbstractPipeTableModelTestIT.java similarity index 95% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/AbstractPipeTableModelDualManualIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/AbstractPipeTableModelTestIT.java index fff0b8c6bf1a..06c7c3867f5b 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/AbstractPipeTableModelDualManualIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/AbstractPipeTableModelTestIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.dual.tablemodel.manual; +package org.apache.iotdb.pipe.it.tablemodel; import org.apache.iotdb.consensus.ConsensusFactory; import org.apache.iotdb.it.env.MultiEnvFactory; @@ -26,7 +26,7 @@ import org.junit.After; import org.junit.Before; -public abstract class AbstractPipeTableModelDualManualIT { +abstract class AbstractPipeTableModelTestIT { protected BaseEnv senderEnv; protected BaseEnv receiverEnv; diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeAlterIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeAlterIT.java similarity index 98% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeAlterIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeAlterIT.java index 93bb1424c2b2..1665f9d2ce2b 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeAlterIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeAlterIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.dual.tablemodel.manual.basic; +package org.apache.iotdb.pipe.it.tablemodel; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; import org.apache.iotdb.confignode.rpc.thrift.TShowPipeInfo; @@ -25,10 +25,8 @@ import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualBasic; +import org.apache.iotdb.itbase.category.MultiClusterIT2TableModel; import org.apache.iotdb.itbase.env.BaseEnv; -import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; -import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; import org.junit.Assert; import org.junit.Test; @@ -44,8 +42,8 @@ import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2DualTableManualBasic.class}) -public class IoTDBPipeAlterIT extends AbstractPipeTableModelDualManualIT { +@Category({MultiClusterIT2TableModel.class}) +public class IoTDBPipeAlterIT extends AbstractPipeTableModelTestIT { @Test public void testBasicAlterPipe() throws Exception { diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeAutoConflictIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeAutoConflictIT.java similarity index 96% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeAutoConflictIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeAutoConflictIT.java index fc29e3e12a74..525be694815a 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeAutoConflictIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeAutoConflictIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.dual.tablemodel.manual.enhanced; +package org.apache.iotdb.pipe.it.tablemodel; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; @@ -27,10 +27,8 @@ import org.apache.iotdb.it.env.MultiEnvFactory; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualEnhanced; +import org.apache.iotdb.itbase.category.MultiClusterIT2TableModel; import org.apache.iotdb.itbase.env.BaseEnv; -import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; -import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -46,8 +44,8 @@ import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2DualTableManualEnhanced.class}) -public class IoTDBPipeAutoConflictIT extends AbstractPipeTableModelDualManualIT { +@Category({MultiClusterIT2TableModel.class}) +public class IoTDBPipeAutoConflictIT extends AbstractPipeTableModelTestIT { @Before public void setUp() { MultiEnvFactory.createEnv(2); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeAutoDropIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeAutoDropIT.java similarity index 94% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeAutoDropIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeAutoDropIT.java index cbf2c3073978..8f19b7f32a7f 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeAutoDropIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeAutoDropIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.dual.tablemodel.manual.enhanced; +package org.apache.iotdb.pipe.it.tablemodel; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; @@ -27,9 +27,7 @@ import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualEnhanced; -import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; -import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; +import org.apache.iotdb.itbase.category.MultiClusterIT2TableModel; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -50,8 +48,8 @@ import static org.awaitility.Awaitility.await; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2DualTableManualEnhanced.class}) -public class IoTDBPipeAutoDropIT extends AbstractPipeTableModelDualManualIT { +@Category({MultiClusterIT2TableModel.class}) +public class IoTDBPipeAutoDropIT extends AbstractPipeTableModelTestIT { @Test public void testAutoDropInHistoricalTransfer() throws Exception { diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeClusterIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeClusterIT.java similarity index 98% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeClusterIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeClusterIT.java index 44fae78b362b..cab7f31a7568 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeClusterIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeClusterIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.dual.tablemodel.manual.enhanced; +package org.apache.iotdb.pipe.it.tablemodel; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.exception.ClientManagerException; @@ -34,9 +34,7 @@ import org.apache.iotdb.it.env.cluster.env.AbstractEnv; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualEnhanced; -import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; -import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; +import org.apache.iotdb.itbase.category.MultiClusterIT2TableModel; import org.apache.iotdb.rpc.TSStatusCode; import org.apache.thrift.TException; @@ -58,8 +56,8 @@ import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2DualTableManualEnhanced.class}) -public class IoTDBPipeClusterIT extends AbstractPipeTableModelDualManualIT { +@Category({MultiClusterIT2TableModel.class}) +public class IoTDBPipeClusterIT extends AbstractPipeTableModelTestIT { @Override @Before diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeConnectorCompressionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeConnectorCompressionIT.java similarity index 97% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeConnectorCompressionIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeConnectorCompressionIT.java index 7bdbc0c2fca7..3b462d0d6ae6 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeConnectorCompressionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeConnectorCompressionIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.dual.tablemodel.manual.enhanced; +package org.apache.iotdb.pipe.it.tablemodel; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; @@ -30,9 +30,7 @@ import org.apache.iotdb.it.env.MultiEnvFactory; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualEnhanced; -import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; -import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; +import org.apache.iotdb.itbase.category.MultiClusterIT2TableModel; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -54,8 +52,8 @@ import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2DualTableManualEnhanced.class}) -public class IoTDBPipeConnectorCompressionIT extends AbstractPipeTableModelDualManualIT { +@Category({MultiClusterIT2TableModel.class}) +public class IoTDBPipeConnectorCompressionIT extends AbstractPipeTableModelTestIT { @Override @Before diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeConnectorParallelIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeConnectorParallelIT.java similarity index 91% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeConnectorParallelIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeConnectorParallelIT.java index 7c2c56a0fd73..663d40c5b07c 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeConnectorParallelIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeConnectorParallelIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.dual.tablemodel.manual.enhanced; +package org.apache.iotdb.pipe.it.tablemodel; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; @@ -25,9 +25,7 @@ import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualEnhanced; -import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; -import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; +import org.apache.iotdb.itbase.category.MultiClusterIT2TableModel; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -43,8 +41,8 @@ import java.util.function.Consumer; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2DualTableManualEnhanced.class}) -public class IoTDBPipeConnectorParallelIT extends AbstractPipeTableModelDualManualIT { +@Category({MultiClusterIT2TableModel.class}) +public class IoTDBPipeConnectorParallelIT extends AbstractPipeTableModelTestIT { @Test public void testIoTConnectorParallel() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeDataSinkIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeDataSinkIT.java similarity index 98% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeDataSinkIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeDataSinkIT.java index 7c98f652f73f..2867a3402503 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeDataSinkIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeDataSinkIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.dual.tablemodel.manual.basic; +package org.apache.iotdb.pipe.it.tablemodel; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; @@ -25,9 +25,7 @@ import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualBasic; -import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; -import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; +import org.apache.iotdb.itbase.category.MultiClusterIT2TableModel; import org.apache.iotdb.rpc.TSStatusCode; import org.apache.tsfile.write.record.Tablet; @@ -49,8 +47,8 @@ import java.util.function.Consumer; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2DualTableManualBasic.class}) -public class IoTDBPipeDataSinkIT extends AbstractPipeTableModelDualManualIT { +@Category({MultiClusterIT2TableModel.class}) +public class IoTDBPipeDataSinkIT extends AbstractPipeTableModelTestIT { @Test public void testThriftConnectorWithRealtimeFirstDisabled() throws Exception { @@ -192,7 +190,7 @@ private void testSinkFormat(final String format) throws Exception { return; } - TableModelUtils.assertCountData("test", "test", 150, receiverEnv, handleFailure); + TableModelUtils.assertCountData("test", "test", 150, receiverEnv); TestUtils.assertDataEventuallyOnEnv( receiverEnv, @@ -237,7 +235,7 @@ private void testSinkFormat(final String format) throws Exception { new HashSet<>(Arrays.asList("0,1.0,", "1,1.0,", "2,1.0,", "3,1.0,", "4,1.0,"))), handleFailure); - TableModelUtils.assertCountData("test", "test", 350, receiverEnv, handleFailure); + TableModelUtils.assertCountData("test", "test", 350, receiverEnv); } } diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeDoubleLivingIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeDoubleLivingIT.java similarity index 96% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeDoubleLivingIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeDoubleLivingIT.java index 55c2aae1290e..f6573728cf29 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeDoubleLivingIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeDoubleLivingIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.dual.tablemodel.manual.enhanced; +package org.apache.iotdb.pipe.it.tablemodel; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; import org.apache.iotdb.confignode.rpc.thrift.TDropPipeReq; @@ -26,10 +26,8 @@ import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualEnhanced; +import org.apache.iotdb.itbase.category.MultiClusterIT2TableModel; import org.apache.iotdb.itbase.env.BaseEnv; -import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; -import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -48,8 +46,8 @@ import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2DualTableManualEnhanced.class}) -public class IoTDBPipeDoubleLivingIT extends AbstractPipeTableModelDualManualIT { +@Category({MultiClusterIT2TableModel.class}) +public class IoTDBPipeDoubleLivingIT extends AbstractPipeTableModelTestIT { @Test public void testDoubleLivingInvalidParameter() throws Exception { @@ -108,8 +106,7 @@ public void testDoubleLivingInvalidParameter() throws Exception { } } - // combination of - // org.apache.iotdb.pipe.it.tablemodel.autocreate.IoTDBPipeLifeCycleIT.testDoubleLiving and + // combination of org.apache.iotdb.pipe.it.tablemodel.IoTDBPipeLifeCycleIT.testDoubleLiving and // org.apache.iotdb.pipe.it.autocreate.IoTDBPipeLifeCycleIT.testDoubleLiving @Test public void testBasicDoubleLiving() { diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeExtractorIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeExtractorIT.java similarity index 99% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeExtractorIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeExtractorIT.java index d432f4c54480..41d6a8434d1e 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeExtractorIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeExtractorIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.dual.tablemodel.manual.basic; +package org.apache.iotdb.pipe.it.tablemodel; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; @@ -27,10 +27,8 @@ import org.apache.iotdb.it.env.MultiEnvFactory; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualBasic; +import org.apache.iotdb.itbase.category.MultiClusterIT2TableModel; import org.apache.iotdb.itbase.env.BaseEnv; -import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; -import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -52,8 +50,8 @@ import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2DualTableManualBasic.class}) -public class IoTDBPipeExtractorIT extends AbstractPipeTableModelDualManualIT { +@Category({MultiClusterIT2TableModel.class}) +public class IoTDBPipeExtractorIT extends AbstractPipeTableModelTestIT { @Before public void setUp() { diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeIsolationIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeIsolationIT.java similarity index 97% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeIsolationIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeIsolationIT.java index b0bdb5dd6737..57a3be38e217 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeIsolationIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeIsolationIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.dual.tablemodel.manual.basic; +package org.apache.iotdb.pipe.it.tablemodel; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; import org.apache.iotdb.confignode.rpc.thrift.TAlterPipeReq; @@ -26,10 +26,8 @@ import org.apache.iotdb.confignode.rpc.thrift.TStopPipeReq; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualBasic; +import org.apache.iotdb.itbase.category.MultiClusterIT2TableModel; import org.apache.iotdb.itbase.env.BaseEnv; -import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; -import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -45,8 +43,8 @@ import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2DualTableManualBasic.class}) -public class IoTDBPipeIsolationIT extends AbstractPipeTableModelDualManualIT { +@Category({MultiClusterIT2TableModel.class}) +public class IoTDBPipeIsolationIT extends AbstractPipeTableModelTestIT { @Test public void testWritePipeIsolation() throws Exception { diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeLifeCycleIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeLifeCycleIT.java similarity index 83% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeLifeCycleIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeLifeCycleIT.java index ff03b5dbc694..88cfc1bffd77 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeLifeCycleIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeLifeCycleIT.java @@ -17,19 +17,15 @@ * under the License. */ -package org.apache.iotdb.pipe.it.dual.tablemodel.manual.basic; +package org.apache.iotdb.pipe.it.tablemodel; import org.apache.iotdb.common.rpc.thrift.TSStatus; -import org.apache.iotdb.commons.auth.entity.PrivilegeType; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualBasic; -import org.apache.iotdb.itbase.env.BaseEnv; -import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; -import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; +import org.apache.iotdb.itbase.category.MultiClusterIT2TableModel; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -38,24 +34,15 @@ import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; -import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; -import static org.apache.iotdb.db.it.utils.TestUtils.assertTableNonQueryTestFail; -import static org.apache.iotdb.db.it.utils.TestUtils.assertTableTestFail; -import static org.apache.iotdb.db.it.utils.TestUtils.createUser; -import static org.apache.iotdb.db.it.utils.TestUtils.executeNonQueriesWithRetry; -import static org.apache.iotdb.db.it.utils.TestUtils.executeNonQueryWithRetry; -import static org.apache.iotdb.db.it.utils.TestUtils.executeQueryWithRetry; -import static org.apache.iotdb.db.it.utils.TestUtils.grantUserSystemPrivileges; - @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2DualTableManualBasic.class}) -public class IoTDBPipeLifeCycleIT extends AbstractPipeTableModelDualManualIT { +@Category({MultiClusterIT2TableModel.class}) +public class IoTDBPipeLifeCycleIT extends AbstractPipeTableModelTestIT { @Test public void testLifeCycleWithHistoryEnabled() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); @@ -64,8 +51,8 @@ public void testLifeCycleWithHistoryEnabled() throws Exception { final int receiverPort = receiverDataNode.getPort(); final Consumer handleFailure = o -> { - executeNonQueryWithRetry(senderEnv, "flush"); - executeNonQueryWithRetry(receiverEnv, "flush"); + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); + TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); }; boolean insertResult = true; @@ -134,8 +121,8 @@ public void testLifeCycleWithHistoryDisabled() throws Exception { final int receiverPort = receiverDataNode.getPort(); final Consumer handleFailure = o -> { - executeNonQueryWithRetry(senderEnv, "flush"); - executeNonQueryWithRetry(receiverEnv, "flush"); + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); + TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); }; boolean insertResult = true; @@ -208,8 +195,8 @@ public void testLifeCycleLogMode() throws Exception { final int receiverPort = receiverDataNode.getPort(); final Consumer handleFailure = o -> { - executeNonQueryWithRetry(senderEnv, "flush"); - executeNonQueryWithRetry(receiverEnv, "flush"); + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); + TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); }; boolean insertResult = true; @@ -270,8 +257,8 @@ public void testLifeCycleFileMode() throws Exception { final int receiverPort = receiverDataNode.getPort(); final Consumer handleFailure = o -> { - executeNonQueryWithRetry(senderEnv, "flush"); - executeNonQueryWithRetry(receiverEnv, "flush"); + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); + TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); }; boolean insertResult = true; @@ -325,8 +312,8 @@ public void testLifeCycleHybridMode() throws Exception { final int receiverPort = receiverDataNode.getPort(); final Consumer handleFailure = o -> { - executeNonQueryWithRetry(senderEnv, "flush"); - executeNonQueryWithRetry(receiverEnv, "flush"); + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); + TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); }; boolean insertResult = true; @@ -384,8 +371,8 @@ public void testLifeCycleWithClusterRestart() throws Exception { final int receiverPort = receiverDataNode.getPort(); final Consumer handleFailure = o -> { - executeNonQueryWithRetry(senderEnv, "flush"); - executeNonQueryWithRetry(receiverEnv, "flush"); + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); + TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); }; boolean insertResult = true; @@ -453,8 +440,8 @@ public void testReceiverRestartWhenTransferring() throws Exception { final int receiverPort = receiverDataNode.getPort(); final Consumer handleFailure = o -> { - executeNonQueryWithRetry(senderEnv, "flush"); - executeNonQueryWithRetry(receiverEnv, "flush"); + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); + TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); }; boolean insertResult = true; @@ -536,8 +523,8 @@ public void testReceiverAlreadyHaveTimeSeries() throws Exception { final int receiverPort = receiverDataNode.getPort(); final Consumer handleFailure = o -> { - executeNonQueryWithRetry(senderEnv, "flush"); - executeNonQueryWithRetry(receiverEnv, "flush"); + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); + TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); }; boolean insertResult = true; @@ -596,8 +583,8 @@ public void testDoubleLiving() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); final Consumer handleFailure = o -> { - executeNonQueryWithRetry(receiverEnv, "flush"); - executeNonQueryWithRetry(senderEnv, "flush"); + TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); }; final String senderIp = senderDataNode.getIp(); @@ -704,104 +691,4 @@ public void testDoubleLiving() throws Exception { TableModelUtils.assertData("test", "test", 0, 500, receiverEnv, handleFailure); } - - @Test - public void testPermission() { - createUser(senderEnv, "test", "test123"); - - assertTableNonQueryTestFail( - senderEnv, - "create pipe testPipe\n" - + "with connector (\n" - + " 'connector'='iotdb-thrift-connector',\n" - + " 'connector.ip'='127.0.0.1',\n" - + " 'connector.port'='6668'\n" - + ")", - "803: Access Denied: No permissions for this operation, please add privilege MAINTAIN", - "test", - "test123", - null); - assertTableNonQueryTestFail( - senderEnv, - "drop pipe testPipe", - "803: Access Denied: No permissions for this operation, please add privilege MAINTAIN", - "test", - "test123", - null); - assertTableTestFail( - senderEnv, - "show pipes", - "803: Access Denied: No permissions for this operation, please add privilege MAINTAIN", - "test", - "test123", - null); - assertTableNonQueryTestFail( - senderEnv, - "start pipe testPipe", - "803: Access Denied: No permissions for this operation, please add privilege MAINTAIN", - "test", - "test123", - null); - assertTableNonQueryTestFail( - senderEnv, - "stop pipe testPipe", - "803: Access Denied: No permissions for this operation, please add privilege MAINTAIN", - "test", - "test123", - null); - - assertTableNonQueryTestFail( - senderEnv, - "create pipePlugin TestProcessor as 'org.apache.iotdb.db.pipe.example.TestProcessor' USING URI 'xxx'", - "803: Access Denied: No permissions for this operation, please add privilege MAINTAIN", - "test", - "test123", - null); - assertTableNonQueryTestFail( - senderEnv, - "drop pipePlugin TestProcessor", - "803: Access Denied: No permissions for this operation, please add privilege MAINTAIN", - "test", - "test123", - null); - assertTableTestFail( - senderEnv, - "show pipe plugins", - "803: Access Denied: No permissions for this operation, please add privilege MAINTAIN", - "test", - "test123", - null); - - grantUserSystemPrivileges(senderEnv, "test", PrivilegeType.MAINTAIN); - - executeNonQueryWithRetry( - senderEnv, - "create pipe testPipe\n" - + "with connector (\n" - + " 'connector'='write-back-connector'\n" - + ")", - "test", - "test123", - null, - BaseEnv.TABLE_SQL_DIALECT); - executeQueryWithRetry( - senderEnv, "show pipes", "test", "test123", null, BaseEnv.TABLE_SQL_DIALECT); - executeNonQueriesWithRetry( - senderEnv, - Arrays.asList("start pipe testPipe", "stop pipe testPipe", "drop pipe testPipe"), - "test", - "test123", - null, - BaseEnv.TABLE_SQL_DIALECT); - - assertTableNonQueryTestFail( - senderEnv, - "create pipePlugin TestProcessor as 'org.apache.iotdb.db.pipe.example.TestProcessor' USING URI 'xxx'", - "701: Untrusted uri xxx", - "test", - "test123", - null); - executeQueryWithRetry( - senderEnv, "show pipe plugins", "test", "test123", null, BaseEnv.TABLE_SQL_DIALECT); - } } diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeNullValueIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeNullValueIT.java similarity index 94% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeNullValueIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeNullValueIT.java index de59d7ef8a63..aae3626ee531 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeNullValueIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeNullValueIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.dual.tablemodel.manual.enhanced; +package org.apache.iotdb.pipe.it.tablemodel; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; @@ -25,9 +25,7 @@ import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualEnhanced; -import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; -import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; +import org.apache.iotdb.itbase.category.MultiClusterIT2TableModel; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -40,8 +38,8 @@ import java.util.function.Consumer; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2DualTableManualEnhanced.class}) -public class IoTDBPipeNullValueIT extends AbstractPipeTableModelDualManualIT { +@Category({MultiClusterIT2TableModel.class}) +public class IoTDBPipeNullValueIT extends AbstractPipeTableModelTestIT { private enum InsertType { SESSION_INSERT_RECORD, diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeProtocolIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeProtocolIT.java similarity index 97% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeProtocolIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeProtocolIT.java index 3fe82b52b97c..046c5f68fece 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeProtocolIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeProtocolIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.dual.tablemodel.manual.basic; +package org.apache.iotdb.pipe.it.tablemodel; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; @@ -28,9 +28,7 @@ import org.apache.iotdb.it.env.MultiEnvFactory; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualBasic; -import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; -import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; +import org.apache.iotdb.itbase.category.MultiClusterIT2TableModel; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -45,8 +43,8 @@ /** Test pipe's basic functionalities under multiple cluster and consensus protocol settings. */ @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2DualTableManualBasic.class}) -public class IoTDBPipeProtocolIT extends AbstractPipeTableModelDualManualIT { +@Category({MultiClusterIT2TableModel.class}) +public class IoTDBPipeProtocolIT extends AbstractPipeTableModelTestIT { @Override @Before public void setUp() { diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeSwitchStatusIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeSwitchStatusIT.java similarity index 97% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeSwitchStatusIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeSwitchStatusIT.java index b6b29deaac2e..4c80f0ae6731 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeSwitchStatusIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeSwitchStatusIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.dual.tablemodel.manual.basic; +package org.apache.iotdb.pipe.it.tablemodel; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; @@ -27,9 +27,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TShowPipeReq; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualBasic; -import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; -import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; +import org.apache.iotdb.itbase.category.MultiClusterIT2TableModel; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -42,8 +40,8 @@ import java.util.Map; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2DualTableManualBasic.class}) -public class IoTDBPipeSwitchStatusIT extends AbstractPipeTableModelDualManualIT { +@Category({MultiClusterIT2TableModel.class}) +public class IoTDBPipeSwitchStatusIT extends AbstractPipeTableModelTestIT { @Test public void testPipeSwitchStatus() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeSyntaxIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeSyntaxIT.java similarity index 98% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeSyntaxIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeSyntaxIT.java index bc5233f1dd8c..96673a93a4a9 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeSyntaxIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeSyntaxIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.dual.tablemodel.manual.basic; +package org.apache.iotdb.pipe.it.tablemodel; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; @@ -26,9 +26,8 @@ import org.apache.iotdb.confignode.rpc.thrift.TShowPipeReq; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualBasic; +import org.apache.iotdb.itbase.category.MultiClusterIT2TableModel; import org.apache.iotdb.itbase.env.BaseEnv; -import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -47,8 +46,8 @@ import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2DualTableManualBasic.class}) -public class IoTDBPipeSyntaxIT extends AbstractPipeTableModelDualManualIT { +@Category({MultiClusterIT2TableModel.class}) +public class IoTDBPipeSyntaxIT extends AbstractPipeTableModelTestIT { @Test public void testValidPipeName() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeTypeConversionISessionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeTypeConversionISessionIT.java similarity index 96% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeTypeConversionISessionIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeTypeConversionISessionIT.java index 606fba4d17e8..dcd48928fdcc 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeTypeConversionISessionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeTypeConversionISessionIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.dual.tablemodel.manual.enhanced; +package org.apache.iotdb.pipe.it.tablemodel; import org.apache.iotdb.commons.utils.function.CheckedTriConsumer; import org.apache.iotdb.db.it.utils.TestUtils; @@ -25,9 +25,8 @@ import org.apache.iotdb.isession.ITableSession; import org.apache.iotdb.isession.SessionDataSet; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualEnhanced; +import org.apache.iotdb.itbase.category.MultiClusterIT2TableModel; import org.apache.iotdb.itbase.env.BaseEnv; -import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; import org.apache.iotdb.rpc.IoTDBConnectionException; import org.apache.iotdb.rpc.StatementExecutionException; @@ -54,7 +53,6 @@ import java.util.List; import java.util.Random; import java.util.concurrent.TimeUnit; -import java.util.function.Consumer; import static org.awaitility.Awaitility.await; import static org.junit.Assert.assertEquals; @@ -62,8 +60,8 @@ import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2DualTableManualEnhanced.class}) -public class IoTDBPipeTypeConversionISessionIT extends AbstractPipeTableModelDualManualIT { +@Category({MultiClusterIT2TableModel.class}) +public class IoTDBPipeTypeConversionISessionIT extends AbstractPipeTableModelTestIT { private static final int generateDataSize = 1000; @Test @@ -124,11 +122,6 @@ private void prepareTypeConversionTest( senderSession.executeNonQueryStatement("flush"); } - final Consumer handleFailure = - o -> { - TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); - TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); - }; // Verify receiver data long timeoutSeconds = 600; List> expectedValues = @@ -145,8 +138,7 @@ private void prepareTypeConversionTest( query(receiverSession, tablet.getSchemas(), tablet.getTableName()), expectedValues, tablet.getTimestamps()); - } catch (Exception | Error e) { - handleFailure.accept(e.getMessage()); + } catch (Exception e) { fail(e.getMessage()); } }); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeTypeConversionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeTypeConversionIT.java similarity index 97% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeTypeConversionIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeTypeConversionIT.java index d9597574c7b1..40e485b580db 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/enhanced/IoTDBPipeTypeConversionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeTypeConversionIT.java @@ -17,14 +17,13 @@ * under the License. */ -package org.apache.iotdb.pipe.it.dual.tablemodel.manual.enhanced; +package org.apache.iotdb.pipe.it.tablemodel; import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.db.pipe.receiver.transform.converter.ValueConverter; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualEnhanced; +import org.apache.iotdb.itbase.category.MultiClusterIT2TableModel; import org.apache.iotdb.itbase.env.BaseEnv; -import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; import org.apache.iotdb.rpc.RpcUtils; import org.apache.tsfile.enums.TSDataType; @@ -46,11 +45,10 @@ import java.util.List; import java.util.Random; import java.util.Set; -import java.util.function.Consumer; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2DualTableManualEnhanced.class}) -public class IoTDBPipeTypeConversionIT extends AbstractPipeTableModelDualManualIT { +@Category({MultiClusterIT2TableModel.class}) +public class IoTDBPipeTypeConversionIT extends AbstractPipeTableModelTestIT { private static final int generateDataSize = 100; @@ -204,18 +202,12 @@ public void testStringToOtherTypeConversion() { private void executeAndVerifyTypeConversion(TSDataType source, TSDataType target) { List pairs = prepareTypeConversionTest(source, target); - final Consumer handleFailure = - o -> { - TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); - TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); - }; TestUtils.assertDataEventuallyOnEnv( receiverEnv, String.format("select time,status,s1 from %s2%s", source.name(), target.name()), "time,status,s1,", createExpectedResultSet(pairs, source, target), - "test", - handleFailure); + "test"); } private List prepareTypeConversionTest(TSDataType sourceType, TSDataType targetType) { diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeWithLoadIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeWithLoadIT.java similarity index 97% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeWithLoadIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeWithLoadIT.java index 72b87d3cc288..d4f689187bfb 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeWithLoadIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeWithLoadIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.dual.tablemodel.manual.basic; +package org.apache.iotdb.pipe.it.tablemodel; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; @@ -27,10 +27,8 @@ import org.apache.iotdb.it.env.MultiEnvFactory; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualBasic; +import org.apache.iotdb.itbase.category.MultiClusterIT2TableModel; import org.apache.iotdb.itbase.env.BaseEnv; -import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; -import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -50,8 +48,8 @@ import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2DualTableManualBasic.class}) -public class IoTDBPipeWithLoadIT extends AbstractPipeTableModelDualManualIT { +@Category({MultiClusterIT2TableModel.class}) +public class IoTDBPipeWithLoadIT extends AbstractPipeTableModelTestIT { @Before public void setUp() { diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBTablePatternFormatIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBTablePatternFormatIT.java similarity index 98% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBTablePatternFormatIT.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBTablePatternFormatIT.java index b3bd7cd85861..ac1e96ab4aa9 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBTablePatternFormatIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBTablePatternFormatIT.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.dual.tablemodel.manual.basic; +package org.apache.iotdb.pipe.it.tablemodel; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; @@ -25,9 +25,7 @@ import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.MultiClusterIT2DualTableManualBasic; -import org.apache.iotdb.pipe.it.dual.tablemodel.TableModelUtils; -import org.apache.iotdb.pipe.it.dual.tablemodel.manual.AbstractPipeTableModelDualManualIT; +import org.apache.iotdb.itbase.category.MultiClusterIT2TableModel; import org.apache.iotdb.rpc.TSStatusCode; import org.junit.Assert; @@ -42,8 +40,8 @@ import java.util.function.Consumer; @RunWith(IoTDBTestRunner.class) -@Category({MultiClusterIT2DualTableManualBasic.class}) -public class IoTDBTablePatternFormatIT extends AbstractPipeTableModelDualManualIT { +@Category({MultiClusterIT2TableModel.class}) +public class IoTDBTablePatternFormatIT extends AbstractPipeTableModelTestIT { @Test public void testTableNamePattern() throws Exception { diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/TableModelUtils.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/TableModelUtils.java similarity index 99% rename from integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/TableModelUtils.java rename to integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/TableModelUtils.java index 0410a50a9775..b116cad502c2 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/TableModelUtils.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/TableModelUtils.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.pipe.it.dual.tablemodel; +package org.apache.iotdb.pipe.it.tablemodel; import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.isession.ITableSession; diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBAuthenticationTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBAuthenticationTableIT.java deleted file mode 100644 index aad3086b8080..000000000000 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBAuthenticationTableIT.java +++ /dev/null @@ -1,1001 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.iotdb.relational.it.db.it; - -import org.apache.iotdb.isession.ITableSession; -import org.apache.iotdb.it.env.EnvFactory; -import org.apache.iotdb.rpc.IoTDBConnectionException; -import org.apache.iotdb.rpc.StatementExecutionException; - -import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.write.record.Tablet; -import org.apache.tsfile.write.record.Tablet.ColumnCategory; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; - -import java.util.Arrays; -import java.util.Locale; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; - -public class IoTDBAuthenticationTableIT { - @BeforeClass - public static void setUpClass() { - Locale.setDefault(Locale.ENGLISH); - EnvFactory.getEnv().initClusterEnvironment(); - } - - @After - public void tearDown() throws Exception { - try (ITableSession sessionRoot = EnvFactory.getEnv().getTableSessionConnection()) { - String[] sqls = - new String[] { - "DROP USER userA", - "DROP USER userB", - "DROP USER userC", - "DROP USER userD", - "DROP ROLE role1", - "DROP ROLE role2", - }; - for (String sql : sqls) { - try { - sessionRoot.executeNonQueryStatement(sql); - } catch (StatementExecutionException ignore) { - } - } - } - } - - @AfterClass - public static void tearDownClass() { - EnvFactory.getEnv().cleanClusterEnvironment(); - } - - @Test - public void testInsert() throws IoTDBConnectionException, StatementExecutionException { - - try (ITableSession sessionRoot = EnvFactory.getEnv().getTableSessionConnection()) { - sessionRoot.executeNonQueryStatement("CREATE DATABASE test"); - sessionRoot.executeNonQueryStatement("USE test"); - - // insert by root - Tablet tablet = - new Tablet( - "table1", - Arrays.asList("id", "attr", "measurement"), - Arrays.asList(TSDataType.STRING, TSDataType.STRING, TSDataType.DOUBLE), - Arrays.asList(ColumnCategory.TAG, ColumnCategory.ATTRIBUTE, ColumnCategory.FIELD)); - tablet.addTimestamp(0, 0); - tablet.addValue(0, 0, "id1"); - tablet.addValue(0, 1, "attr1"); - tablet.addValue(0, 2, 0.1); - - sessionRoot.insert(tablet); - - sessionRoot.executeNonQueryStatement( - "INSERT INTO table1 (time, id, attr, measurement) VALUES (1, 'id2', 'attr2', 0.2)"); - - // revoke root - try { - sessionRoot.executeNonQueryStatement("REVOKE INSERT ON ANY FROM USER root"); - fail("Should have thrown an exception"); - } catch (StatementExecutionException e) { - assertEquals("803: Cannot grant/revoke privileges to/from admin", e.getMessage()); - } - try { - sessionRoot.executeNonQueryStatement("REVOKE INSERT ON DATABASE test FROM USER root"); - fail("Should have thrown an exception"); - } catch (StatementExecutionException e) { - assertEquals("803: Cannot grant/revoke privileges of admin user", e.getMessage()); - } - try { - sessionRoot.executeNonQueryStatement("REVOKE INSERT ON TABLE table1 FROM USER root"); - fail("Should have thrown an exception"); - } catch (StatementExecutionException e) { - assertEquals("803: Cannot grant/revoke privileges of admin user", e.getMessage()); - } - - // test users - sessionRoot.executeNonQueryStatement("CREATE USER userA 'userA'"); - sessionRoot.executeNonQueryStatement("CREATE USER userB 'userB'"); - // grant an irrelevant privilege so that the new users can use database - sessionRoot.executeNonQueryStatement("GRANT SELECT ON DATABASE test TO USER userA"); - sessionRoot.executeNonQueryStatement("GRANT SELECT ON DATABASE test TO USER userB"); - - try (ITableSession sessionA = - EnvFactory.getEnv().getTableSessionConnection("userA", "userA"); - ITableSession sessionB = - EnvFactory.getEnv().getTableSessionConnection("userB", "userB")) { - sessionA.executeNonQueryStatement("USE test"); - sessionB.executeNonQueryStatement("USE test"); - // userA no privilege - try { - sessionA.executeNonQueryStatement( - "INSERT INTO table1 (time, id, attr, measurement) VALUES (1, 'id2', 'attr2', 0.2)"); - fail("Should have thrown an exception"); - } catch (StatementExecutionException e) { - assertEquals( - "803: Access Denied: No permissions for this operation, please add privilege INSERT ON test.table1", - e.getMessage()); - } - - // grant and revoke - ALL - sessionRoot.executeNonQueryStatement("GRANT ALL TO USER userA"); - sessionA.executeNonQueryStatement( - "INSERT INTO table1 (time, id, attr, measurement) VALUES (1, 'id2', 'attr2', 0.2)"); - sessionRoot.executeNonQueryStatement("REVOKE ALL FROM USER userA"); - try { - sessionA.executeNonQueryStatement( - "INSERT INTO table1 (time, id, attr, measurement) VALUES (1, 'id2', 'attr2', 0.2)"); - fail("Should have thrown an exception"); - } catch (StatementExecutionException e) { - assertEquals( - "803: Access Denied: No permissions for this operation, please add privilege INSERT ON test.table1", - e.getMessage()); - } - - // grant and revoke - ANY - sessionRoot.executeNonQueryStatement("GRANT INSERT ON ANY TO USER userA"); - sessionA.executeNonQueryStatement( - "INSERT INTO table1 (time, id, attr, measurement) VALUES (1, 'id2', 'attr2', 0.2)"); - sessionRoot.executeNonQueryStatement("REVOKE INSERT ON ANY FROM USER userA"); - try { - sessionA.executeNonQueryStatement( - "INSERT INTO table1 (time, id, attr, measurement) VALUES (1, 'id2', 'attr2', 0.2)"); - fail("Should have thrown an exception"); - } catch (StatementExecutionException e) { - assertEquals( - "803: Access Denied: No permissions for this operation, please add privilege INSERT ON test.table1", - e.getMessage()); - } - - // grant and revoke - database - sessionRoot.executeNonQueryStatement("GRANT INSERT ON DATABASE test TO USER userA"); - sessionA.executeNonQueryStatement( - "INSERT INTO table1 (time, id, attr, measurement) VALUES (1, 'id2', 'attr2', 0.2)"); - sessionRoot.executeNonQueryStatement("REVOKE INSERT ON DATABASE test FROM USER userA"); - try { - sessionA.executeNonQueryStatement( - "INSERT INTO table1 (time, id, attr, measurement) VALUES (1, 'id2', 'attr2', 0.2)"); - fail("Should have thrown an exception"); - } catch (StatementExecutionException e) { - assertEquals( - "803: Access Denied: No permissions for this operation, please add privilege INSERT ON test.table1", - e.getMessage()); - } - - // grant and revoke - table - sessionRoot.executeNonQueryStatement("GRANT INSERT ON TABLE table1 TO USER userA"); - sessionA.executeNonQueryStatement( - "INSERT INTO table1 (time, id, attr, measurement) VALUES (1, 'id2', 'attr2', 0.2)"); - sessionRoot.executeNonQueryStatement("REVOKE INSERT ON TABLE table1 FROM USER userA"); - try { - sessionA.executeNonQueryStatement( - "INSERT INTO table1 (time, id, attr, measurement) VALUES (1, 'id2', 'attr2', 0.2)"); - fail("Should have thrown an exception"); - } catch (StatementExecutionException e) { - assertEquals( - "803: Access Denied: No permissions for this operation, please add privilege INSERT ON test.table1", - e.getMessage()); - } - - // can write but cannot auto-create - sessionRoot.executeNonQueryStatement("GRANT INSERT ON DATABASE test TO USER userA"); - tablet.setTableName("table2"); - try { - sessionA.insert(tablet); - fail("Should have thrown an exception"); - } catch (StatementExecutionException e) { - assertEquals( - "301: [EXECUTE_STATEMENT_ERROR(301)] Exception occurred: insertTablet failed. Access Denied: No permissions for this operation, please add privilege CREATE ON test.table2", - e.getMessage()); - } - sessionRoot.executeNonQueryStatement("GRANT CREATE ON DATABASE test TO USER userA"); - sessionA.insert(tablet); - sessionRoot.executeNonQueryStatement("REVOKE CREATE ON DATABASE test FROM USER userA"); - sessionRoot.executeNonQueryStatement("REVOKE INSERT ON DATABASE test FROM USER userA"); - - // can write but cannot add column - sessionRoot.executeNonQueryStatement("GRANT INSERT ON DATABASE test TO USER userA"); - tablet = - new Tablet( - "table2", - Arrays.asList("id", "attr", "measurement", "id2", "attr2", "measurement2"), - Arrays.asList( - TSDataType.STRING, - TSDataType.STRING, - TSDataType.DOUBLE, - TSDataType.STRING, - TSDataType.STRING, - TSDataType.DOUBLE), - Arrays.asList( - ColumnCategory.TAG, - ColumnCategory.ATTRIBUTE, - ColumnCategory.FIELD, - ColumnCategory.TAG, - ColumnCategory.ATTRIBUTE, - ColumnCategory.FIELD)); - tablet.addTimestamp(0, 0); - tablet.addValue(0, 0, "id1"); - tablet.addValue(0, 1, "attr1"); - tablet.addValue(0, 2, 0.1); - tablet.addValue(0, 3, "id2"); - tablet.addValue(0, 4, "attr2"); - tablet.addValue(0, 5, 0.2); - try { - sessionA.insert(tablet); - fail("Should have thrown an exception"); - } catch (StatementExecutionException e) { - assertEquals( - "301: [EXECUTE_STATEMENT_ERROR(301)] Exception occurred: insertTablet failed. Access Denied: No permissions for this operation, please add privilege ALTER ON test.table2", - e.getMessage()); - } - sessionRoot.executeNonQueryStatement("GRANT ALTER ON TABLE table2 TO USER userA"); - sessionA.insert(tablet); - sessionRoot.executeNonQueryStatement("REVOKE ALTER ON TABLE table2 FROM USER userA"); - sessionRoot.executeNonQueryStatement("REVOKE INSERT ON DATABASE test FROM USER userA"); - - // grant multiple and revoke one-by-one - sessionRoot.executeNonQueryStatement("GRANT INSERT ON ANY TO USER userA"); - sessionRoot.executeNonQueryStatement("GRANT INSERT ON DATABASE test TO USER userA"); - sessionRoot.executeNonQueryStatement("GRANT INSERT ON TABLE table1 TO USER userA"); - sessionA.executeNonQueryStatement( - "INSERT INTO table1 (time, id, attr, measurement) VALUES (1, 'id2', 'attr2', 0.2)"); - sessionRoot.executeNonQueryStatement("REVOKE INSERT ON ANY FROM USER userA"); - sessionA.executeNonQueryStatement( - "INSERT INTO table1 (time, id, attr, measurement) VALUES (1, 'id2', 'attr2', 0.2)"); - sessionRoot.executeNonQueryStatement("REVOKE INSERT ON DATABASE test FROM USER userA"); - sessionA.executeNonQueryStatement( - "INSERT INTO table1 (time, id, attr, measurement) VALUES (1, 'id2', 'attr2', 0.2)"); - sessionRoot.executeNonQueryStatement("REVOKE INSERT ON TABLE table1 FROM USER userA"); - try { - sessionA.executeNonQueryStatement( - "INSERT INTO table1 (time, id, attr, measurement) VALUES (1, 'id2', 'attr2', 0.2)"); - fail("Should have thrown an exception"); - } catch (StatementExecutionException e) { - assertEquals( - "803: Access Denied: No permissions for this operation, please add privilege INSERT ON test.table1", - e.getMessage()); - } - - // userA cannot revoke himself - sessionRoot.executeNonQueryStatement("GRANT INSERT ON ANY TO USER userA"); - try { - sessionA.executeNonQueryStatement("REVOKE INSERT ON ANY FROM USER userA"); - fail("Should have thrown an exception"); - } catch (StatementExecutionException e) { - assertEquals( - "803: Access Denied: No permissions for this operation, please add grant option to privilege INSERT", - e.getMessage()); - } - sessionRoot.executeNonQueryStatement("REVOKE INSERT ON ANY FROM USER userA"); - - // userA can revoke himself - sessionRoot.executeNonQueryStatement("GRANT INSERT ON ANY TO USER userA WITH GRANT OPTION"); - sessionA.executeNonQueryStatement("REVOKE INSERT ON ANY FROM USER userA"); - try { - sessionA.executeNonQueryStatement( - "INSERT INTO table1 (time, id, attr, measurement) VALUES (1, 'id2', 'attr2', 0.2)"); - fail("Should have thrown an exception"); - } catch (StatementExecutionException e) { - assertEquals( - "803: Access Denied: No permissions for this operation, please add privilege INSERT ON test.table1", - e.getMessage()); - } - // after revoked cannot revoke again - try { - sessionA.executeNonQueryStatement("REVOKE INSERT ON ANY FROM USER userA"); - fail("Should have thrown an exception"); - } catch (StatementExecutionException e) { - assertEquals( - "803: Access Denied: No permissions for this operation, please add grant option to privilege INSERT", - e.getMessage()); - } - - // userA cannot grant to userB - try { - sessionA.executeNonQueryStatement("GRANT INSERT ON ANY TO USER userB"); - fail("Should have thrown an exception"); - } catch (StatementExecutionException e) { - assertEquals( - "803: Access Denied: No permissions for this operation, please add grant option to privilege INSERT", - e.getMessage()); - } - sessionRoot.executeNonQueryStatement("GRANT INSERT ON ANY TO USER userA"); - try { - sessionA.executeNonQueryStatement("GRANT INSERT ON ANY TO USER userB"); - fail("Should have thrown an exception"); - } catch (StatementExecutionException e) { - assertEquals( - "803: Access Denied: No permissions for this operation, please add grant option to privilege INSERT", - e.getMessage()); - } - try { - sessionB.executeNonQueryStatement("REVOKE INSERT ON ANY FROM USER userA"); - fail("Should have thrown an exception"); - } catch (StatementExecutionException e) { - assertEquals( - "803: Access Denied: No permissions for this operation, please add grant option to privilege INSERT", - e.getMessage()); - } - - // userA can grant to userB - sessionRoot.executeNonQueryStatement("GRANT INSERT ON ANY TO USER userA WITH GRANT OPTION"); - sessionA.executeNonQueryStatement("GRANT INSERT ON ANY TO USER userB WITH GRANT OPTION"); - sessionB.executeNonQueryStatement( - "INSERT INTO table1 (time, id, attr, measurement) VALUES (1, 'id2', 'attr2', 0.2)"); - // userB can revoke userA - sessionB.executeNonQueryStatement("REVOKE INSERT ON ANY FROM USER userA"); - try { - sessionA.executeNonQueryStatement( - "INSERT INTO table1 (time, id, attr, measurement) VALUES (1, 'id2', 'attr2', 0.2)"); - fail("Should have thrown an exception"); - } catch (StatementExecutionException e) { - assertEquals( - "803: Access Denied: No permissions for this operation, please add privilege INSERT ON test.table1", - e.getMessage()); - } - // userB can revoke himself - sessionB.executeNonQueryStatement("REVOKE INSERT ON ANY FROM USER userB"); - try { - sessionB.executeNonQueryStatement("REVOKE INSERT ON ANY FROM USER userA"); - fail("Should have thrown an exception"); - } catch (StatementExecutionException e) { - assertEquals( - "803: Access Denied: No permissions for this operation, please add grant option to privilege INSERT", - e.getMessage()); - } - } - - // test role - sessionRoot.executeNonQueryStatement("CREATE USER userC 'userC'"); - sessionRoot.executeNonQueryStatement("CREATE USER userD 'userD'"); - sessionRoot.executeNonQueryStatement("CREATE ROLE role1"); - sessionRoot.executeNonQueryStatement("CREATE ROLE role2"); - sessionRoot.executeNonQueryStatement("GRANT ROLE role1 TO userC"); - sessionRoot.executeNonQueryStatement("GRANT ROLE role2 TO userD"); - - try (ITableSession sessionC = - EnvFactory.getEnv().getTableSessionConnection("userC", "userC"); - ITableSession sessionD = - EnvFactory.getEnv().getTableSessionConnection("userD", "userD")) { - // grant an irrelevant privilege so that the new users can use database - sessionRoot.executeNonQueryStatement("GRANT SELECT ON DATABASE test TO USER userC"); - sessionRoot.executeNonQueryStatement("GRANT SELECT ON DATABASE test TO USER userD"); - sessionC.executeNonQueryStatement("USE test"); - sessionD.executeNonQueryStatement("USE test"); - // userC no privilege - try { - sessionC.executeNonQueryStatement( - "INSERT INTO table1 (time, id, attr, measurement) VALUES (1, 'id2', 'attr2', 0.2)"); - fail("Should have thrown an exception"); - } catch (StatementExecutionException e) { - assertEquals( - "803: Access Denied: No permissions for this operation, please add privilege INSERT ON test.table1", - e.getMessage()); - } - - // grant and revoke - ALL - sessionRoot.executeNonQueryStatement("GRANT ALL TO ROLE role1"); - sessionC.executeNonQueryStatement( - "INSERT INTO table1 (time, id, attr, measurement) VALUES (1, 'id2', 'attr2', 0.2)"); - sessionRoot.executeNonQueryStatement("REVOKE ALL FROM ROLE role1"); - try { - sessionC.executeNonQueryStatement( - "INSERT INTO table1 (time, id, attr, measurement) VALUES (1, 'id2', 'attr2', 0.2)"); - fail("Should have thrown an exception"); - } catch (StatementExecutionException e) { - assertEquals( - "803: Access Denied: No permissions for this operation, please add privilege INSERT ON test.table1", - e.getMessage()); - } - - // grant and revoke - ANY - sessionRoot.executeNonQueryStatement("GRANT INSERT ON ANY TO ROLE role1"); - sessionC.executeNonQueryStatement( - "INSERT INTO table1 (time, id, attr, measurement) VALUES (1, 'id2', 'attr2', 0.2)"); - sessionRoot.executeNonQueryStatement("REVOKE INSERT ON ANY FROM ROLE role1"); - try { - sessionC.executeNonQueryStatement( - "INSERT INTO table1 (time, id, attr, measurement) VALUES (1, 'id2', 'attr2', 0.2)"); - fail("Should have thrown an exception"); - } catch (StatementExecutionException e) { - assertEquals( - "803: Access Denied: No permissions for this operation, please add privilege INSERT ON test.table1", - e.getMessage()); - } - - // grant and revoke - database - sessionRoot.executeNonQueryStatement("GRANT INSERT ON DATABASE test TO ROLE role1"); - sessionC.executeNonQueryStatement( - "INSERT INTO table1 (time, id, attr, measurement) VALUES (1, 'id2', 'attr2', 0.2)"); - sessionRoot.executeNonQueryStatement("REVOKE INSERT ON DATABASE test FROM ROLE role1"); - try { - sessionC.executeNonQueryStatement( - "INSERT INTO table1 (time, id, attr, measurement) VALUES (1, 'id2', 'attr2', 0.2)"); - fail("Should have thrown an exception"); - } catch (StatementExecutionException e) { - assertEquals( - "803: Access Denied: No permissions for this operation, please add privilege INSERT ON test.table1", - e.getMessage()); - } - - // grant and revoke - table - sessionRoot.executeNonQueryStatement("GRANT INSERT ON TABLE table1 TO ROLE role1"); - sessionC.executeNonQueryStatement( - "INSERT INTO table1 (time, id, attr, measurement) VALUES (1, 'id2', 'attr2', 0.2)"); - sessionRoot.executeNonQueryStatement("REVOKE INSERT ON TABLE table1 FROM ROLE role1"); - try { - sessionC.executeNonQueryStatement( - "INSERT INTO table1 (time, id, attr, measurement) VALUES (1, 'id2', 'attr2', 0.2)"); - fail("Should have thrown an exception"); - } catch (StatementExecutionException e) { - assertEquals( - "803: Access Denied: No permissions for this operation, please add privilege INSERT ON test.table1", - e.getMessage()); - } - - // grant multiple and revoke one-by-one - sessionRoot.executeNonQueryStatement("GRANT INSERT ON ANY TO ROLE role1"); - sessionRoot.executeNonQueryStatement("GRANT INSERT ON DATABASE test TO ROLE role1"); - sessionRoot.executeNonQueryStatement("GRANT INSERT ON TABLE table1 TO ROLE role1"); - sessionC.executeNonQueryStatement( - "INSERT INTO table1 (time, id, attr, measurement) VALUES (1, 'id2', 'attr2', 0.2)"); - sessionRoot.executeNonQueryStatement("REVOKE INSERT ON ANY FROM ROLE role1"); - sessionC.executeNonQueryStatement( - "INSERT INTO table1 (time, id, attr, measurement) VALUES (1, 'id2', 'attr2', 0.2)"); - sessionRoot.executeNonQueryStatement("REVOKE INSERT ON DATABASE test FROM ROLE role1"); - sessionC.executeNonQueryStatement( - "INSERT INTO table1 (time, id, attr, measurement) VALUES (1, 'id2', 'attr2', 0.2)"); - sessionRoot.executeNonQueryStatement("REVOKE INSERT ON TABLE table1 FROM ROLE role1"); - try { - sessionC.executeNonQueryStatement( - "INSERT INTO table1 (time, id, attr, measurement) VALUES (1, 'id2', 'attr2', 0.2)"); - fail("Should have thrown an exception"); - } catch (StatementExecutionException e) { - assertEquals( - "803: Access Denied: No permissions for this operation, please add privilege INSERT ON test.table1", - e.getMessage()); - } - - // role1 cannot revoke himself - sessionRoot.executeNonQueryStatement("GRANT INSERT ON ANY TO ROLE role1"); - try { - sessionC.executeNonQueryStatement("REVOKE INSERT ON ANY FROM ROLE role1"); - fail("Should have thrown an exception"); - } catch (StatementExecutionException e) { - assertEquals( - "803: Access Denied: No permissions for this operation, please add grant option to privilege INSERT", - e.getMessage()); - } - sessionRoot.executeNonQueryStatement("REVOKE INSERT ON ANY FROM ROLE role1"); - - // role1 can revoke himself - sessionRoot.executeNonQueryStatement("GRANT INSERT ON ANY TO ROLE role1 WITH GRANT OPTION"); - sessionC.executeNonQueryStatement("REVOKE INSERT ON ANY FROM ROLE role1"); - try { - sessionC.executeNonQueryStatement( - "INSERT INTO table1 (time, id, attr, measurement) VALUES (1, 'id2', 'attr2', 0.2)"); - fail("Should have thrown an exception"); - } catch (StatementExecutionException e) { - assertEquals( - "803: Access Denied: No permissions for this operation, please add privilege INSERT ON test.table1", - e.getMessage()); - } - // after revoked cannot revoke again - try { - sessionC.executeNonQueryStatement("REVOKE INSERT ON ANY FROM ROLE role1"); - fail("Should have thrown an exception"); - } catch (StatementExecutionException e) { - assertEquals( - "803: Access Denied: No permissions for this operation, please add grant option to privilege INSERT", - e.getMessage()); - } - - // role1 cannot grant to role2 - try { - sessionC.executeNonQueryStatement("GRANT INSERT ON ANY TO ROLE role2"); - fail("Should have thrown an exception"); - } catch (StatementExecutionException e) { - assertEquals( - "803: Access Denied: No permissions for this operation, please add grant option to privilege INSERT", - e.getMessage()); - } - sessionRoot.executeNonQueryStatement("GRANT INSERT ON ANY TO ROLE role1"); - try { - sessionC.executeNonQueryStatement("GRANT INSERT ON ANY TO ROLE role2"); - fail("Should have thrown an exception"); - } catch (StatementExecutionException e) { - assertEquals( - "803: Access Denied: No permissions for this operation, please add grant option to privilege INSERT", - e.getMessage()); - } - try { - sessionD.executeNonQueryStatement( - "INSERT INTO table1 (time, id, attr, measurement) VALUES (1, 'id2', 'attr2', 0.2)"); - fail("Should have thrown an exception"); - } catch (StatementExecutionException e) { - assertEquals( - "803: Access Denied: No permissions for this operation, please add privilege INSERT ON test.table1", - e.getMessage()); - } - - // userC can grant to userD - sessionRoot.executeNonQueryStatement("GRANT INSERT ON ANY TO ROLE role1 WITH GRANT OPTION"); - sessionC.executeNonQueryStatement("GRANT INSERT ON ANY TO ROLE role2 WITH GRANT OPTION"); - sessionD.executeNonQueryStatement( - "INSERT INTO table1 (time, id, attr, measurement) VALUES (1, 'id2', 'attr2', 0.2)"); - // userD can revoke userC - sessionD.executeNonQueryStatement("REVOKE INSERT ON ANY FROM ROLE role1"); - try { - sessionC.executeNonQueryStatement( - "INSERT INTO table1 (time, id, attr, measurement) VALUES (1, 'id2', 'attr2', 0.2)"); - fail("Should have thrown an exception"); - } catch (StatementExecutionException e) { - assertEquals( - "803: Access Denied: No permissions for this operation, please add privilege INSERT ON test.table1", - e.getMessage()); - } - // userD can revoke himself - sessionD.executeNonQueryStatement("REVOKE INSERT ON ANY FROM ROLE role2"); - try { - sessionD.executeNonQueryStatement("REVOKE INSERT ON ANY FROM ROLE role1"); - fail("Should have thrown an exception"); - } catch (StatementExecutionException e) { - assertEquals( - "803: Access Denied: No permissions for this operation, please add grant option to privilege INSERT", - e.getMessage()); - } - - // lose privilege after role is revoked - sessionRoot.executeNonQueryStatement("GRANT INSERT ON ANY TO ROLE role1"); - sessionRoot.executeNonQueryStatement("REVOKE ROLE role1 FROM userC"); - try { - sessionC.executeNonQueryStatement( - "INSERT INTO table1 (time, id, attr, measurement) VALUES (1, 'id2', 'attr2', 0.2)"); - fail("Should have thrown an exception"); - } catch (StatementExecutionException e) { - assertEquals( - "803: Access Denied: No permissions for this operation, please add privilege INSERT ON test.table1", - e.getMessage()); - } - } - } - } - - @Test - public void testDelete() throws IoTDBConnectionException, StatementExecutionException { - - try (ITableSession sessionRoot = EnvFactory.getEnv().getTableSessionConnection()) { - sessionRoot.executeNonQueryStatement("CREATE DATABASE test2"); - sessionRoot.executeNonQueryStatement("USE test2"); - - // insert by root - Tablet tablet = - new Tablet( - "table1", - Arrays.asList("id", "attr", "measurement"), - Arrays.asList(TSDataType.STRING, TSDataType.STRING, TSDataType.DOUBLE), - Arrays.asList(ColumnCategory.TAG, ColumnCategory.ATTRIBUTE, ColumnCategory.FIELD)); - tablet.addTimestamp(0, 0); - tablet.addValue(0, 0, "id1"); - tablet.addValue(0, 1, "attr1"); - tablet.addValue(0, 2, 0.1); - - sessionRoot.insert(tablet); - - sessionRoot.executeNonQueryStatement("DELETE FROM table1"); - - // revoke root - try { - sessionRoot.executeNonQueryStatement("REVOKE DELETE ON ANY FROM USER root"); - fail("Should have thrown an exception"); - } catch (StatementExecutionException e) { - assertEquals("803: Cannot grant/revoke privileges to/from admin", e.getMessage()); - } - try { - sessionRoot.executeNonQueryStatement("REVOKE DELETE ON DATABASE test FROM USER root"); - fail("Should have thrown an exception"); - } catch (StatementExecutionException e) { - assertEquals("803: Cannot grant/revoke privileges of admin user", e.getMessage()); - } - try { - sessionRoot.executeNonQueryStatement("REVOKE DELETE ON TABLE table1 FROM USER root"); - fail("Should have thrown an exception"); - } catch (StatementExecutionException e) { - assertEquals("803: Cannot grant/revoke privileges of admin user", e.getMessage()); - } - - // test users - sessionRoot.executeNonQueryStatement("CREATE USER userA 'userA'"); - sessionRoot.executeNonQueryStatement("CREATE USER userB 'userB'"); - - try (ITableSession sessionA = - EnvFactory.getEnv().getTableSessionConnection("userA", "userA"); - ITableSession sessionB = - EnvFactory.getEnv().getTableSessionConnection("userB", "userB")) { - // grant an irrelevant privilege so that the new users can use database - sessionRoot.executeNonQueryStatement("GRANT SELECT ON DATABASE test2 TO USER userA"); - sessionRoot.executeNonQueryStatement("GRANT SELECT ON DATABASE test2 TO USER userB"); - sessionA.executeNonQueryStatement("USE test2"); - sessionB.executeNonQueryStatement("USE test2"); - // userA no privilege - try { - sessionA.executeNonQueryStatement("DELETE FROM table1"); - fail("Should have thrown an exception"); - } catch (StatementExecutionException e) { - assertEquals( - "803: Access Denied: No permissions for this operation, please add privilege DELETE ON test2.table1", - e.getMessage()); - } - - // grant and revoke - ALL - sessionRoot.executeNonQueryStatement("GRANT ALL TO USER userA"); - sessionA.executeNonQueryStatement("DELETE FROM table1"); - sessionRoot.executeNonQueryStatement("REVOKE ALL FROM USER userA"); - try { - sessionA.executeNonQueryStatement("DELETE FROM table1"); - fail("Should have thrown an exception"); - } catch (StatementExecutionException e) { - assertEquals( - "803: Access Denied: No permissions for this operation, please add privilege DELETE ON test2.table1", - e.getMessage()); - } - - // grant and revoke - ANY - sessionRoot.executeNonQueryStatement("GRANT DELETE ON ANY TO USER userA"); - sessionA.executeNonQueryStatement("DELETE FROM table1"); - sessionRoot.executeNonQueryStatement("REVOKE DELETE ON ANY FROM USER userA"); - try { - sessionA.executeNonQueryStatement("DELETE FROM table1"); - fail("Should have thrown an exception"); - } catch (StatementExecutionException e) { - assertEquals( - "803: Access Denied: No permissions for this operation, please add privilege DELETE ON test2.table1", - e.getMessage()); - } - - // grant and revoke - database - sessionRoot.executeNonQueryStatement("GRANT DELETE ON DATABASE test2 TO USER userA"); - sessionA.executeNonQueryStatement("DELETE FROM table1"); - sessionRoot.executeNonQueryStatement("REVOKE DELETE ON DATABASE test2 FROM USER userA"); - try { - sessionA.executeNonQueryStatement("DELETE FROM table1"); - fail("Should have thrown an exception"); - } catch (StatementExecutionException e) { - assertEquals( - "803: Access Denied: No permissions for this operation, please add privilege DELETE ON test2.table1", - e.getMessage()); - } - - // grant and revoke - table - sessionRoot.executeNonQueryStatement("GRANT DELETE ON TABLE table1 TO USER userA"); - sessionA.executeNonQueryStatement("DELETE FROM table1"); - sessionRoot.executeNonQueryStatement("REVOKE DELETE ON TABLE table1 FROM USER userA"); - try { - sessionA.executeNonQueryStatement("DELETE FROM table1"); - fail("Should have thrown an exception"); - } catch (StatementExecutionException e) { - assertEquals( - "803: Access Denied: No permissions for this operation, please add privilege DELETE ON test2.table1", - e.getMessage()); - } - - // grant multiple and revoke one-by-one - sessionRoot.executeNonQueryStatement("GRANT DELETE ON ANY TO USER userA"); - sessionRoot.executeNonQueryStatement("GRANT DELETE ON DATABASE test2 TO USER userA"); - sessionRoot.executeNonQueryStatement("GRANT DELETE ON TABLE table1 TO USER userA"); - sessionA.executeNonQueryStatement("DELETE FROM table1"); - sessionRoot.executeNonQueryStatement("REVOKE DELETE ON ANY FROM USER userA"); - sessionA.executeNonQueryStatement("DELETE FROM table1"); - sessionRoot.executeNonQueryStatement("REVOKE DELETE ON DATABASE test2 FROM USER userA"); - sessionA.executeNonQueryStatement("DELETE FROM table1"); - sessionRoot.executeNonQueryStatement("REVOKE DELETE ON TABLE table1 FROM USER userA"); - try { - sessionA.executeNonQueryStatement("DELETE FROM table1"); - fail("Should have thrown an exception"); - } catch (StatementExecutionException e) { - assertEquals( - "803: Access Denied: No permissions for this operation, please add privilege DELETE ON test2.table1", - e.getMessage()); - } - - // userA cannot revoke himself - sessionRoot.executeNonQueryStatement("GRANT DELETE ON ANY TO USER userA"); - try { - sessionA.executeNonQueryStatement("REVOKE DELETE ON ANY FROM USER userA"); - fail("Should have thrown an exception"); - } catch (StatementExecutionException e) { - assertEquals( - "803: Access Denied: No permissions for this operation, please add grant option to privilege DELETE", - e.getMessage()); - } - sessionRoot.executeNonQueryStatement("REVOKE DELETE ON ANY FROM USER userA"); - - // userA can revoke himself - sessionRoot.executeNonQueryStatement("GRANT DELETE ON ANY TO USER userA WITH GRANT OPTION"); - sessionA.executeNonQueryStatement("REVOKE DELETE ON ANY FROM USER userA"); - try { - sessionA.executeNonQueryStatement("DELETE FROM table1"); - fail("Should have thrown an exception"); - } catch (StatementExecutionException e) { - assertEquals( - "803: Access Denied: No permissions for this operation, please add privilege DELETE ON test2.table1", - e.getMessage()); - } - // after revoked cannot revoke again - try { - sessionA.executeNonQueryStatement("REVOKE DELETE ON ANY FROM USER userA"); - fail("Should have thrown an exception"); - } catch (StatementExecutionException e) { - assertEquals( - "803: Access Denied: No permissions for this operation, please add grant option to privilege DELETE", - e.getMessage()); - } - - // userA cannot grant to userB - try { - sessionA.executeNonQueryStatement("GRANT DELETE ON ANY TO USER userB"); - fail("Should have thrown an exception"); - } catch (StatementExecutionException e) { - assertEquals( - "803: Access Denied: No permissions for this operation, please add grant option to privilege DELETE", - e.getMessage()); - } - sessionRoot.executeNonQueryStatement("GRANT DELETE ON ANY TO USER userA"); - try { - sessionA.executeNonQueryStatement("GRANT DELETE ON ANY TO USER userB"); - fail("Should have thrown an exception"); - } catch (StatementExecutionException e) { - assertEquals( - "803: Access Denied: No permissions for this operation, please add grant option to privilege DELETE", - e.getMessage()); - } - try { - sessionB.executeNonQueryStatement("REVOKE DELETE ON ANY FROM USER userA"); - fail("Should have thrown an exception"); - } catch (StatementExecutionException e) { - assertEquals( - "803: Access Denied: No permissions for this operation, please add grant option to privilege DELETE", - e.getMessage()); - } - - // userA can grant to userB - sessionRoot.executeNonQueryStatement("GRANT DELETE ON ANY TO USER userA WITH GRANT OPTION"); - sessionA.executeNonQueryStatement("GRANT DELETE ON ANY TO USER userB WITH GRANT OPTION"); - sessionB.executeNonQueryStatement("DELETE FROM table1"); - // userB can revoke userA - sessionB.executeNonQueryStatement("REVOKE DELETE ON ANY FROM USER userA"); - try { - sessionA.executeNonQueryStatement("DELETE FROM table1"); - fail("Should have thrown an exception"); - } catch (StatementExecutionException e) { - assertEquals( - "803: Access Denied: No permissions for this operation, please add privilege DELETE ON test2.table1", - e.getMessage()); - } - // userB can revoke himself - sessionB.executeNonQueryStatement("REVOKE DELETE ON ANY FROM USER userB"); - try { - sessionB.executeNonQueryStatement("REVOKE DELETE ON ANY FROM USER userA"); - fail("Should have thrown an exception"); - } catch (StatementExecutionException e) { - assertEquals( - "803: Access Denied: No permissions for this operation, please add grant option to privilege DELETE", - e.getMessage()); - } - } - - // test role - sessionRoot.executeNonQueryStatement("CREATE USER userC 'userC'"); - sessionRoot.executeNonQueryStatement("CREATE USER userD 'userD'"); - sessionRoot.executeNonQueryStatement("CREATE ROLE role1"); - sessionRoot.executeNonQueryStatement("CREATE ROLE role2"); - sessionRoot.executeNonQueryStatement("GRANT ROLE role1 TO userC"); - sessionRoot.executeNonQueryStatement("GRANT ROLE role2 TO userD"); - - try (ITableSession sessionC = - EnvFactory.getEnv().getTableSessionConnection("userC", "userC"); - ITableSession sessionD = - EnvFactory.getEnv().getTableSessionConnection("userD", "userD")) { - // grant an irrelevant privilege so that the new users can use database - sessionRoot.executeNonQueryStatement("GRANT SELECT ON DATABASE test2 TO USER userC"); - sessionRoot.executeNonQueryStatement("GRANT SELECT ON DATABASE test2 TO USER userD"); - sessionC.executeNonQueryStatement("USE test2"); - sessionD.executeNonQueryStatement("USE test2"); - // userC no privilege - try { - sessionC.executeNonQueryStatement("DELETE FROM table1"); - fail("Should have thrown an exception"); - } catch (StatementExecutionException e) { - assertEquals( - "803: Access Denied: No permissions for this operation, please add privilege DELETE ON test2.table1", - e.getMessage()); - } - - // grant and revoke - ALL - sessionRoot.executeNonQueryStatement("GRANT ALL TO ROLE role1"); - sessionC.executeNonQueryStatement("DELETE FROM table1"); - sessionRoot.executeNonQueryStatement("REVOKE ALL FROM ROLE role1"); - try { - sessionC.executeNonQueryStatement("DELETE FROM table1"); - fail("Should have thrown an exception"); - } catch (StatementExecutionException e) { - assertEquals( - "803: Access Denied: No permissions for this operation, please add privilege DELETE ON test2.table1", - e.getMessage()); - } - - // grant and revoke - ANY - sessionRoot.executeNonQueryStatement("GRANT DELETE ON ANY TO ROLE role1"); - sessionC.executeNonQueryStatement("DELETE FROM table1"); - sessionRoot.executeNonQueryStatement("REVOKE DELETE ON ANY FROM ROLE role1"); - try { - sessionC.executeNonQueryStatement("DELETE FROM table1"); - fail("Should have thrown an exception"); - } catch (StatementExecutionException e) { - assertEquals( - "803: Access Denied: No permissions for this operation, please add privilege DELETE ON test2.table1", - e.getMessage()); - } - - // grant and revoke - database - sessionRoot.executeNonQueryStatement("GRANT DELETE ON DATABASE test2 TO ROLE role1"); - sessionC.executeNonQueryStatement("DELETE FROM table1"); - sessionRoot.executeNonQueryStatement("REVOKE DELETE ON DATABASE test2 FROM ROLE role1"); - try { - sessionC.executeNonQueryStatement("DELETE FROM table1"); - fail("Should have thrown an exception"); - } catch (StatementExecutionException e) { - assertEquals( - "803: Access Denied: No permissions for this operation, please add privilege DELETE ON test2.table1", - e.getMessage()); - } - - // grant and revoke - table - sessionRoot.executeNonQueryStatement("GRANT DELETE ON TABLE table1 TO ROLE role1"); - sessionC.executeNonQueryStatement("DELETE FROM table1"); - sessionRoot.executeNonQueryStatement("REVOKE DELETE ON TABLE table1 FROM ROLE role1"); - try { - sessionC.executeNonQueryStatement("DELETE FROM table1"); - fail("Should have thrown an exception"); - } catch (StatementExecutionException e) { - assertEquals( - "803: Access Denied: No permissions for this operation, please add privilege DELETE ON test2.table1", - e.getMessage()); - } - - // grant multiple and revoke one-by-one - sessionRoot.executeNonQueryStatement("GRANT DELETE ON ANY TO ROLE role1"); - sessionRoot.executeNonQueryStatement("GRANT DELETE ON DATABASE test2 TO ROLE role1"); - sessionRoot.executeNonQueryStatement("GRANT DELETE ON TABLE table1 TO ROLE role1"); - sessionC.executeNonQueryStatement("DELETE FROM table1"); - sessionRoot.executeNonQueryStatement("REVOKE DELETE ON ANY FROM ROLE role1"); - sessionC.executeNonQueryStatement("DELETE FROM table1"); - sessionRoot.executeNonQueryStatement("REVOKE DELETE ON DATABASE test2 FROM ROLE role1"); - sessionC.executeNonQueryStatement("DELETE FROM table1"); - sessionRoot.executeNonQueryStatement("REVOKE DELETE ON TABLE table1 FROM ROLE role1"); - try { - sessionC.executeNonQueryStatement("DELETE FROM table1"); - fail("Should have thrown an exception"); - } catch (StatementExecutionException e) { - assertEquals( - "803: Access Denied: No permissions for this operation, please add privilege DELETE ON test2.table1", - e.getMessage()); - } - - // role1 cannot revoke himself - sessionRoot.executeNonQueryStatement("GRANT DELETE ON ANY TO ROLE role1"); - try { - sessionC.executeNonQueryStatement("REVOKE DELETE ON ANY FROM ROLE role1"); - fail("Should have thrown an exception"); - } catch (StatementExecutionException e) { - assertEquals( - "803: Access Denied: No permissions for this operation, please add grant option to privilege DELETE", - e.getMessage()); - } - sessionRoot.executeNonQueryStatement("REVOKE DELETE ON ANY FROM ROLE role1"); - - // role1 can revoke himself - sessionRoot.executeNonQueryStatement("GRANT DELETE ON ANY TO ROLE role1 WITH GRANT OPTION"); - sessionC.executeNonQueryStatement("REVOKE DELETE ON ANY FROM ROLE role1"); - try { - sessionC.executeNonQueryStatement("DELETE FROM table1"); - fail("Should have thrown an exception"); - } catch (StatementExecutionException e) { - assertEquals( - "803: Access Denied: No permissions for this operation, please add privilege DELETE ON test2.table1", - e.getMessage()); - } - // after revoked cannot revoke again - try { - sessionC.executeNonQueryStatement("REVOKE DELETE ON ANY FROM ROLE role1"); - fail("Should have thrown an exception"); - } catch (StatementExecutionException e) { - assertEquals( - "803: Access Denied: No permissions for this operation, please add grant option to privilege DELETE", - e.getMessage()); - } - - // role1 cannot grant to role2 - try { - sessionC.executeNonQueryStatement("GRANT DELETE ON ANY TO ROLE role2"); - fail("Should have thrown an exception"); - } catch (StatementExecutionException e) { - assertEquals( - "803: Access Denied: No permissions for this operation, please add grant option to privilege DELETE", - e.getMessage()); - } - sessionRoot.executeNonQueryStatement("GRANT DELETE ON ANY TO ROLE role1"); - try { - sessionC.executeNonQueryStatement("GRANT DELETE ON ANY TO ROLE role2"); - fail("Should have thrown an exception"); - } catch (StatementExecutionException e) { - assertEquals( - "803: Access Denied: No permissions for this operation, please add grant option to privilege DELETE", - e.getMessage()); - } - try { - sessionD.executeNonQueryStatement("DELETE FROM table1"); - fail("Should have thrown an exception"); - } catch (StatementExecutionException e) { - assertEquals( - "803: Access Denied: No permissions for this operation, please add privilege DELETE ON test2.table1", - e.getMessage()); - } - - // role1 can grant to role2 - sessionRoot.executeNonQueryStatement("GRANT DELETE ON ANY TO ROLE role1 WITH GRANT OPTION"); - sessionC.executeNonQueryStatement("GRANT DELETE ON ANY TO ROLE role2 WITH GRANT OPTION"); - sessionD.executeNonQueryStatement("DELETE FROM table1"); - // role2 can revoke role1 - sessionD.executeNonQueryStatement("REVOKE DELETE ON ANY FROM ROLE role1"); - try { - sessionC.executeNonQueryStatement("DELETE FROM table1"); - fail("Should have thrown an exception"); - } catch (StatementExecutionException e) { - assertEquals( - "803: Access Denied: No permissions for this operation, please add privilege DELETE ON test2.table1", - e.getMessage()); - } - // role2 can revoke himself - sessionD.executeNonQueryStatement("REVOKE DELETE ON ANY FROM ROLE role2"); - try { - sessionD.executeNonQueryStatement("REVOKE DELETE ON ANY FROM ROLE role1"); - fail("Should have thrown an exception"); - } catch (StatementExecutionException e) { - assertEquals( - "803: Access Denied: No permissions for this operation, please add grant option to privilege DELETE", - e.getMessage()); - } - - // lose privilege after role is revoked - sessionRoot.executeNonQueryStatement("GRANT DELETE ON ANY TO ROLE role1"); - sessionRoot.executeNonQueryStatement("REVOKE ROLE role1 FROM userC"); - try { - sessionC.executeNonQueryStatement("DELETE FROM table1"); - fail("Should have thrown an exception"); - } catch (StatementExecutionException e) { - assertEquals( - "803: Access Denied: No permissions for this operation, please add privilege DELETE ON test2.table1", - e.getMessage()); - } - } - } - } -} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java index ab4f64366143..95eee41d007a 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java @@ -24,6 +24,8 @@ import org.apache.iotdb.isession.SessionDataSet; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.ClusterIT; +import org.apache.iotdb.itbase.category.LocalStandaloneIT; import org.apache.iotdb.itbase.category.ManualIT; import org.apache.iotdb.itbase.category.TableClusterIT; import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; @@ -1645,6 +1647,7 @@ private List collectDataRanges(Statement statement, long timeUpperBou } @Test + @Category({LocalStandaloneIT.class, ClusterIT.class}) public void deleteTableOfTheSameNameTest() throws IoTDBConnectionException, StatementExecutionException { int testNum = 24; @@ -1773,47 +1776,6 @@ public void testConcurrentFlushAndRandomDeviceDeletion() } } - @Test - public void testCaseSensitivity() throws IoTDBConnectionException, StatementExecutionException { - try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { - session.executeNonQueryStatement("CREATE DATABASE IF NOT EXISTS db1"); - session.executeNonQueryStatement("USE db1"); - session.executeNonQueryStatement("CREATE TABLE case_sensitivity (tag1 TAG, s1 INT32)"); - - session.executeNonQueryStatement( - "INSERT INTO case_sensitivity (time, tag1, s1) VALUES (1, 'd1', 1)"); - session.executeNonQueryStatement( - "INSERT INTO case_sensitivity (time, tag1, s1) VALUES (2, 'd2', 2)"); - session.executeNonQueryStatement( - "INSERT INTO case_sensitivity (time, tag1, s1) VALUES (3, 'd3', 3)"); - - session.executeNonQueryStatement("DELETE FROM DB1.case_sensitivity where time = 1"); - SessionDataSet dataSet = - session.executeQueryStatement("select * from db1.case_sensitivity order by time"); - RowRecord rec = dataSet.next(); - assertEquals(2, rec.getFields().get(0).getLongV()); - assertEquals("d2", rec.getFields().get(1).toString()); - assertEquals(2, rec.getFields().get(2).getIntV()); - rec = dataSet.next(); - assertEquals(3, rec.getFields().get(0).getLongV()); - assertEquals("d3", rec.getFields().get(1).toString()); - assertEquals(3, rec.getFields().get(2).getIntV()); - assertFalse(dataSet.hasNext()); - - session.executeNonQueryStatement("DELETE FROM db1.CASE_sensitivity where time = 2"); - dataSet = session.executeQueryStatement("select * from db1.case_sensitivity order by time"); - rec = dataSet.next(); - assertEquals(3, rec.getFields().get(0).getLongV()); - assertEquals("d3", rec.getFields().get(1).toString()); - assertEquals(3, rec.getFields().get(2).getIntV()); - assertFalse(dataSet.hasNext()); - - session.executeNonQueryStatement("DELETE FROM db1.CASE_sensitivity where TAG1 = 'd3'"); - dataSet = session.executeQueryStatement("select * from db1.case_sensitivity order by time"); - assertFalse(dataSet.hasNext()); - } - } - @Ignore("performance") @Test public void testDeletionWritePerformance() throws SQLException, IOException { diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBInsertTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBInsertTableIT.java index af51ab3fedd2..e751ab6423f2 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBInsertTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBInsertTableIT.java @@ -491,17 +491,6 @@ public void testInsertMultiRowWithWrongTimestampPrecision() throws SQLException assertTrue(e.getMessage().contains("Current system timestamp precision is ms")); } } - try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); - Statement st1 = connection.createStatement()) { - try { - st1.execute("use \"test\""); - st1.execute( - "insert into wf16(tag1, time, status) values('wt01', -1618283005586000, true), ('wt01', -1618283005586001, false)"); - fail(); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("Current system timestamp precision is ms")); - } - } } @Test diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLoadConfigurationTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLoadConfigurationTableIT.java deleted file mode 100644 index 53e8bc9487e3..000000000000 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBLoadConfigurationTableIT.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.relational.it.db.it; - -import org.apache.iotdb.it.env.EnvFactory; -import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; -import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.TableClusterIT; -import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; -import org.apache.iotdb.itbase.env.BaseEnv; - -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; - -import java.io.File; -import java.io.FileWriter; -import java.io.IOException; -import java.nio.channels.FileChannel; -import java.nio.file.StandardOpenOption; -import java.sql.Connection; -import java.sql.Statement; - -@RunWith(IoTDBTestRunner.class) -@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) -public class IoTDBLoadConfigurationTableIT { - @BeforeClass - public static void setUp() throws Exception { - EnvFactory.getEnv().initClusterEnvironment(); - } - - @AfterClass - public static void tearDown() throws Exception { - EnvFactory.getEnv().cleanClusterEnvironment(); - } - - @Test - public void loadConfiguration() throws IOException { - DataNodeWrapper dataNodeWrapper = EnvFactory.getEnv().getDataNodeWrapper(0); - String confPath = - dataNodeWrapper.getNodePath() - + File.separator - + "conf" - + File.separator - + "iotdb-system.properties"; - long length = new File(confPath).length(); - try (FileWriter fileWriter = new FileWriter(confPath, true)) { - fileWriter.write(System.lineSeparator()); - fileWriter.write("target_compaction_file_size=t"); - } - - try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); - Statement statement = connection.createStatement()) { - statement.execute("LOAD CONFIGURATION"); - Assert.fail(); - } catch (Exception e) { - Assert.assertTrue(e.getMessage().contains("NumberFormatException")); - } finally { - try (FileChannel fileChannel = - FileChannel.open(new File(confPath).toPath(), StandardOpenOption.WRITE)) { - fileChannel.truncate(length); - } - } - } -} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBSetSystemStatusTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBSetSystemStatusTableIT.java deleted file mode 100644 index 5a3c435fb8b4..000000000000 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBSetSystemStatusTableIT.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.relational.it.db.it; - -import org.apache.iotdb.it.env.EnvFactory; -import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.TableClusterIT; -import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; -import org.apache.iotdb.itbase.env.BaseEnv; -import org.apache.iotdb.itbase.exception.InconsistentDataException; - -import org.awaitility.Awaitility; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; - -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.Statement; -import java.util.concurrent.TimeUnit; - -@RunWith(IoTDBTestRunner.class) -@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) -public class IoTDBSetSystemStatusTableIT { - @BeforeClass - public static void setUp() throws Exception { - EnvFactory.getEnv().initClusterEnvironment(); - } - - @AfterClass - public static void tearDown() throws Exception { - EnvFactory.getEnv().cleanClusterEnvironment(); - } - - @Test - public void setSystemStatus() { - try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); - Statement statement = connection.createStatement()) { - statement.execute("SET SYSTEM TO READONLY ON CLUSTER"); - Awaitility.await() - .atMost(10, TimeUnit.SECONDS) - .pollDelay(1, TimeUnit.SECONDS) - .until( - () -> { - ResultSet resultSet = statement.executeQuery("SHOW DATANODES"); - int num = 0; - try { - while (resultSet.next()) { - String status = resultSet.getString("Status"); - if (status.equals("ReadOnly")) { - num++; - } - } - } catch (InconsistentDataException e) { - return false; - } - return num == EnvFactory.getEnv().getDataNodeWrapperList().size(); - }); - - statement.execute("SET SYSTEM TO RUNNING ON CLUSTER"); - Awaitility.await() - .atMost(10, TimeUnit.SECONDS) - .pollDelay(1, TimeUnit.SECONDS) - .until( - () -> { - ResultSet resultSet = statement.executeQuery("SHOW DATANODES"); - int num = 0; - try { - while (resultSet.next()) { - String status = resultSet.getString("Status"); - if (status.equals("Running")) { - num++; - } - } - } catch (InconsistentDataException e) { - return false; - } - return num == EnvFactory.getEnv().getDataNodeWrapperList().size(); - }); - } catch (Exception e) { - Assert.fail(e.getMessage()); - } - } -} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/query/IoTDBGreatestLeastTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/query/IoTDBGreatestLeastTableIT.java deleted file mode 100644 index 529b4daeabe8..000000000000 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/query/IoTDBGreatestLeastTableIT.java +++ /dev/null @@ -1,305 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.iotdb.relational.it.query.old.query; - -import org.apache.iotdb.it.env.EnvFactory; -import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.TableClusterIT; -import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; -import org.apache.iotdb.itbase.env.BaseEnv; - -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; - -import java.sql.Connection; -import java.sql.Statement; - -import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; -import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; - -@RunWith(IoTDBTestRunner.class) -@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) -public class IoTDBGreatestLeastTableIT { - - private static final String DATABASE_NAME = "db"; - - private static final String[] SQLs = - new String[] { - "CREATE DATABASE " + DATABASE_NAME, - "USE " + DATABASE_NAME, - "CREATE TABLE boolean_table(device_id STRING TAG, bool1 BOOLEAN FIELD, bool2 BOOLEAN FIELD)", - "CREATE TABLE number_table(device_id STRING TAG, int1 INT32 FIELD, int2 INT32 FIELD, long1 INT64 FIELD, long2 INT64 FIELD, float1 FLOAT FIELD, float2 FLOAT FIELD, double1 DOUBLE FIELD, double2 DOUBLE FIELD)", - "CREATE TABLE string_table(device_id STRING TAG, string1 STRING FIELD, string2 STRING FIELD, text1 TEXT FIELD, text2 TEXT FIELD)", - "CREATE TABLE mix_type_table(device_id STRING TAG, s1 INT32 FIELD, s2 INT64 FIELD, s3 FLOAT FIELD, s4 DOUBLE FIELD, s5 BOOLEAN FIELD, s6 STRING FIELD, s7 TEXT FIELD)", - "CREATE TABLE null_table(device_id STRING TAG, string1 STRING FIELD, string2 STRING FIELD, int1 INT32 FIELD, int2 INT32 FIELD, double1 DOUBLE FIELD, double2 DOUBLE FIELD, timestamp1 TIMESTAMP FIELD, timestamp2 TIMESTAMP FIELD)", - "CREATE TABLE any_null_table(device_id STRING TAG, string1 STRING FIELD, string2 STRING FIELD, int1 INT32 FIELD, int2 INT32 FIELD, double1 DOUBLE FIELD, double2 DOUBLE FIELD, timestamp1 TIMESTAMP FIELD, timestamp2 TIMESTAMP FIELD)", - // normal case - "INSERT INTO number_table(time, device_id, int1, int2, long1, long2, float1, float2, double1, double2) VALUES (10, 'd1', 1000000, 2000000, 1000000, 2000000, 10.1, 20.2, 10.1, 20.2)", - "INSERT INTO string_table(time, device_id, string1, string2, text1, text2) VALUES(10, 'd1', 'aaa', 'bbb', 'aaa', 'bbb')", - "INSERT INTO boolean_table(time, device_id, bool1, bool2) VALUES(10, 'd1', true, false)", - "INSERT INTO mix_type_table(time, device_id, s1, s2, s3, s4, s5, s6, s7) VALUES(10, 'd1', 1, 1, 1.0, 1.0, true, 'a', 'a')", - "INSERT INTO null_table(time, device_id) VALUES(10, 'd1')", - "INSERT INTO any_null_table(time, device_id, string2, int2, double2, timestamp2) VALUES(10, 'd1', 'test', 10, 10.0, 10)", - }; - - @BeforeClass - public static void setUp() throws Exception { - EnvFactory.getEnv().initClusterEnvironment(); - insertData(); - } - - protected static void insertData() { - try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); - Statement statement = connection.createStatement()) { - - for (String sql : SQLs) { - statement.execute(sql); - } - } catch (Exception e) { - e.printStackTrace(); - } - } - - @AfterClass - public static void tearDown() throws Exception { - EnvFactory.getEnv().cleanClusterEnvironment(); - } - - @Test - public void testNumberTypeGreatestFunction() { - - tableResultSetEqualTest( - "SELECT GREATEST(int1, int2) FROM number_table", - new String[] {"_col0"}, - new String[] {"2000000,"}, - DATABASE_NAME); - - tableResultSetEqualTest( - "SELECT GREATEST(long1, long2) FROM number_table", - new String[] {"_col0"}, - new String[] {"2000000,"}, - DATABASE_NAME); - - tableResultSetEqualTest( - "SELECT GREATEST(float1, float2) FROM number_table", - new String[] {"_col0"}, - new String[] {"20.2,"}, - DATABASE_NAME); - - tableResultSetEqualTest( - "SELECT GREATEST(double1, double2) FROM number_table", - new String[] {"_col0"}, - new String[] {"20.2,"}, - DATABASE_NAME); - } - - @Test - public void testNumberTypeLeastFunction() { - tableResultSetEqualTest( - "SELECT LEAST(int1, int2) FROM number_table", - new String[] {"_col0"}, - new String[] {"1000000,"}, - DATABASE_NAME); - - tableResultSetEqualTest( - "SELECT LEAST(long1, long2) FROM number_table", - new String[] {"_col0"}, - new String[] {"1000000,"}, - DATABASE_NAME); - - tableResultSetEqualTest( - "SELECT LEAST(float1, float2) FROM number_table", - new String[] {"_col0"}, - new String[] {"10.1,"}, - DATABASE_NAME); - - tableResultSetEqualTest( - "SELECT LEAST(double1, double2) FROM number_table", - new String[] {"_col0"}, - new String[] {"10.1,"}, - DATABASE_NAME); - } - - @Test - public void testStringTypeGreatestFunction() { - tableResultSetEqualTest( - "SELECT GREATEST(string1, string2) FROM string_table", - new String[] {"_col0"}, - new String[] {"bbb,"}, - DATABASE_NAME); - - tableResultSetEqualTest( - "SELECT GREATEST(text1, text2) FROM string_table", - new String[] {"_col0"}, - new String[] {"bbb,"}, - DATABASE_NAME); - } - - @Test - public void testStringTypeLeastFunction() { - tableResultSetEqualTest( - "SELECT LEAST(string1, string2) FROM string_table", - new String[] {"_col0"}, - new String[] {"aaa,"}, - DATABASE_NAME); - - tableResultSetEqualTest( - "SELECT LEAST(text1, text2) FROM string_table", - new String[] {"_col0"}, - new String[] {"aaa,"}, - DATABASE_NAME); - } - - @Test - public void testBooleanTypeGreatestFunction() { - tableResultSetEqualTest( - "SELECT GREATEST(bool1, bool2) FROM boolean_table", - new String[] {"_col0"}, - new String[] {"true,"}, - DATABASE_NAME); - } - - @Test - public void testBooleanTypeLeastFunction() { - tableResultSetEqualTest( - "SELECT LEAST(bool1, bool2) FROM boolean_table", - new String[] {"_col0"}, - new String[] {"false,"}, - DATABASE_NAME); - } - - @Test - public void testAllNullValue() { - tableResultSetEqualTest( - "SELECT GREATEST(string1, string2) FROM null_table", - new String[] {"_col0"}, - new String[] {"null,"}, - DATABASE_NAME); - - tableResultSetEqualTest( - "SELECT LEAST(string1, string2) FROM null_table", - new String[] {"_col0"}, - new String[] {"null,"}, - DATABASE_NAME); - - tableResultSetEqualTest( - "SELECT GREATEST(int1, int2) FROM null_table", - new String[] {"_col0"}, - new String[] {"null,"}, - DATABASE_NAME); - - tableResultSetEqualTest( - "SELECT LEAST(int1, int2) FROM null_table", - new String[] {"_col0"}, - new String[] {"null,"}, - DATABASE_NAME); - - tableResultSetEqualTest( - "SELECT GREATEST(double1, double2) FROM null_table", - new String[] {"_col0"}, - new String[] {"null,"}, - DATABASE_NAME); - - tableResultSetEqualTest( - "SELECT LEAST(double1, double2) FROM null_table", - new String[] {"_col0"}, - new String[] {"null,"}, - DATABASE_NAME); - - tableResultSetEqualTest( - "SELECT GREATEST(timestamp1, timestamp2) FROM null_table", - new String[] {"_col0"}, - new String[] {"null,"}, - DATABASE_NAME); - - tableResultSetEqualTest( - "SELECT LEAST(timestamp1, timestamp2) FROM null_table", - new String[] {"_col0"}, - new String[] {"null,"}, - DATABASE_NAME); - } - - @Test - public void testAnyNullValue() { - tableResultSetEqualTest( - "SELECT GREATEST(string1, string2) FROM any_null_table", - new String[] {"_col0"}, - new String[] {"test,"}, - DATABASE_NAME); - - tableResultSetEqualTest( - "SELECT LEAST(string1, string2) FROM any_null_table", - new String[] {"_col0"}, - new String[] {"test,"}, - DATABASE_NAME); - - tableResultSetEqualTest( - "SELECT GREATEST(int1, int2) FROM any_null_table", - new String[] {"_col0"}, - new String[] {"10,"}, - DATABASE_NAME); - - tableResultSetEqualTest( - "SELECT LEAST(int1, int2) FROM any_null_table", - new String[] {"_col0"}, - new String[] {"10,"}, - DATABASE_NAME); - - tableResultSetEqualTest( - "SELECT GREATEST(double1, double2) FROM any_null_table", - new String[] {"_col0"}, - new String[] {"10.0,"}, - DATABASE_NAME); - - tableResultSetEqualTest( - "SELECT LEAST(double1, double2) FROM any_null_table", - new String[] {"_col0"}, - new String[] {"10.0,"}, - DATABASE_NAME); - - tableResultSetEqualTest( - "SELECT GREATEST(timestamp1, timestamp2) FROM any_null_table", - new String[] {"_col0"}, - new String[] {"1970-01-01T00:00:00.010Z,"}, - DATABASE_NAME); - - tableResultSetEqualTest( - "SELECT LEAST(timestamp1, timestamp2) FROM any_null_table", - new String[] {"_col0"}, - new String[] {"1970-01-01T00:00:00.010Z,"}, - DATABASE_NAME); - } - - @Test - public void testAnomalies() { - // do not support different type - for (int i = 1; i <= 7; i++) { - for (int j = i + 1; j <= 7; j++) { - tableAssertTestFail( - String.format("SELECT LEAST(s%d, s%d) FROM mix_type_table", i, j), - "701: Scalar function least must have at least two arguments, and all type must be the same.", - DATABASE_NAME); - - tableAssertTestFail( - String.format("SELECT GREATEST(s%d, s%d) FROM mix_type_table", i, j), - "701: Scalar function greatest must have at least two arguments, and all type must be the same.", - DATABASE_NAME); - } - } - } -} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBMaintainAuthIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBMaintainAuthIT.java deleted file mode 100644 index a67113c30e90..000000000000 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBMaintainAuthIT.java +++ /dev/null @@ -1,289 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.relational.it.query.recent; - -import org.apache.iotdb.it.env.EnvFactory; -import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.TableClusterIT; -import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; -import org.apache.iotdb.rpc.TSStatusCode; - -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; - -import static org.apache.iotdb.db.auth.AuthorityChecker.ONLY_ADMIN_ALLOWED; -import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; -import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; -import static org.apache.iotdb.db.it.utils.TestUtils.tableExecuteTest; -import static org.apache.iotdb.db.it.utils.TestUtils.tableQueryNoVerifyResultTest; - -@RunWith(IoTDBTestRunner.class) -@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) -public class IoTDBMaintainAuthIT { - private static final String DATABASE_NAME = "test"; - private static final String CREATE_USER_FORMAT = "create user %s '%s'"; - private static final String USER_1 = "user1"; - private static final String USER_2 = "user2"; - private static final String PASSWORD = "password"; - - private static final String[] createSqls = - new String[] { - "CREATE DATABASE " + DATABASE_NAME, - "USE " + DATABASE_NAME, - "CREATE TABLE table1(device_id STRING TAG, s1 INT32 FIELD)", - "INSERT INTO table1(time,device_id,s1) values(1, 'd1', 1)", - String.format(CREATE_USER_FORMAT, USER_1, PASSWORD), - "GRANT MAINTAIN TO USER " + USER_1, - "GRANT SELECT ON TABLE table1 TO USER " + USER_1, - "GRANT SELECT ON information_schema.queries TO USER " + USER_1, - String.format(CREATE_USER_FORMAT, USER_2, PASSWORD) - }; - - @BeforeClass - public static void setUp() throws Exception { - EnvFactory.getEnv().initClusterEnvironment(); - prepareTableData(createSqls); - } - - @AfterClass - public static void tearDown() throws Exception { - EnvFactory.getEnv().cleanClusterEnvironment(); - } - - @Test - public void maintainAuthTest() { - // case 1: explain - // user1 with select on table1 - String[] expectedHeader = new String[] {"distribution plan"}; - tableQueryNoVerifyResultTest( - "explain select * from test.table1", expectedHeader, USER_1, PASSWORD); - // user2 without select on table1 - tableAssertTestFail( - "explain select * from test.table1", - TSStatusCode.NO_PERMISSION.getStatusCode() - + ": Access Denied: No permissions for this operation, please add privilege SELECT ON test.table1", - USER_2, - PASSWORD); - - // case 2: explain analyze [verbose] - // user1 with select on table1 - expectedHeader = new String[] {"Explain Analyze"}; - tableQueryNoVerifyResultTest( - "explain analyze select * from test.table1", expectedHeader, USER_1, PASSWORD); - tableQueryNoVerifyResultTest( - "explain analyze verbose select * from test.table1", expectedHeader, USER_1, PASSWORD); - // user2 without select on table1 - tableAssertTestFail( - "explain analyze select * from test.table1", - TSStatusCode.NO_PERMISSION.getStatusCode() - + ": Access Denied: No permissions for this operation, please add privilege SELECT ON test.table1", - USER_2, - PASSWORD); - tableAssertTestFail( - "explain analyze verbose select * from test.table1", - TSStatusCode.NO_PERMISSION.getStatusCode() - + ": Access Denied: No permissions for this operation, please add privilege SELECT ON test.table1", - USER_2, - PASSWORD); - - // case 3: show current_sql_dialect - expectedHeader = new String[] {"CurrentSqlDialect"}; - tableQueryNoVerifyResultTest("SHOW CURRENT_SQL_DIALECT", expectedHeader, USER_2, PASSWORD); - - // case 4: show current_user - expectedHeader = new String[] {"CurrentUser"}; - tableQueryNoVerifyResultTest("SHOW CURRENT_USER", expectedHeader, USER_2, PASSWORD); - - // case 5: show version - expectedHeader = new String[] {"Version", "BuildInfo"}; - tableQueryNoVerifyResultTest("SHOW VERSION", expectedHeader, USER_2, PASSWORD); - - // case 6: show current_timestamp - expectedHeader = new String[] {"CurrentTimestamp"}; - tableQueryNoVerifyResultTest("SHOW CURRENT_TIMESTAMP", expectedHeader, USER_2, PASSWORD); - - // case 7: show variables - expectedHeader = new String[] {"Variable", "Value"}; - // user1 with MAINTAIN - tableQueryNoVerifyResultTest("SHOW VARIABLES", expectedHeader, USER_1, PASSWORD); - // user2 without MAINTAIN - tableAssertTestFail( - "SHOW VARIABLES", - TSStatusCode.NO_PERMISSION.getStatusCode() - + ": Access Denied: No permissions for this operation, please add privilege MAINTAIN", - USER_2, - PASSWORD); - - // case 8: show cluster_id - expectedHeader = new String[] {"ClusterId"}; - // user1 with MAINTAIN - tableQueryNoVerifyResultTest("SHOW CLUSTER_ID", expectedHeader, USER_1, PASSWORD); - // user2 without MAINTAIN - tableAssertTestFail( - "SHOW CLUSTER_ID", - TSStatusCode.NO_PERMISSION.getStatusCode() - + ": Access Denied: No permissions for this operation, please add privilege MAINTAIN", - USER_2, - PASSWORD); - - // case 9: flush - // user1 with MAINTAIN - tableExecuteTest("FLUSH", USER_1, PASSWORD); - // user2 without MAINTAIN - tableAssertTestFail( - "FLUSH", - TSStatusCode.NO_PERMISSION.getStatusCode() - + ": Access Denied: No permissions for this operation, please add privilege MAINTAIN", - USER_2, - PASSWORD); - - // case 10: clear cache - // user1 with MAINTAIN - tableExecuteTest("CLEAR CACHE", USER_1, PASSWORD); - // user2 without MAINTAIN - tableAssertTestFail( - "CLEAR CACHE", - TSStatusCode.NO_PERMISSION.getStatusCode() - + ": Access Denied: No permissions for this operation, please add privilege MAINTAIN", - USER_2, - PASSWORD); - - // case 11: set configuration - // user1 with MAINTAIN - tableExecuteTest("SET CONFIGURATION query_timeout_threshold='100000'", USER_1, PASSWORD); - // user2 without MAINTAIN - tableAssertTestFail( - "SET CONFIGURATION query_timeout_threshold='100000'", - TSStatusCode.NO_PERMISSION.getStatusCode() - + ": Access Denied: No permissions for this operation, please add privilege MAINTAIN", - USER_2, - PASSWORD); - - // case 12: show queries - // user1 with select on information_schema.queries - expectedHeader = - new String[] {"query_id", "start_time", "datanode_id", "elapsed_time", "statement", "user"}; - tableQueryNoVerifyResultTest("SHOW QUERIES", expectedHeader, USER_1, PASSWORD); - // user2 without select on information_schema.queries - tableAssertTestFail( - "SHOW QUERIES", - TSStatusCode.NO_PERMISSION.getStatusCode() - + ": Access Denied: No permissions for this operation, please add privilege SELECT ON information_schema.queries", - USER_2, - PASSWORD); - - // case 13: kill query - // user1 with MAINTAIN - tableAssertTestFail( - "kill query '20250206_093300_00001_1'", - TSStatusCode.NO_SUCH_QUERY.getStatusCode() + ": No such query", - USER_1, - PASSWORD); - // user2 without MAINTAIN - tableAssertTestFail( - "kill query '20250206_093300_00001_1'", - TSStatusCode.NO_PERMISSION.getStatusCode() - + ": Access Denied: No permissions for this operation, please add privilege MAINTAIN", - USER_2, - PASSWORD); - - // case 14: load configuration - // user1 with MAINTAIN - tableExecuteTest("LOAD CONFIGURATION", USER_1, PASSWORD); - // user2 without MAINTAIN - tableAssertTestFail( - "LOAD CONFIGURATION", - TSStatusCode.NO_PERMISSION.getStatusCode() - + ": Access Denied: No permissions for this operation, please add privilege MAINTAIN", - USER_2, - PASSWORD); - - // case 15: set system status - // user1 with MAINTAIN - tableExecuteTest("SET SYSTEM TO RUNNING", USER_1, PASSWORD); - // user2 without MAINTAIN - tableAssertTestFail( - "SET SYSTEM TO RUNNING", - TSStatusCode.NO_PERMISSION.getStatusCode() - + ": Access Denied: No permissions for this operation, please add privilege MAINTAIN", - USER_2, - PASSWORD); - - // case 16: start repair data - // user1 with MAINTAIN - tableExecuteTest("START REPAIR DATA", USER_1, PASSWORD); - // user2 without MAINTAIN - tableAssertTestFail( - "START REPAIR DATA", - TSStatusCode.NO_PERMISSION.getStatusCode() - + ": Access Denied: No permissions for this operation, please add privilege MAINTAIN", - USER_2, - PASSWORD); - - // case 17: stop repair data - // user1 with MAINTAIN - tableExecuteTest("STOP REPAIR DATA", USER_1, PASSWORD); - // user2 without MAINTAIN - tableAssertTestFail( - "STOP REPAIR DATA", - TSStatusCode.NO_PERMISSION.getStatusCode() - + ": Access Denied: No permissions for this operation, please add privilege MAINTAIN", - USER_2, - PASSWORD); - - // case 18: create function - // user1 with MAINTAIN - tableAssertTestFail( - "create function udsf as 'org.apache.iotdb.db.query.udf.example.relational.ContainNull'", - TSStatusCode.NO_PERMISSION.getStatusCode() + ": Access Denied: " + ONLY_ADMIN_ALLOWED, - USER_1, - PASSWORD); - // user2 without MAINTAIN - tableAssertTestFail( - "create function udsf as 'org.apache.iotdb.db.query.udf.example.relational.ContainNull'", - TSStatusCode.NO_PERMISSION.getStatusCode() + ": Access Denied: " + ONLY_ADMIN_ALLOWED, - USER_2, - PASSWORD); - - // case 19: show functions - // user1 with MAINTAIN - expectedHeader = new String[] {"FunctionName", "FunctionType", "ClassName(UDF)", "State"}; - tableQueryNoVerifyResultTest("SHOW FUNCTIONS", expectedHeader, USER_1, PASSWORD); - // user2 without MAINTAIN - tableQueryNoVerifyResultTest("SHOW FUNCTIONS", expectedHeader, USER_2, PASSWORD); - - // case 20: create function - // user1 with MAINTAIN - tableAssertTestFail( - "drop function udsf", - TSStatusCode.NO_PERMISSION.getStatusCode() + ": Access Denied: " + ONLY_ADMIN_ALLOWED, - USER_1, - PASSWORD); - // user2 without MAINTAIN - tableAssertTestFail( - "drop function udsf", - TSStatusCode.NO_PERMISSION.getStatusCode() + ": Access Denied: " + ONLY_ADMIN_ALLOWED, - USER_2, - PASSWORD); - } -} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBNullIdQueryIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBNullIdQueryIT.java index eba15523aa23..6f067df19dff 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBNullIdQueryIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBNullIdQueryIT.java @@ -34,11 +34,9 @@ import java.sql.Connection; import java.sql.ResultSet; import java.sql.ResultSetMetaData; -import java.sql.SQLException; import java.sql.Statement; import java.sql.Types; -import static org.apache.iotdb.db.it.utils.TestUtils.createUser; import static org.apache.iotdb.db.it.utils.TestUtils.defaultFormatDataTime; import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; @@ -504,35 +502,4 @@ public void showStatementTest() { expectedHeader = new String[] {"CurrentTimestamp"}; tableResultSetFuzzyTest("show current_timestamp", expectedHeader, 1, DATABASE_NAME); } - - @Test - public void setSqlDialectTest() throws SQLException { - createUser("tempuser", "temppw"); - - try (Connection userCon = EnvFactory.getEnv().getConnection("tempuser", "temppw"); - Statement userStmt = userCon.createStatement()) { - assertCurrentSqlDialect(true, userStmt); - - // set Tree to Table - userStmt.execute("set sql_dialect=table"); - assertCurrentSqlDialect(false, userStmt); - - // set Table to Tree - userStmt.execute("set sql_dialect=tree"); - assertCurrentSqlDialect(true, userStmt); - } - } - - public static void assertCurrentSqlDialect(boolean expectedTree, Statement statement) - throws SQLException { - ResultSet resultSet = statement.executeQuery("show current_sql_dialect"); - ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); - assertEquals("CurrentSqlDialect", resultSetMetaData.getColumnName(1)); - int count = 0; - while (resultSet.next()) { - assertEquals(expectedTree ? "TREE" : "TABLE", resultSet.getString(1)); - count++; - } - assertEquals(1, count); - } } diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBNullValueIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBNullValueIT.java deleted file mode 100644 index 531d88101ba6..000000000000 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBNullValueIT.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.relational.it.query.recent; - -import org.apache.iotdb.it.env.EnvFactory; -import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.TableClusterIT; -import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; - -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; - -import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; -import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; - -@RunWith(IoTDBTestRunner.class) -@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) -public class IoTDBNullValueIT { - private static final String DATABASE_NAME = "test"; - - private static final String[] createSqls = - new String[] { - "CREATE DATABASE " + DATABASE_NAME, - "USE " + DATABASE_NAME, - "create table table1(id1 tag, s1 string)", - "insert into table1 values(0, 'd1', null), (1,'d1', 1)", - "flush", - "insert into table1 values(0, 'd1', 0)", - "flush" - }; - - @BeforeClass - public static void setUp() throws Exception { - EnvFactory.getEnv().getConfig().getCommonConfig().setEnableCrossSpaceCompaction(false); - EnvFactory.getEnv().initClusterEnvironment(); - prepareTableData(createSqls); - } - - @AfterClass - public static void tearDown() { - EnvFactory.getEnv().cleanClusterEnvironment(); - } - - @Test - public void nullTest() { - - // case 1: all without time filter using previous fill without timeDuration - String[] expectedHeader = new String[] {"time", "id1", "s1"}; - String[] retArray = - new String[] { - "1970-01-01T00:00:00.000Z,d1,0,", "1970-01-01T00:00:00.001Z,d1,1,", - }; - tableResultSetEqualTest("select * from table1", expectedHeader, retArray, DATABASE_NAME); - } -} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBQueryAuthIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBQueryAuthIT.java deleted file mode 100644 index 4557a66e7d14..000000000000 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBQueryAuthIT.java +++ /dev/null @@ -1,320 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.relational.it.query.recent; - -import org.apache.iotdb.it.env.EnvFactory; -import org.apache.iotdb.it.framework.IoTDBTestRunner; -import org.apache.iotdb.itbase.category.TableClusterIT; -import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; -import org.apache.iotdb.rpc.TSStatusCode; - -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; - -import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; -import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; -import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; - -@RunWith(IoTDBTestRunner.class) -@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) -public class IoTDBQueryAuthIT { - - private static final String DATABASE_NAME = "test"; - private static final String CREATE_USER_FORMAT = "create user %s '%s'"; - private static final String USER_1 = "user1"; - private static final String USER_2 = "user2"; - private static final String USER_3 = "user3"; - private static final String USER_4 = "user4"; - private static final String USER_5 = "user5"; - private static final String USER_6 = "user6"; - private static final String USER_7 = "user7"; - private static final String USER_8 = "user8"; - private static final String USER_9 = "user9"; - private static final String PASSWORD = "password"; - - private static final String[] createSqls = - new String[] { - "CREATE DATABASE " + DATABASE_NAME, - "USE " + DATABASE_NAME, - "CREATE TABLE table1(device_id STRING TAG, s1 INT32 FIELD)", - "INSERT INTO table1(time,device_id,s1) values(1, 'd1', 1)", - "CREATE TABLE table2(device_id STRING TAG, s1 INT32 FIELD)", - "INSERT INTO table2(time,device_id,s1) values(1, 'd1', 1)", - String.format(CREATE_USER_FORMAT, USER_1, PASSWORD), - "GRANT SELECT ON ANY TO USER " + USER_1, - String.format(CREATE_USER_FORMAT, USER_2, PASSWORD), - "GRANT SELECT ON DATABASE " + DATABASE_NAME + " TO USER " + USER_2, - String.format(CREATE_USER_FORMAT, USER_3, PASSWORD), - "GRANT SELECT ON TABLE table1 TO USER " + USER_3, - String.format(CREATE_USER_FORMAT, USER_4, PASSWORD), - "GRANT DROP ON TABLE table2 TO USER " + USER_4, - String.format(CREATE_USER_FORMAT, USER_5, PASSWORD), - "GRANT INSERT ON TABLE table2 TO USER " + USER_5, - String.format(CREATE_USER_FORMAT, USER_6, PASSWORD), - "GRANT CREATE ON TABLE table2 TO USER " + USER_6, - String.format(CREATE_USER_FORMAT, USER_7, PASSWORD), - "GRANT ALTER ON TABLE table2 TO USER " + USER_7, - String.format(CREATE_USER_FORMAT, USER_8, PASSWORD), - "GRANT DELETE ON TABLE table2 TO USER " + USER_8, - String.format(CREATE_USER_FORMAT, USER_9, PASSWORD), - }; - - @BeforeClass - public static void setUp() throws Exception { - EnvFactory.getEnv().initClusterEnvironment(); - prepareTableData(createSqls); - } - - @AfterClass - public static void tearDown() throws Exception { - EnvFactory.getEnv().cleanClusterEnvironment(); - } - - @Test - public void queryAuthTest() { - // case 1: user1 with SELECT ON ANY - String[] expectedHeader1 = new String[] {"time", "device_id", "s1"}; - String[] retArray1 = - new String[] { - "1970-01-01T00:00:00.001Z,d1,1,", - }; - String[] expectedHeader2 = new String[] {"device_id"}; - String[] retArray2 = new String[] {"d1,"}; - String[] expectedHeader3 = new String[] {"count(devices)"}; - String[] retArray3 = new String[] {"1,"}; - - tableResultSetEqualTest( - "select * from table1", expectedHeader1, retArray1, USER_1, PASSWORD, DATABASE_NAME); - tableResultSetEqualTest( - "show devices from table1", expectedHeader2, retArray2, USER_1, PASSWORD, DATABASE_NAME); - tableResultSetEqualTest( - "count devices from table1", expectedHeader3, retArray3, USER_1, PASSWORD, DATABASE_NAME); - - // case 2: user2 with SELECT ON database - tableResultSetEqualTest( - "select * from table1", expectedHeader1, retArray1, USER_2, PASSWORD, DATABASE_NAME); - tableResultSetEqualTest( - "show devices from table1", expectedHeader2, retArray2, USER_2, PASSWORD, DATABASE_NAME); - tableResultSetEqualTest( - "count devices from table1", expectedHeader3, retArray3, USER_2, PASSWORD, DATABASE_NAME); - - // case 3: user3 with SELECT ON table1 - tableResultSetEqualTest( - "select * from table1", expectedHeader1, retArray1, USER_3, PASSWORD, DATABASE_NAME); - tableResultSetEqualTest( - "show devices from table1", expectedHeader2, retArray2, USER_3, PASSWORD, DATABASE_NAME); - tableResultSetEqualTest( - "count devices from table1", expectedHeader3, retArray3, USER_3, PASSWORD, DATABASE_NAME); - - // case 4: user3 with SELECT ON table1, without SELECT ON table2 - tableAssertTestFail( - "select * from table2", - TSStatusCode.NO_PERMISSION.getStatusCode() - + ": Access Denied: No permissions for this operation, please add privilege SELECT ON test.table2", - USER_3, - PASSWORD, - DATABASE_NAME); - tableAssertTestFail( - "show devices from table2", - TSStatusCode.NO_PERMISSION.getStatusCode() - + ": Access Denied: No permissions for this operation, please add privilege SELECT ON test.table2", - USER_3, - PASSWORD, - DATABASE_NAME); - tableAssertTestFail( - "count devices from table2", - TSStatusCode.NO_PERMISSION.getStatusCode() - + ": Access Denied: No permissions for this operation, please add privilege SELECT ON test.table2", - USER_3, - PASSWORD, - DATABASE_NAME); - - // case 5: user4 with only DROP ON table2 - tableAssertTestFail( - "select * from table2", - TSStatusCode.NO_PERMISSION.getStatusCode() - + ": Access Denied: No permissions for this operation, please add privilege SELECT ON test.table2", - USER_4, - PASSWORD, - DATABASE_NAME); - tableAssertTestFail( - "show devices from table2", - TSStatusCode.NO_PERMISSION.getStatusCode() - + ": Access Denied: No permissions for this operation, please add privilege SELECT ON test.table2", - USER_4, - PASSWORD, - DATABASE_NAME); - tableAssertTestFail( - "count devices from table2", - TSStatusCode.NO_PERMISSION.getStatusCode() - + ": Access Denied: No permissions for this operation, please add privilege SELECT ON test.table2", - USER_4, - PASSWORD, - DATABASE_NAME); - - // case 6: user5 with only INSERT ON table2 - tableAssertTestFail( - "select * from table2", - TSStatusCode.NO_PERMISSION.getStatusCode() - + ": Access Denied: No permissions for this operation, please add privilege SELECT ON test.table2", - USER_5, - PASSWORD, - DATABASE_NAME); - tableAssertTestFail( - "show devices from table2", - TSStatusCode.NO_PERMISSION.getStatusCode() - + ": Access Denied: No permissions for this operation, please add privilege SELECT ON test.table2", - USER_5, - PASSWORD, - DATABASE_NAME); - tableAssertTestFail( - "count devices from table2", - TSStatusCode.NO_PERMISSION.getStatusCode() - + ": Access Denied: No permissions for this operation, please add privilege SELECT ON test.table2", - USER_5, - PASSWORD, - DATABASE_NAME); - - // case 7: user6 with only CREATE ON table2 - tableAssertTestFail( - "select * from table2", - TSStatusCode.NO_PERMISSION.getStatusCode() - + ": Access Denied: No permissions for this operation, please add privilege SELECT ON test.table2", - USER_6, - PASSWORD, - DATABASE_NAME); - tableAssertTestFail( - "show devices from table2", - TSStatusCode.NO_PERMISSION.getStatusCode() - + ": Access Denied: No permissions for this operation, please add privilege SELECT ON test.table2", - USER_6, - PASSWORD, - DATABASE_NAME); - tableAssertTestFail( - "count devices from table2", - TSStatusCode.NO_PERMISSION.getStatusCode() - + ": Access Denied: No permissions for this operation, please add privilege SELECT ON test.table2", - USER_6, - PASSWORD, - DATABASE_NAME); - - // case 8: user7 with only ALTER ON table2 - tableAssertTestFail( - "select * from table2", - TSStatusCode.NO_PERMISSION.getStatusCode() - + ": Access Denied: No permissions for this operation, please add privilege SELECT ON test.table2", - USER_7, - PASSWORD, - DATABASE_NAME); - tableAssertTestFail( - "show devices from table2", - TSStatusCode.NO_PERMISSION.getStatusCode() - + ": Access Denied: No permissions for this operation, please add privilege SELECT ON test.table2", - USER_7, - PASSWORD, - DATABASE_NAME); - tableAssertTestFail( - "count devices from table2", - TSStatusCode.NO_PERMISSION.getStatusCode() - + ": Access Denied: No permissions for this operation, please add privilege SELECT ON test.table2", - USER_7, - PASSWORD, - DATABASE_NAME); - - // case 9: user8 with only DELETE ON table2 - tableAssertTestFail( - "select * from table2", - TSStatusCode.NO_PERMISSION.getStatusCode() - + ": Access Denied: No permissions for this operation, please add privilege SELECT ON test.table2", - USER_8, - PASSWORD, - DATABASE_NAME); - tableAssertTestFail( - "show devices from table2", - TSStatusCode.NO_PERMISSION.getStatusCode() - + ": Access Denied: No permissions for this operation, please add privilege SELECT ON test.table2", - USER_8, - PASSWORD, - DATABASE_NAME); - tableAssertTestFail( - "count devices from table2", - TSStatusCode.NO_PERMISSION.getStatusCode() - + ": Access Denied: No permissions for this operation, please add privilege SELECT ON test.table2", - USER_8, - PASSWORD, - DATABASE_NAME); - - // case 10: user9 with nothing - tableAssertTestFail( - "select * from test.table2", - TSStatusCode.NO_PERMISSION.getStatusCode() - + ": Access Denied: No permissions for this operation, please add privilege SELECT ON test.table2", - USER_9, - PASSWORD); - tableAssertTestFail( - "show devices from test.table2", - TSStatusCode.NO_PERMISSION.getStatusCode() - + ": Access Denied: No permissions for this operation, please add privilege SELECT ON test.table2", - USER_9, - PASSWORD); - tableAssertTestFail( - "count devices from test.table2", - TSStatusCode.NO_PERMISSION.getStatusCode() - + ": Access Denied: No permissions for this operation, please add privilege SELECT ON test.table2", - USER_9, - PASSWORD); - - // case 11: user1 with SELECT ON ANY - String[] expectedHeader4 = new String[] {"time", "device_id", "table1_s1", "table2_s1"}; - String[] retArray4 = - new String[] { - "1970-01-01T00:00:00.001Z,d1,1,1,", - }; - - tableResultSetEqualTest( - "select table1.time as time, table1.device_id as device_id, table1.s1 as table1_s1, table2.s1 as table2_s1 from table1 inner join table2 on table1.time=table2.time and table1.device_id=table2.device_id", - expectedHeader4, - retArray4, - USER_1, - PASSWORD, - DATABASE_NAME); - - // case 12: user2 with SELECT ON database - tableResultSetEqualTest( - "select table1.time as time, table1.device_id as device_id, table1.s1 as table1_s1, table2.s1 as table2_s1 from table1 inner join table2 on table1.time=table2.time and table1.device_id=table2.device_id", - expectedHeader4, - retArray4, - USER_2, - PASSWORD, - DATABASE_NAME); - - // case 3: user3 with SELECT ON just table1 - tableAssertTestFail( - "select table1.time as time, table1.device_id as device_id, table1.s1 as table1_s1, table2.s1 as table2_s1 from table1 inner join table2 on table1.time=table2.time and table1.device_id=table2.device_id", - TSStatusCode.NO_PERMISSION.getStatusCode() - + ": Access Denied: No permissions for this operation, please add privilege SELECT ON test.table2", - USER_3, - PASSWORD, - DATABASE_NAME); - } -} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDatabaseIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDatabaseIT.java index 219088b74f06..160495e40fb7 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDatabaseIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDatabaseIT.java @@ -27,7 +27,6 @@ import org.apache.iotdb.itbase.env.BaseEnv; import org.junit.After; -import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -363,16 +362,7 @@ public void testInformationSchema() throws SQLException { statement.executeQuery("show tables"), "TableName,TTL(ms),", new HashSet<>( - Arrays.asList( - "databases,INF,", - "tables,INF,", - "columns,INF,", - "queries,INF,", - "regions,INF,", - "topics,INF,", - "pipe_plugins,INF,", - "pipes,INF,", - "subscriptions,INF,"))); + Arrays.asList("databases,INF,", "tables,INF,", "columns,INF,", "queries,INF,"))); TestUtils.assertResultSetEqual( statement.executeQuery("desc databases"), @@ -415,44 +405,7 @@ public void testInformationSchema() throws SQLException { "start_time,TIMESTAMP,ATTRIBUTE,", "datanode_id,INT32,ATTRIBUTE,", "elapsed_time,FLOAT,ATTRIBUTE,", - "statement,STRING,ATTRIBUTE,", - "user,STRING,ATTRIBUTE,"))); - TestUtils.assertResultSetEqual( - statement.executeQuery("desc pipes"), - "ColumnName,DataType,Category,", - new HashSet<>( - Arrays.asList( - "id,STRING,TAG,", - "creation_time,TIMESTAMP,ATTRIBUTE,", - "state,STRING,ATTRIBUTE,", - "pipe_source,STRING,ATTRIBUTE,", - "pipe_processor,STRING,ATTRIBUTE,", - "pipe_sink,STRING,ATTRIBUTE,", - "exception_message,STRING,ATTRIBUTE,", - "remaining_event_count,INT64,ATTRIBUTE,", - "estimated_remaining_seconds,DOUBLE,ATTRIBUTE,"))); - TestUtils.assertResultSetEqual( - statement.executeQuery("desc pipe_plugins"), - "ColumnName,DataType,Category,", - new HashSet<>( - Arrays.asList( - "plugin_name,STRING,TAG,", - "plugin_type,STRING,ATTRIBUTE,", - "class_name,STRING,ATTRIBUTE,", - "plugin_jar,STRING,ATTRIBUTE,"))); - TestUtils.assertResultSetEqual( - statement.executeQuery("desc topics"), - "ColumnName,DataType,Category,", - new HashSet<>( - Arrays.asList("topic_name,STRING,TAG,", "topic_configs,STRING,ATTRIBUTE,"))); - TestUtils.assertResultSetEqual( - statement.executeQuery("desc subscriptions"), - "ColumnName,DataType,Category,", - new HashSet<>( - Arrays.asList( - "topic_name,STRING,TAG,", - "consumer_group_name,STRING,TAG,", - "subscribed_consumers,STRING,ATTRIBUTE,"))); + "statement,STRING,ATTRIBUTE,"))); // Test table query statement.execute("create database test"); @@ -474,16 +427,11 @@ public void testInformationSchema() throws SQLException { "information_schema,tables,INF,USING,", "information_schema,columns,INF,USING,", "information_schema,queries,INF,USING,", - "information_schema,regions,INF,USING,", - "information_schema,topics,INF,USING,", - "information_schema,pipe_plugins,INF,USING,", - "information_schema,pipes,INF,USING,", - "information_schema,subscriptions,INF,USING,", "test,test,INF,USING,"))); TestUtils.assertResultSetEqual( statement.executeQuery("count devices from tables where status = 'USING'"), "count(devices),", - Collections.singleton("10,")); + Collections.singleton("5,")); TestUtils.assertResultSetEqual( statement.executeQuery( "select * from columns where table_name = 'queries' or database = 'test'"), @@ -495,36 +443,10 @@ public void testInformationSchema() throws SQLException { "information_schema,queries,datanode_id,INT32,ATTRIBUTE,USING,", "information_schema,queries,elapsed_time,FLOAT,ATTRIBUTE,USING,", "information_schema,queries,statement,STRING,ATTRIBUTE,USING,", - "information_schema,queries,user,STRING,ATTRIBUTE,USING,", "test,test,time,TIMESTAMP,TIME,USING,", "test,test,a,STRING,TAG,USING,", "test,test,b,STRING,ATTRIBUTE,USING,", "test,test,c,INT32,FIELD,USING,"))); - - statement.execute( - "create pipe a2b with source('double-living'='true') with sink ('sink'='write-back-sink')"); - TestUtils.assertResultSetEqual( - statement.executeQuery("select id, pipe_sink from pipes where creation_time > 0"), - "id,pipe_sink,", - Collections.singleton("a2b,{sink=write-back-sink},")); - TestUtils.assertResultSetEqual( - statement.executeQuery("select * from pipe_plugins"), - "plugin_name,plugin_type,class_name,plugin_jar,", - new HashSet<>( - Arrays.asList( - "IOTDB-THRIFT-SSL-SINK,Builtin,org.apache.iotdb.commons.pipe.agent.plugin.builtin.connector.iotdb.thrift.IoTDBThriftSslConnector,null,", - "IOTDB-AIR-GAP-SINK,Builtin,org.apache.iotdb.commons.pipe.agent.plugin.builtin.connector.iotdb.airgap.IoTDBAirGapConnector,null,", - "DO-NOTHING-SINK,Builtin,org.apache.iotdb.commons.pipe.agent.plugin.builtin.connector.donothing.DoNothingConnector,null,", - "DO-NOTHING-PROCESSOR,Builtin,org.apache.iotdb.commons.pipe.agent.plugin.builtin.processor.donothing.DoNothingProcessor,null,", - "IOTDB-THRIFT-SINK,Builtin,org.apache.iotdb.commons.pipe.agent.plugin.builtin.connector.iotdb.thrift.IoTDBThriftConnector,null,", - "IOTDB-SOURCE,Builtin,org.apache.iotdb.commons.pipe.agent.plugin.builtin.extractor.iotdb.IoTDBExtractor,null,"))); - - statement.execute("create topic tp with ('start-time'='2025-01-13T10:03:19.229+08:00')"); - TestUtils.assertResultSetEqual( - statement.executeQuery("select * from topics where topic_name = 'tp'"), - "topic_name,topic_configs,", - Collections.singleton( - "tp,{__system.sql-dialect=table, start-time=2025-01-13T10:03:19.229+08:00},")); } } @@ -586,65 +508,4 @@ public void testMixedDatabase() throws SQLException { TestUtils.assertResultSetSize(statement.executeQuery("show databases"), 1); } } - - @Test - public void testDBAuth() throws SQLException { - try (final Connection adminCon = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); - final Statement adminStmt = adminCon.createStatement()) { - adminStmt.execute("create user test 'password'"); - adminStmt.execute("create database db"); - } - - try (final Connection userCon = - EnvFactory.getEnv().getConnection("test", "password", BaseEnv.TABLE_SQL_DIALECT); - final Statement userStmt = userCon.createStatement()) { - TestUtils.assertResultSetEqual( - userStmt.executeQuery("show databases"), - "Database,TTL(ms),SchemaReplicationFactor,DataReplicationFactor,TimePartitionInterval,", - Collections.emptySet()); - } - - try (final Connection adminCon = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); - final Statement adminStmt = adminCon.createStatement()) { - adminStmt.execute("GRANT SELECT ON DATABASE DB to user test"); - } - - try (final Connection userCon = - EnvFactory.getEnv().getConnection("test", "password", BaseEnv.TABLE_SQL_DIALECT); - final Statement userStmt = userCon.createStatement()) { - try (final ResultSet resultSet = userStmt.executeQuery("SHOW DATABASES")) { - final ResultSetMetaData metaData = resultSet.getMetaData(); - assertEquals(showDBColumnHeaders.size(), metaData.getColumnCount()); - for (int i = 0; i < showDBColumnHeaders.size(); i++) { - assertEquals(showDBColumnHeaders.get(i).getColumnName(), metaData.getColumnName(i + 1)); - } - Assert.assertTrue(resultSet.next()); - assertEquals("db", resultSet.getString(1)); - Assert.assertFalse(resultSet.next()); - } - - Assert.assertThrows( - SQLException.class, - () -> { - userStmt.execute("alter database db set properties ttl=6600000"); - }); - - Assert.assertThrows( - SQLException.class, - () -> { - userStmt.execute("drop database db"); - }); - } - - try (final Connection adminCon = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); - final Statement adminStmt = adminCon.createStatement()) { - adminStmt.execute("GRANT DROP ON ANY to user test"); - } - - try (final Connection userCon = - EnvFactory.getEnv().getConnection("test", "password", BaseEnv.TABLE_SQL_DIALECT); - final Statement userStmt = userCon.createStatement()) { - userStmt.execute("drop database db"); - } - } } diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java index 901917743102..d27e298f8776 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java @@ -27,7 +27,6 @@ import org.apache.iotdb.itbase.env.BaseEnv; import org.junit.AfterClass; -import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -42,7 +41,6 @@ import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.describeTableColumnHeaders; import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.describeTableDetailsColumnHeaders; -import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.showDBColumnHeaders; import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.showTablesColumnHeaders; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -562,59 +560,4 @@ public void testManageTable() { fail(e.getMessage()); } } - - @Test - public void testTableAuth() throws SQLException { - try (final Connection adminCon = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); - final Statement adminStmt = adminCon.createStatement()) { - adminStmt.execute("create user test 'password'"); - adminStmt.execute("create database db"); - adminStmt.execute("use db"); - adminStmt.execute("create table test (a tag, b attribute, c int32)"); - } - - try (final Connection userCon = - EnvFactory.getEnv().getConnection("test", "password", BaseEnv.TABLE_SQL_DIALECT); - final Statement userStmt = userCon.createStatement()) { - Assert.assertThrows( - SQLException.class, - () -> { - userStmt.execute("select * from db.test"); - }); - } - - try (final Connection adminCon = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); - final Statement adminStmt = adminCon.createStatement()) { - adminStmt.execute("GRANT SELECT ON db.test to user test"); - } - - try (final Connection userCon = - EnvFactory.getEnv().getConnection("test", "password", BaseEnv.TABLE_SQL_DIALECT); - final Statement userStmt = userCon.createStatement()) { - try (final ResultSet resultSet = userStmt.executeQuery("SHOW DATABASES")) { - final ResultSetMetaData metaData = resultSet.getMetaData(); - assertEquals(showDBColumnHeaders.size(), metaData.getColumnCount()); - for (int i = 0; i < showDBColumnHeaders.size(); i++) { - assertEquals(showDBColumnHeaders.get(i).getColumnName(), metaData.getColumnName(i + 1)); - } - Assert.assertTrue(resultSet.next()); - assertEquals("db", resultSet.getString(1)); - Assert.assertFalse(resultSet.next()); - } - - userStmt.execute("select * from db.test"); - } - - try (final Connection adminCon = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); - final Statement adminStmt = adminCon.createStatement()) { - adminStmt.execute("GRANT DROP ON DATABASE DB to user test"); - } - - try (final Connection userCon = - EnvFactory.getEnv().getConnection("test", "password", BaseEnv.TABLE_SQL_DIALECT); - final Statement userStmt = userCon.createStatement()) { - userStmt.execute("use db"); - userStmt.execute("drop table test"); - } - } } diff --git a/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionInsertNullIT.java b/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionInsertNullIT.java index 4eb14e35c939..bbee36782fc4 100644 --- a/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionInsertNullIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionInsertNullIT.java @@ -35,7 +35,6 @@ import org.apache.tsfile.write.record.Tablet; import org.apache.tsfile.write.schema.MeasurementSchema; import org.junit.After; -import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -426,61 +425,4 @@ public void insertAlignedTabletNullTest() { fail(e.getMessage()); } } - - @Test - public void insertTabletNullMeasurementTest() { - try (ISession session = EnvFactory.getEnv().getSessionConnection()) { - String deviceId = "root.sg1.clsu.aligned_d1"; - Tablet tablet = - new Tablet( - deviceId, - Arrays.asList( - new MeasurementSchema("s1", TSDataType.BOOLEAN), - new MeasurementSchema(null, TSDataType.INT32)), - 1); - tablet.addTimestamp(0, 300); - tablet.addValue("s1", 0, true); - tablet.addValue(null, 0, 1); - session.insertAlignedTablet(tablet); - fail(); - } catch (Exception e) { - Assert.assertEquals("measurement should be non null value", e.getMessage()); - } - - try (ISession session = EnvFactory.getEnv().getSessionConnection()) { - String deviceId = "root.sg1.clsu.aligned_d1"; - Tablet tablet = - new Tablet( - deviceId, - Arrays.asList( - new MeasurementSchema("s1", TSDataType.BOOLEAN), - new MeasurementSchema(null, TSDataType.INT32)), - 1); - tablet.addTimestamp(0, 300); - tablet.addValue(0, 0, true); - tablet.addValue(0, 1, 1); - session.insertAlignedTablet(tablet); - fail(); - } catch (Exception e) { - Assert.assertEquals("measurement should be non null value", e.getMessage()); - } - - try (ISession session = EnvFactory.getEnv().getSessionConnection()) { - String deviceId = "root.sg1.clsu.aligned_d1"; - Tablet tablet = - new Tablet( - deviceId, - Arrays.asList( - new MeasurementSchema("s1", TSDataType.BOOLEAN), - new MeasurementSchema(null, TSDataType.INT32)), - 1); - tablet.addTimestamp(0, 300); - tablet.addValue("s1", 0, true); - // doesn't insert 2nd measurement - session.insertAlignedTablet(tablet); - fail(); - } catch (Exception e) { - Assert.assertEquals("measurement should be non null value", e.getMessage()); - } - } } diff --git a/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/IoTDBSubscriptionBasicIT.java b/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/IoTDBSubscriptionBasicIT.java index 93ed3d484065..be478548c311 100644 --- a/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/IoTDBSubscriptionBasicIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/subscription/it/local/IoTDBSubscriptionBasicIT.java @@ -19,12 +19,10 @@ package org.apache.iotdb.subscription.it.local; -import org.apache.iotdb.commons.auth.entity.PrivilegeType; import org.apache.iotdb.isession.ISession; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.LocalStandaloneIT; -import org.apache.iotdb.itbase.env.BaseEnv; import org.apache.iotdb.rpc.subscription.config.TopicConstant; import org.apache.iotdb.session.subscription.SubscriptionTreeSession; import org.apache.iotdb.session.subscription.consumer.AckStrategy; @@ -50,7 +48,6 @@ import java.io.IOException; import java.time.Duration; -import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Properties; @@ -60,12 +57,6 @@ import java.util.concurrent.locks.LockSupport; import java.util.stream.Collectors; -import static org.apache.iotdb.db.it.utils.TestUtils.assertTableNonQueryTestFail; -import static org.apache.iotdb.db.it.utils.TestUtils.assertTableTestFail; -import static org.apache.iotdb.db.it.utils.TestUtils.createUser; -import static org.apache.iotdb.db.it.utils.TestUtils.executeNonQueryWithRetry; -import static org.apache.iotdb.db.it.utils.TestUtils.executeQueriesWithRetry; -import static org.apache.iotdb.db.it.utils.TestUtils.grantUserSystemPrivileges; import static org.apache.iotdb.subscription.it.IoTDBSubscriptionITConstant.AWAIT; import static org.junit.Assert.fail; @@ -630,62 +621,4 @@ public void testMissingConsumerId() { fail(e.getMessage()); } } - - @Test - public void testTablePermission() { - createUser(EnvFactory.getEnv(), "test", "test123"); - - assertTableNonQueryTestFail( - EnvFactory.getEnv(), - "create topic topic1", - "803: Access Denied: No permissions for this operation, please add privilege MAINTAIN", - "test", - "test123", - null); - assertTableTestFail( - EnvFactory.getEnv(), - "show topics", - "803: Access Denied: No permissions for this operation, please add privilege MAINTAIN", - "test", - "test123", - null); - assertTableTestFail( - EnvFactory.getEnv(), - "show subscriptions", - "803: Access Denied: No permissions for this operation, please add privilege MAINTAIN", - "test", - "test123", - null); - assertTableNonQueryTestFail( - EnvFactory.getEnv(), - "drop topic topic1", - "803: Access Denied: No permissions for this operation, please add privilege MAINTAIN", - "test", - "test123", - null); - - grantUserSystemPrivileges(EnvFactory.getEnv(), "test", PrivilegeType.MAINTAIN); - - executeNonQueryWithRetry( - EnvFactory.getEnv(), - "create topic topic1", - "test", - "test123", - null, - BaseEnv.TABLE_SQL_DIALECT); - executeQueriesWithRetry( - EnvFactory.getEnv(), - Arrays.asList("show topics", "show subscriptions"), - "test", - "test123", - null, - BaseEnv.TABLE_SQL_DIALECT); - executeNonQueryWithRetry( - EnvFactory.getEnv(), - "drop topic topic1", - "test", - "test123", - null, - BaseEnv.TABLE_SQL_DIALECT); - } } diff --git a/iotdb-api/external-api/pom.xml b/iotdb-api/external-api/pom.xml index 3622d5482147..18e171c3aec2 100644 --- a/iotdb-api/external-api/pom.xml +++ b/iotdb-api/external-api/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-api - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT external-api IoTDB: API: External API diff --git a/iotdb-api/pipe-api/pom.xml b/iotdb-api/pipe-api/pom.xml index e587605019ba..b0368594e5e3 100644 --- a/iotdb-api/pipe-api/pom.xml +++ b/iotdb-api/pipe-api/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-api - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT pipe-api IoTDB: API: Pipe API diff --git a/iotdb-api/pom.xml b/iotdb-api/pom.xml index 82c7d055d5b0..01aba4586534 100644 --- a/iotdb-api/pom.xml +++ b/iotdb-api/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-parent - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT iotdb-api pom diff --git a/iotdb-api/trigger-api/pom.xml b/iotdb-api/trigger-api/pom.xml index 22419d80ea17..64560a259bb3 100644 --- a/iotdb-api/trigger-api/pom.xml +++ b/iotdb-api/trigger-api/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-api - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT trigger-api IoTDB: API: Trigger API diff --git a/iotdb-api/udf-api/pom.xml b/iotdb-api/udf-api/pom.xml index 72355686168a..8a66274237ff 100644 --- a/iotdb-api/udf-api/pom.xml +++ b/iotdb-api/udf-api/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-api - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT udf-api IoTDB: API: UDF API diff --git a/iotdb-client/cli/pom.xml b/iotdb-client/cli/pom.xml index 600684bc4ec7..a735626e9eaf 100644 --- a/iotdb-client/cli/pom.xml +++ b/iotdb-client/cli/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-client - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT iotdb-cli IoTDB: Client: CLI @@ -37,37 +37,37 @@ org.apache.iotdb iotdb-session - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.apache.iotdb iotdb-jdbc - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.apache.iotdb iotdb-antlr - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.apache.iotdb node-commons - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.apache.iotdb iotdb-server - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.apache.iotdb isession - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.apache.iotdb service-rpc - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.apache.tsfile @@ -82,17 +82,17 @@ org.apache.iotdb iotdb-thrift - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.apache.iotdb iotdb-thrift-commons - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.apache.iotdb pipe-api - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.slf4j diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/common/ImportTsFileOperation.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/common/ImportTsFileOperation.java deleted file mode 100644 index 3769d855d161..000000000000 --- a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/common/ImportTsFileOperation.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.tool.common; - -import org.apache.iotdb.cli.utils.IoTPrinter; - -public enum ImportTsFileOperation { - NONE, - MV, - HARDLINK, - CP, - DELETE, - ; - - public static boolean isValidOperation(String operation) { - return "none".equalsIgnoreCase(operation) - || "mv".equalsIgnoreCase(operation) - || "cp".equalsIgnoreCase(operation) - || "delete".equalsIgnoreCase(operation); - } - - public static ImportTsFileOperation getOperation(String operation, boolean isFileStoreEquals) { - switch (operation.toLowerCase()) { - case "none": - return ImportTsFileOperation.NONE; - case "mv": - return ImportTsFileOperation.MV; - case "cp": - if (isFileStoreEquals) { - return ImportTsFileOperation.HARDLINK; - } else { - return ImportTsFileOperation.CP; - } - case "delete": - return ImportTsFileOperation.DELETE; - default: - new IoTPrinter(System.out).println("Args error: os/of must be one of none, mv, cp, delete"); - System.exit(Constants.CODE_ERROR); - return null; - } - } -} diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/AbstractDataTool.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/AbstractDataTool.java index eeaf5a542b0d..388993900eb5 100644 --- a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/AbstractDataTool.java +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/AbstractDataTool.java @@ -31,7 +31,7 @@ import org.apache.iotdb.rpc.StatementExecutionException; import org.apache.iotdb.session.Session; import org.apache.iotdb.tool.common.Constants; -import org.apache.iotdb.tool.common.ImportTsFileOperation; +import org.apache.iotdb.tool.tsfile.ImportTsFile; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.HelpFormatter; @@ -110,9 +110,9 @@ public abstract class AbstractDataTool { protected static String successDir = "success/"; protected static String timestampPrecision = "ms"; protected static String failedFileDirectory = null; - protected static ImportTsFileOperation failOperation; + protected static ImportTsFile.Operation failOperation; protected static ZoneId zoneId = ZoneId.systemDefault(); - protected static ImportTsFileOperation successOperation; + protected static ImportTsFile.Operation successOperation; protected static String targetFile = Constants.DUMP_FILE_NAME_DEFAULT; protected static final LongAdder loadFileFailedNum = new LongAdder(); protected static final LongAdder loadFileSuccessfulNum = new LongAdder(); diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/ImportData.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/ImportData.java index 2985d4c2e49d..e0c4c4d72b1e 100644 --- a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/ImportData.java +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/data/ImportData.java @@ -27,8 +27,8 @@ import org.apache.iotdb.rpc.StatementExecutionException; import org.apache.iotdb.session.Session; import org.apache.iotdb.tool.common.Constants; -import org.apache.iotdb.tool.common.ImportTsFileOperation; import org.apache.iotdb.tool.common.OptionsUtil; +import org.apache.iotdb.tool.tsfile.ImportTsFile; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.CommandLineParser; @@ -307,27 +307,27 @@ private static void parseSpecialParams(CommandLine commandLine) throws ArgsError final String of = commandLine.getOptionValue(Constants.ON_FAIL_ARGS); final String onFail = StringUtils.isNotBlank(of) ? of.trim().toLowerCase() : null; if (Constants.TSFILE_SUFFIXS.equalsIgnoreCase(fileType) - && (!ImportTsFileOperation.isValidOperation(onSuccess) - || !ImportTsFileOperation.isValidOperation(onFail))) { + && (!ImportTsFile.Operation.isValidOperation(onSuccess) + || !ImportTsFile.Operation.isValidOperation(onFail))) { ioTPrinter.println("Args error: os/of must be one of none, mv, cp, delete"); System.exit(Constants.CODE_ERROR); - } - if (Constants.TSFILE_SUFFIXS.equalsIgnoreCase(fileType)) { boolean isSuccessDirEqualsSourceDir = false; - if (ImportTsFileOperation.MV.name().equalsIgnoreCase(onSuccess) - || ImportTsFileOperation.CP.name().equalsIgnoreCase(onSuccess)) { + if (ImportTsFile.Operation.MV.name().equalsIgnoreCase(onSuccess) + || ImportTsFile.Operation.CP.name().equalsIgnoreCase(onSuccess)) { File dir = createSuccessDir(commandLine); isSuccessDirEqualsSourceDir = isFileStoreEquals(targetPath, dir); } boolean isFailDirEqualsSourceDir = false; - if (ImportTsFileOperation.MV.name().equalsIgnoreCase(onFail) - || ImportTsFileOperation.CP.name().equalsIgnoreCase(onFail)) { + if (ImportTsFile.Operation.MV.name().equalsIgnoreCase(onFail) + || ImportTsFile.Operation.CP.name().equalsIgnoreCase(onFail)) { File dir = createFailDir(commandLine); isFailDirEqualsSourceDir = isFileStoreEquals(targetPath, dir); } - successOperation = ImportTsFileOperation.getOperation(onSuccess, isSuccessDirEqualsSourceDir); - failOperation = ImportTsFileOperation.getOperation(onFail, isFailDirEqualsSourceDir); + + successOperation = + ImportTsFile.Operation.getOperation(onSuccess, isSuccessDirEqualsSourceDir); + failOperation = ImportTsFile.Operation.getOperation(onFail, isFailDirEqualsSourceDir); } } diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/tsfile/ImportTsFile.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/tsfile/ImportTsFile.java index c017ab2b0e03..cdfbd5b4437a 100644 --- a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/tsfile/ImportTsFile.java +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/tsfile/ImportTsFile.java @@ -22,7 +22,6 @@ import org.apache.iotdb.cli.utils.IoTPrinter; import org.apache.iotdb.commons.utils.NodeUrlUtils; import org.apache.iotdb.session.pool.SessionPool; -import org.apache.iotdb.tool.common.ImportTsFileOperation; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.CommandLineParser; @@ -78,8 +77,8 @@ public class ImportTsFile extends AbstractTsFileTool { private static String successDir = "success/"; private static String failDir = "fail/"; - private static ImportTsFileOperation successOperation; - private static ImportTsFileOperation failOperation; + private static Operation successOperation; + private static Operation failOperation; private static int threadNum = 8; @@ -268,28 +267,27 @@ private static void parseSpecialParams(CommandLine commandLine) { final String onSuccess = commandLine.getOptionValue(ON_SUCCESS_ARGS).trim().toLowerCase(); final String onFail = commandLine.getOptionValue(ON_FAIL_ARGS).trim().toLowerCase(); - if (!ImportTsFileOperation.isValidOperation(onSuccess) - || !ImportTsFileOperation.isValidOperation(onFail)) { + if (!Operation.isValidOperation(onSuccess) || !Operation.isValidOperation(onFail)) { IOT_PRINTER.println("Args error: os/of must be one of none, mv, cp, delete"); System.exit(CODE_ERROR); } boolean isSuccessDirEqualsSourceDir = false; - if (ImportTsFileOperation.MV.name().equalsIgnoreCase(onSuccess) - || ImportTsFileOperation.CP.name().equalsIgnoreCase(onSuccess)) { + if (Operation.MV.name().equalsIgnoreCase(onSuccess) + || Operation.CP.name().equalsIgnoreCase(onSuccess)) { File dir = createSuccessDir(commandLine); isSuccessDirEqualsSourceDir = isFileStoreEquals(source, dir); } boolean isFailDirEqualsSourceDir = false; - if (ImportTsFileOperation.MV.name().equalsIgnoreCase(onFail) - || ImportTsFileOperation.CP.name().equalsIgnoreCase(onFail)) { + if (Operation.MV.name().equalsIgnoreCase(onFail) + || Operation.CP.name().equalsIgnoreCase(onFail)) { File dir = createFailDir(commandLine); isFailDirEqualsSourceDir = isFileStoreEquals(source, dir); } - successOperation = ImportTsFileOperation.getOperation(onSuccess, isSuccessDirEqualsSourceDir); - failOperation = ImportTsFileOperation.getOperation(onFail, isFailDirEqualsSourceDir); + successOperation = Operation.getOperation(onSuccess, isSuccessDirEqualsSourceDir); + failOperation = Operation.getOperation(onFail, isFailDirEqualsSourceDir); if (commandLine.getOptionValue(THREAD_NUM_ARGS) != null) { threadNum = Integer.parseInt(commandLine.getOptionValue(THREAD_NUM_ARGS)); @@ -432,4 +430,41 @@ public static void asyncImportTsFiles() { } }); } + + public enum Operation { + NONE, + MV, + HARDLINK, + CP, + DELETE, + ; + + public static boolean isValidOperation(String operation) { + return "none".equalsIgnoreCase(operation) + || "mv".equalsIgnoreCase(operation) + || "cp".equalsIgnoreCase(operation) + || "delete".equalsIgnoreCase(operation); + } + + public static Operation getOperation(String operation, boolean isFileStoreEquals) { + switch (operation.toLowerCase()) { + case "none": + return Operation.NONE; + case "mv": + return Operation.MV; + case "cp": + if (isFileStoreEquals) { + return Operation.HARDLINK; + } else { + return Operation.CP; + } + case "delete": + return Operation.DELETE; + default: + IOT_PRINTER.println("Args error: os/of must be one of none, mv, cp, delete"); + System.exit(CODE_ERROR); + return null; + } + } + } } diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/tsfile/ImportTsFileBase.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/tsfile/ImportTsFileBase.java index f1fd3bf2d30f..6d6f95dcb827 100644 --- a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/tsfile/ImportTsFileBase.java +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/tsfile/ImportTsFileBase.java @@ -20,7 +20,6 @@ package org.apache.iotdb.tool.tsfile; import org.apache.iotdb.cli.utils.IoTPrinter; -import org.apache.iotdb.tool.common.ImportTsFileOperation; import java.io.File; import java.nio.file.FileAlreadyExistsException; @@ -41,9 +40,9 @@ public abstract class ImportTsFileBase implements Runnable { private static final LongAdder processingLoadFailedFileSuccessfulNum = new LongAdder(); private static String timePrecision = "ms"; private static String successDir; - private static ImportTsFileOperation successOperation; + private static ImportTsFile.Operation successOperation; private static String failDir; - private static ImportTsFileOperation failOperation; + private static ImportTsFile.Operation failOperation; @Override public void run() { @@ -233,9 +232,9 @@ protected static void printResult(final long startTime) { public static void setSuccessAndFailDirAndOperation( final String successDir, - final ImportTsFileOperation successOperation, + final ImportTsFile.Operation successOperation, final String failDir, - final ImportTsFileOperation failOperation) { + final ImportTsFile.Operation failOperation) { ImportTsFileBase.successDir = successDir; ImportTsFileBase.successOperation = successOperation; ImportTsFileBase.failDir = failDir; diff --git a/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/ImportTsFileOperationTest.java b/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/ImportTsFileOperationTest.java deleted file mode 100644 index 63541f586362..000000000000 --- a/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/ImportTsFileOperationTest.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.tool; - -import org.apache.iotdb.tool.common.ImportTsFileOperation; - -import org.junit.Test; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -public class ImportTsFileOperationTest { - - @Test - public void testIsValidOperation() { - assertTrue(ImportTsFileOperation.isValidOperation("none")); - assertTrue(ImportTsFileOperation.isValidOperation("mv")); - assertTrue(ImportTsFileOperation.isValidOperation("cp")); - assertTrue(ImportTsFileOperation.isValidOperation("delete")); - assertFalse(ImportTsFileOperation.isValidOperation("invalid")); - } - - @Test - public void testGetOperation() { - assertEquals(ImportTsFileOperation.NONE, ImportTsFileOperation.getOperation("none", false)); - assertEquals(ImportTsFileOperation.MV, ImportTsFileOperation.getOperation("mv", false)); - assertEquals(ImportTsFileOperation.HARDLINK, ImportTsFileOperation.getOperation("cp", true)); - assertEquals(ImportTsFileOperation.CP, ImportTsFileOperation.getOperation("cp", false)); - assertEquals(ImportTsFileOperation.DELETE, ImportTsFileOperation.getOperation("delete", false)); - } -} diff --git a/iotdb-client/client-cpp/pom.xml b/iotdb-client/client-cpp/pom.xml index 7cfe4b0d8a07..2196fb5f2346 100644 --- a/iotdb-client/client-cpp/pom.xml +++ b/iotdb-client/client-cpp/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-client - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT client-cpp pom @@ -43,7 +43,7 @@ org.apache.iotdb iotdb-thrift-commons - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT provided diff --git a/iotdb-client/client-py/pom.xml b/iotdb-client/client-py/pom.xml index a7f3ed35a11d..f411afcbb92e 100644 --- a/iotdb-client/client-py/pom.xml +++ b/iotdb-client/client-py/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-client - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT iotdb-python-api IoTDB: Client: Python-API @@ -34,13 +34,13 @@ org.apache.iotdb iotdb-thrift-commons - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT provided org.apache.iotdb iotdb-thrift - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT provided diff --git a/iotdb-client/isession/pom.xml b/iotdb-client/isession/pom.xml index e51dc149ab3b..ed8306daa381 100644 --- a/iotdb-client/isession/pom.xml +++ b/iotdb-client/isession/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-client - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT isession IoTDB: Client: isession @@ -32,7 +32,7 @@ org.apache.iotdb service-rpc - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.apache.tsfile @@ -47,12 +47,12 @@ org.apache.iotdb iotdb-thrift - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.apache.iotdb iotdb-thrift-commons - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.apache.thrift diff --git a/iotdb-client/jdbc/pom.xml b/iotdb-client/jdbc/pom.xml index ebd2651c84c1..625c75ca1407 100644 --- a/iotdb-client/jdbc/pom.xml +++ b/iotdb-client/jdbc/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-client - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT iotdb-jdbc IoTDB: Client: Jdbc @@ -38,12 +38,12 @@ org.apache.iotdb service-rpc - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.apache.iotdb iotdb-thrift-commons - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.apache.tsfile @@ -58,7 +58,7 @@ org.apache.iotdb iotdb-thrift - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.apache.thrift diff --git a/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/Constant.java b/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/Constant.java index cf1f0a2fe7fe..66b1e57d092a 100644 --- a/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/Constant.java +++ b/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/Constant.java @@ -42,9 +42,6 @@ private Constant() {} public static final String STATISTICS_RESULT_LINES = "* Lines of result: %d"; public static final String STATISTICS_PRC_INFO = "* Num of RPC: %d, avg cost: %d ms"; - public static final String TREE = "tree"; - public static final String TABLE = "table"; - // version number public enum Version { V_0_12, diff --git a/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBConnection.java b/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBConnection.java index 07d30370999c..f0949429270a 100644 --- a/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBConnection.java +++ b/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBConnection.java @@ -617,10 +617,6 @@ protected void changeDefaultDatabase(String database) { params.setDb(database); } - protected void changeDefaultSqlDialect(String sqlDialect) { - params.setSqlDialect(sqlDialect); - } - public int getTimeFactor() { return timeFactor; } diff --git a/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBConnectionParams.java b/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBConnectionParams.java index 1112caabd4e2..a31d2198d279 100644 --- a/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBConnectionParams.java +++ b/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBConnectionParams.java @@ -27,8 +27,6 @@ import java.time.ZoneId; import java.util.Optional; -import static org.apache.iotdb.jdbc.Constant.TREE; - public class IoTDBConnectionParams { private String host = Config.IOTDB_DEFAULT_HOST; @@ -52,7 +50,7 @@ public class IoTDBConnectionParams { private String trustStore; private String trustStorePwd; - private String sqlDialect = TREE; + private String sqlDialect = "tree"; private String db; diff --git a/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBStatement.java b/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBStatement.java index a3b6205512b7..f29b18031f11 100644 --- a/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBStatement.java +++ b/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBStatement.java @@ -46,9 +46,6 @@ import java.util.BitSet; import java.util.List; -import static org.apache.iotdb.jdbc.Constant.TABLE; -import static org.apache.iotdb.jdbc.Constant.TREE; - public class IoTDBStatement implements Statement { private final IoTDBConnection connection; @@ -329,10 +326,6 @@ private boolean executeSQL(String sql) throws TException, SQLException { connection.changeDefaultDatabase(execResp.getDatabase()); } - if (execResp.isSetTableModel()) { - connection.changeDefaultSqlDialect(execResp.tableModel ? TABLE : TREE); - } - if (execResp.isSetColumns()) { queryId = execResp.getQueryId(); if (execResp.queryResult == null) { diff --git a/iotdb-client/pom.xml b/iotdb-client/pom.xml index 167f8a5f07e5..d4ea94352634 100644 --- a/iotdb-client/pom.xml +++ b/iotdb-client/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-parent - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT iotdb-client pom diff --git a/iotdb-client/service-rpc/pom.xml b/iotdb-client/service-rpc/pom.xml index 0b9ccd0af091..9515493257e9 100644 --- a/iotdb-client/service-rpc/pom.xml +++ b/iotdb-client/service-rpc/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-client - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT service-rpc IoTDB: Client: Service-RPC @@ -60,12 +60,12 @@ org.apache.iotdb iotdb-thrift - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.apache.iotdb iotdb-thrift-commons - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.apache.thrift diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/RpcUtils.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/RpcUtils.java index 7541f500e45d..d41c1c8fd970 100644 --- a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/RpcUtils.java +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/RpcUtils.java @@ -374,20 +374,6 @@ public static boolean isUseDatabase(String sql) { return sql.length() > 4 && "use ".equalsIgnoreCase(sql.substring(0, 4)); } - public static boolean isSetSqlDialect(String sql) { - // check if startWith 'set ' - if (sql.length() <= 15 || !"set ".equalsIgnoreCase(sql.substring(0, 4))) { - return false; - } - - // check if the following content of sql is 'sql_dialect' - sql = sql.substring(4).trim(); - if (sql.length() <= 11) { - return false; - } - return sql.substring(0, 11).equalsIgnoreCase("sql_dialect"); - } - public static long getMilliSecond(long time, int timeFactor) { return time / timeFactor * 1_000; } diff --git a/iotdb-client/service-rpc/src/test/java/org/apache/iotdb/rpc/RpcUtilsTest.java b/iotdb-client/service-rpc/src/test/java/org/apache/iotdb/rpc/RpcUtilsTest.java index a69802760745..12969e96904f 100644 --- a/iotdb-client/service-rpc/src/test/java/org/apache/iotdb/rpc/RpcUtilsTest.java +++ b/iotdb-client/service-rpc/src/test/java/org/apache/iotdb/rpc/RpcUtilsTest.java @@ -64,14 +64,4 @@ public void parseLongToDateWithPrecision() { "1970-01-01T07:59:59.999+08:00", RpcUtils.parseLongToDateWithPrecision(formatter, -1, zoneId, "ms")); } - - @Test - public void testIsSetSqlDialect() { - Assert.assertTrue(RpcUtils.isSetSqlDialect("set sql_dialect=table")); - Assert.assertTrue(RpcUtils.isSetSqlDialect("set sql_dialect =table")); - Assert.assertTrue(RpcUtils.isSetSqlDialect("set sql_dialect =table")); - Assert.assertTrue(RpcUtils.isSetSqlDialect("set sql_dialect =table")); - Assert.assertFalse(RpcUtils.isSetSqlDialect("setsql_dialect =table")); - Assert.assertFalse(RpcUtils.isSetSqlDialect("set sql_dia")); - } } diff --git a/iotdb-client/session/pom.xml b/iotdb-client/session/pom.xml index 40ed5ab0a96d..a79d3c25bafe 100644 --- a/iotdb-client/session/pom.xml +++ b/iotdb-client/session/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-client - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT iotdb-session IoTDB: Client: Session @@ -37,17 +37,17 @@ org.apache.iotdb service-rpc - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.apache.iotdb isession - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.apache.iotdb iotdb-thrift-commons - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.apache.tsfile @@ -62,7 +62,7 @@ org.apache.iotdb iotdb-thrift - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.slf4j diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/Session.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/Session.java index 6908cad0632e..1e65be39082f 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/Session.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/Session.java @@ -192,7 +192,7 @@ public class Session implements ISession { protected long retryIntervalInMs = SessionConfig.RETRY_INTERVAL_IN_MS; - protected volatile String sqlDialect = SessionConfig.SQL_DIALECT; + protected String sqlDialect = SessionConfig.SQL_DIALECT; // may be null protected volatile String database; @@ -215,9 +215,6 @@ public class Session implements ISession { "All values are null and this submission is ignored,deviceIds are [{}],times are [{}],measurements are [{}]"; private static final String ALL_INSERT_DATA_IS_NULL = "All inserted data is null."; - protected static final String TABLE = "table"; - protected static final String TREE = "tree"; - public Session(String host, int rpcPort) { this( host, @@ -997,10 +994,8 @@ private SessionConnection getQuerySessionConnection() { public void executeNonQueryStatement(String sql) throws IoTDBConnectionException, StatementExecutionException { String previousDB = database; - String previousDialect = sqlDialect; defaultSessionConnection.executeNonQueryStatement(sql); - if ((!Objects.equals(previousDB, database) || !Objects.equals(previousDialect, sqlDialect)) - && endPointToSessionConnection != null) { + if (!Objects.equals(previousDB, database) && endPointToSessionConnection != null) { Iterator> iterator = endPointToSessionConnection.entrySet().iterator(); while (iterator.hasNext()) { @@ -1010,7 +1005,7 @@ public void executeNonQueryStatement(String sql) try { sessionConnection.executeNonQueryStatement(sql); } catch (Throwable t) { - logger.warn("failed to execute '{}' for {}", sql, entry.getKey()); + logger.warn("failed to change database for {}", entry.getKey()); iterator.remove(); } } @@ -2965,9 +2960,6 @@ private TSInsertTabletReq genTSInsertTabletReq(Tablet tablet, boolean sorted, bo TSInsertTabletReq request = new TSInsertTabletReq(); for (IMeasurementSchema measurementSchema : tablet.getSchemas()) { - if (measurementSchema.getMeasurementName() == null) { - throw new IllegalArgumentException("measurement should be non null value"); - } request.addToMeasurements(measurementSchema.getMeasurementName()); request.addToTypes(measurementSchema.getType().ordinal()); } @@ -3092,9 +3084,6 @@ private void updateTSInsertTabletsReq( List dataTypes = new ArrayList<>(); request.setIsAligned(isAligned); for (IMeasurementSchema measurementSchema : tablet.getSchemas()) { - if (measurementSchema.getMeasurementName() == null) { - throw new IllegalArgumentException("measurement should be non null value"); - } measurements.add(measurementSchema.getMeasurementName()); dataTypes.add(measurementSchema.getType().ordinal()); } @@ -4161,14 +4150,6 @@ public String getDatabase() { return database; } - protected void changeSqlDialect(String sqlDialect) { - this.sqlDialect = sqlDialect; - } - - public String getSqlDialect() { - return sqlDialect; - } - public static class Builder extends AbstractSessionBuilder { public Builder host(String host) { diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/SessionConnection.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/SessionConnection.java index 31abf69ec2ed..c0dc516ba114 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/SessionConnection.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/SessionConnection.java @@ -85,9 +85,6 @@ import java.util.concurrent.TimeUnit; import java.util.function.Supplier; -import static org.apache.iotdb.session.Session.TABLE; -import static org.apache.iotdb.session.Session.TREE; - @SuppressWarnings("java:S2142") public class SessionConnection { @@ -110,7 +107,7 @@ public class SessionConnection { private final long retryIntervalInMs; - private String sqlDialect; + private final String sqlDialect; private String database; @@ -474,13 +471,6 @@ private TSStatus executeNonQueryStatementInternal(TSExecuteStatementReq request) session.changeDatabase(dbName); this.database = dbName; } - if (resp.isSetTableModel()) { - String sqlDialect = resp.tableModel ? TABLE : TREE; - if (!sqlDialect.equalsIgnoreCase(this.sqlDialect)) { - session.changeSqlDialect(sqlDialect); - this.sqlDialect = sqlDialect; - } - } return resp.status; } diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/TableSessionBuilder.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/TableSessionBuilder.java index fd59846c6e68..9929f4b9cdc7 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/TableSessionBuilder.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/TableSessionBuilder.java @@ -27,8 +27,6 @@ import java.util.Collections; import java.util.List; -import static org.apache.iotdb.session.Session.TABLE; - /** * A builder class for constructing instances of {@link ITableSession}. * @@ -269,7 +267,7 @@ public ITableSession build() throws IoTDBConnectionException { this.nodeUrls = Collections.singletonList(SessionConfig.DEFAULT_HOST + ":" + SessionConfig.DEFAULT_PORT); } - this.sqlDialect = TABLE; + this.sqlDialect = "table"; Session newSession = new Session(this); newSession.open(enableCompression, connectionTimeoutInMs); return new TableSession(newSession); diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/pool/SessionPool.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/pool/SessionPool.java index e7515d60e476..86b4700c90eb 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/pool/SessionPool.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/pool/SessionPool.java @@ -59,7 +59,6 @@ import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; -import static org.apache.iotdb.rpc.RpcUtils.isSetSqlDialect; import static org.apache.iotdb.rpc.RpcUtils.isUseDatabase; /** @@ -3081,8 +3080,8 @@ public SessionDataSetWrapper executeQueryStatement(String sql, long timeoutInMs) public void executeNonQueryStatement(String sql) throws StatementExecutionException, IoTDBConnectionException { - // 'use XXX' and 'set sql_dialect' is forbidden in SessionPool.executeNonQueryStatement - if (isUseDatabase(sql) || isSetSqlDialect(sql)) { + // use XXX is forbidden in SessionPool.executeNonQueryStatement + if (isUseDatabase(sql)) { throw new IllegalArgumentException( String.format("SessionPool doesn't support executing %s directly", sql)); } diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/pool/TableSessionWrapper.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/pool/TableSessionWrapper.java index 242ab1a63532..c595b0feb219 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/pool/TableSessionWrapper.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/pool/TableSessionWrapper.java @@ -121,21 +121,6 @@ public void close() throws IoTDBConnectionException { return; } } - - if (!Objects.equals(session.getSqlDialect(), sessionPool.sqlDialect)) { - try { - session.executeNonQueryStatement("set sql_dialect=" + sessionPool.sqlDialect); - } catch (StatementExecutionException e) { - LOGGER.warn( - "Failed to change back sql_dialect by executing: set sql_dialect={}", - sessionPool.sqlDialect, - e); - session.close(); - session = null; - return; - } - } - sessionPool.putBack(session); session = null; } diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionConsumer.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionConsumer.java index 7dfae59a935d..c783626fc20a 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionConsumer.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionConsumer.java @@ -391,9 +391,7 @@ AbstractSubscriptionProvider constructProviderAndHandshake(final TEndPoint endPo } catch (final Exception ignored) { } throw new SubscriptionConnectionException( - String.format( - "Failed to handshake with subscription provider %s because of %s", provider, e), - e); + String.format("Failed to handshake with subscription provider %s", provider), e); } // update consumer id and consumer group id if not exist diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionProviders.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionProviders.java index 142719df8b1e..a9d239d5886f 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionProviders.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionProviders.java @@ -85,8 +85,7 @@ void openProviders(final AbstractSubscriptionConsumer consumer) throws Subscript try { defaultProvider = consumer.constructProviderAndHandshake(endPoint); } catch (final Exception e) { - LOGGER.warn( - "{} failed to create connection with {} because of {}", consumer, endPoint, e, e); + LOGGER.warn("Failed to create connection with {}", endPoint, e); continue; // try next endpoint } defaultDataNodeId = defaultProvider.getDataNodeId(); @@ -96,9 +95,8 @@ void openProviders(final AbstractSubscriptionConsumer consumer) throws Subscript try { allEndPoints = defaultProvider.getSessionConnection().fetchAllEndPoints(); } catch (final Exception e) { - LOGGER.warn( - "{} failed to fetch all endpoints from {} because of {}", consumer, endPoint, e, e); - break; + LOGGER.warn("Failed to fetch all endpoints from {}, will retry later...", endPoint, e); + break; // retry later } for (final Map.Entry entry : allEndPoints.entrySet()) { @@ -111,12 +109,8 @@ void openProviders(final AbstractSubscriptionConsumer consumer) throws Subscript provider = consumer.constructProviderAndHandshake(entry.getValue()); } catch (final Exception e) { LOGGER.warn( - "{} failed to create connection with {} because of {}", - consumer, - entry.getValue(), - e, - e); - continue; + "Failed to create connection with {}, will retry later...", entry.getValue(), e); + continue; // retry later } addProvider(entry.getKey(), provider); } @@ -140,7 +134,7 @@ void closeProviders() { try { provider.close(); } catch (final Exception e) { - LOGGER.warn("Failed to close subscription provider {} because of {}", provider, e, e); + LOGGER.warn(e.getMessage()); } } subscriptionProviders.clear(); @@ -248,10 +242,8 @@ private void heartbeatInternal(final AbstractSubscriptionConsumer consumer) { provider.setAvailable(); } catch (final Exception e) { LOGGER.warn( - "{} failed to sending heartbeat to subscription provider {} because of {}, set subscription provider unavailable", - consumer, + "something unexpected happened when sending heartbeat to subscription provider {}, set subscription provider unavailable", provider, - e, e); provider.setUnavailable(); } @@ -278,7 +270,7 @@ private void syncInternal(final AbstractSubscriptionConsumer consumer) { try { openProviders(consumer); } catch (final Exception e) { - LOGGER.warn("Failed to open providers for consumer {} because of {}", consumer, e, e); + LOGGER.warn("something unexpected happened when syncing subscription endpoints...", e); return; } } @@ -287,8 +279,8 @@ private void syncInternal(final AbstractSubscriptionConsumer consumer) { try { allEndPoints = consumer.fetchAllEndPointsWithRedirection(); } catch (final Exception e) { - LOGGER.warn("Failed to fetch all endpoints for consumer {} because of {}", consumer, e, e); - return; + LOGGER.warn("Failed to fetch all endpoints, will retry later...", e); + return; // retry later } // add new providers or handshake existing providers @@ -302,8 +294,8 @@ private void syncInternal(final AbstractSubscriptionConsumer consumer) { newProvider = consumer.constructProviderAndHandshake(endPoint); } catch (final Exception e) { LOGGER.warn( - "{} failed to create connection with {} because of {}", consumer, endPoint, e, e); - continue; + "Failed to create connection with endpoint {}, will retry later...", endPoint, e); + continue; // retry later } addProvider(entry.getKey(), newProvider); } else { @@ -313,10 +305,8 @@ private void syncInternal(final AbstractSubscriptionConsumer consumer) { provider.setAvailable(); } catch (final Exception e) { LOGGER.warn( - "{} failed to sending heartbeat to subscription provider {} because of {}, set subscription provider unavailable", - consumer, + "something unexpected happened when sending heartbeat to subscription provider {}, set subscription provider unavailable", provider, - e, e); provider.setUnavailable(); } @@ -326,10 +316,8 @@ private void syncInternal(final AbstractSubscriptionConsumer consumer) { closeAndRemoveProvider(entry.getKey()); } catch (final Exception e) { LOGGER.warn( - "Exception occurred when {} closing and removing subscription provider {} because of {}", - consumer, - provider, - e, + "Exception occurred when closing and removing subscription provider with data node id {}", + entry.getKey(), e); } } @@ -344,10 +332,8 @@ private void syncInternal(final AbstractSubscriptionConsumer consumer) { closeAndRemoveProvider(dataNodeId); } catch (final Exception e) { LOGGER.warn( - "Exception occurred when {} closing and removing subscription provider {} because of {}", - consumer, - provider, - e, + "Exception occurred when closing and removing subscription provider with data node id {}", + dataNodeId, e); } } diff --git a/iotdb-collector/collector-core/pom.xml b/iotdb-collector/collector-core/pom.xml index d5d7a4dc47fc..ff75a8927148 100644 --- a/iotdb-collector/collector-core/pom.xml +++ b/iotdb-collector/collector-core/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-collector - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT collector-core IoTDB: Collector: Core @@ -32,30 +32,30 @@ org.apache.iotdb collector-openapi - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.apache.iotdb service-rpc - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.apache.iotdb pipe-api - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.apache.iotdb node-commons - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT io.netty - netty-common + netty-handler io.netty - netty-handler + netty-common net.minidev diff --git a/iotdb-collector/collector-openapi/pom.xml b/iotdb-collector/collector-openapi/pom.xml index 4cf09b5192c3..3ba2259f6636 100644 --- a/iotdb-collector/collector-openapi/pom.xml +++ b/iotdb-collector/collector-openapi/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-collector - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT collector-openapi IoTDB: Collector: OpenAPI diff --git a/iotdb-collector/pom.xml b/iotdb-collector/pom.xml index 47d4e5b4a52e..035436e1fbc1 100644 --- a/iotdb-collector/pom.xml +++ b/iotdb-collector/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-parent - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT iotdb-collector pom diff --git a/iotdb-core/ainode/pom.xml b/iotdb-core/ainode/pom.xml index ad6679c77010..34f3291b79fd 100644 --- a/iotdb-core/ainode/pom.xml +++ b/iotdb-core/ainode/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-core - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT iotdb-ainode IoTDB: Core: AINode @@ -33,25 +33,25 @@ org.apache.iotdb iotdb-thrift-commons - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT provided org.apache.iotdb iotdb-thrift - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT provided org.apache.iotdb iotdb-thrift-confignode - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT provided org.apache.iotdb iotdb-thrift-ainode - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT provided diff --git a/iotdb-core/ainode/pyproject.toml b/iotdb-core/ainode/pyproject.toml index efbbdb1edaf7..c7f773b05908 100644 --- a/iotdb-core/ainode/pyproject.toml +++ b/iotdb-core/ainode/pyproject.toml @@ -34,12 +34,12 @@ classifiers = [ "Topic :: Software Development :: Libraries :: Python Modules", ] include = [ - {path = "iotdb/thrift/*", format = "wheel"}, - {path = "iotdb/thrift/common/*", format = "wheel"}, - {path = "iotdb/thrift/confignode/*", format = "wheel"}, - {path = "iotdb/thrift/datanode/*", format = "wheel"}, - {path = "iotdb/thrift/ainode/*", format = "wheel"}, - {path = "iotdb/conf/*", format = "wheel"}, + "iotdb/thrift/*", + "iotdb/thrift/common/*", + "iotdb/thrift/confignode/*", + "iotdb/thrift/datanode/*", + "iotdb/thrift/ainode/*", + "iotdb/conf/*", ] packages = [ { include = "iotdb" } @@ -50,7 +50,7 @@ python = ">=3.8, <3.13" numpy = "^1.21.4" pandas = "^1.3.5" -torch = ">=2.2.0" +torch = "2.2.0" pylru = "^1.2.1" thrift = "^0.13.0" diff --git a/iotdb-core/ainode/resources/conf/ainode-env.sh b/iotdb-core/ainode/resources/conf/ainode-env.sh index 1ec434ad2d98..b2737ebe1505 100644 --- a/iotdb-core/ainode/resources/conf/ainode-env.sh +++ b/iotdb-core/ainode/resources/conf/ainode-env.sh @@ -110,7 +110,7 @@ fi echo "Installing AINode..." cd "$SCRIPT_DIR/../lib/" shopt -s nullglob -for i in *.whl; do +for i in *.whl *.tar.gz; do if [[ $i =~ "ainode" ]]; then echo Installing AINode body: $i if [ -z "$p_pypi_mirror" ]; then diff --git a/iotdb-core/antlr/pom.xml b/iotdb-core/antlr/pom.xml index 8b183bcb1111..1012420ff7ac 100644 --- a/iotdb-core/antlr/pom.xml +++ b/iotdb-core/antlr/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-core - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT iotdb-antlr IoTDB: Core: Antlr-Parser diff --git a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IdentifierParser.g4 b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IdentifierParser.g4 index c0f382d0b530..63ff0c7afa49 100644 --- a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IdentifierParser.g4 +++ b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IdentifierParser.g4 @@ -73,8 +73,6 @@ keyWords | CQ | CQS | CREATE - | CURRENT_SQL_DIALECT - | CURRENT_USER | DATA | DATA_REPLICATION_FACTOR | DATA_REGION_GROUP_NUM @@ -216,7 +214,6 @@ keyWords | SOFFSET | SOURCE | SPACE - | SQL_DIALECT | STORAGE | START | STARTTIME @@ -227,7 +224,6 @@ keyWords | SUBSCRIPTIONS | SUBSTRING | SYSTEM - | TABLE | TAGS | TAIL | TASK @@ -246,7 +242,6 @@ keyWords | TOPIC | TOPICS | TRACING - | TREE | TRIGGER | TRIGGERS | TRUE diff --git a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 index de0b05a95e30..5e7b05378256 100644 --- a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 +++ b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 @@ -88,7 +88,7 @@ utilityStatement | setSystemStatus | showVersion | showFlushInfo | showLockInfo | showQueryResource | showQueries | showCurrentTimestamp | killQuery | grantWatermarkEmbedding | revokeWatermarkEmbedding | loadConfiguration | loadTimeseries | loadFile - | removeFile | unloadFile | setSqlDialect | showCurrentSqlDialect | showCurrentUser + | removeFile | unloadFile ; /** @@ -1183,18 +1183,6 @@ unloadFile : UNLOAD srcFileName=STRING_LITERAL dstFileDir=STRING_LITERAL ; -setSqlDialect - : SET SQL_DIALECT OPERATOR_SEQ (TABLE | TREE) - ; - -showCurrentSqlDialect - : SHOW CURRENT_SQL_DIALECT - ; - -showCurrentUser - : SHOW CURRENT_USER - ; - // attribute clauses syncAttributeClauses : attributePair (COMMA? attributePair)* diff --git a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 index 8d4ba400adce..0a3453cfc541 100644 --- a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 +++ b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 @@ -205,14 +205,6 @@ CREATE : C R E A T E ; -CURRENT_SQL_DIALECT - : C U R R E N T '_' S Q L '_' D I A L E C T - ; - -CURRENT_USER - : C U R R E N T '_' U S E R - ; - DATA : D A T A ; @@ -770,10 +762,6 @@ SPACE : S P A C E ; -SQL_DIALECT - : S Q L '_' D I A L E C T - ; - STORAGE : S T O R A G E ; @@ -814,10 +802,6 @@ SYSTEM : S Y S T E M ; -TABLE - : T A B L E - ; - TAGS : T A G S ; @@ -890,10 +874,6 @@ TRACING : T R A C I N G ; -TREE - : T R E E - ; - TRIGGER : T R I G G E R ; diff --git a/iotdb-core/confignode/pom.xml b/iotdb-core/confignode/pom.xml index 2118228b5000..3b7998f091e8 100644 --- a/iotdb-core/confignode/pom.xml +++ b/iotdb-core/confignode/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-core - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT iotdb-confignode IoTDB: Core: ConfigNode @@ -42,62 +42,62 @@ org.apache.iotdb service-rpc - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.apache.iotdb iotdb-consensus - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.apache.iotdb iotdb-server - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.apache.iotdb pipe-api - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.apache.iotdb trigger-api - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.apache.iotdb metrics-interface - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.apache.iotdb iotdb-thrift - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.apache.iotdb iotdb-thrift-commons - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.apache.iotdb iotdb-thrift-confignode - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.apache.iotdb iotdb-thrift-ainode - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.apache.iotdb node-commons - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.apache.iotdb udf-api - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.apache.tsfile diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanVisitor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanVisitor.java index b5c1923eccab..d2a46baf5858 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanVisitor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanVisitor.java @@ -19,8 +19,7 @@ package org.apache.iotdb.confignode.consensus.request; -import org.apache.iotdb.confignode.consensus.request.write.auth.AuthorRelationalPlan; -import org.apache.iotdb.confignode.consensus.request.write.auth.AuthorTreePlan; +import org.apache.iotdb.confignode.consensus.request.write.auth.AuthorPlan; import org.apache.iotdb.confignode.consensus.request.write.database.DatabaseSchemaPlan; import org.apache.iotdb.confignode.consensus.request.write.database.DeleteDatabasePlan; import org.apache.iotdb.confignode.consensus.request.write.database.SetTTLPlan; @@ -66,83 +65,29 @@ public R process(final ConfigPhysicalPlan plan, final C context) { case PipeDeactivateTemplate: return visitPipeDeactivateTemplate((PipeDeactivateTemplatePlan) plan, context); case CreateRole: - return visitCreateRole((AuthorTreePlan) plan, context); + return visitCreateRole((AuthorPlan) plan, context); case DropRole: - return visitDropRole((AuthorTreePlan) plan, context); + return visitDropRole((AuthorPlan) plan, context); case GrantRole: - return visitGrantRole((AuthorTreePlan) plan, context); + return visitGrantRole((AuthorPlan) plan, context); case RevokeRole: - return visitRevokeRole((AuthorTreePlan) plan, context); + return visitRevokeRole((AuthorPlan) plan, context); case CreateUser: - return visitCreateUser((AuthorTreePlan) plan, context); + return visitCreateUser((AuthorPlan) plan, context); case CreateUserWithRawPassword: - return visitCreateRawUser((AuthorTreePlan) plan, context); + return visitCreateRawUser((AuthorPlan) plan, context); case UpdateUser: - return visitUpdateUser((AuthorTreePlan) plan, context); + return visitUpdateUser((AuthorPlan) plan, context); case DropUser: - return visitDropUser((AuthorTreePlan) plan, context); + return visitDropUser((AuthorPlan) plan, context); case GrantUser: - return visitGrantUser((AuthorTreePlan) plan, context); + return visitGrantUser((AuthorPlan) plan, context); case RevokeUser: - return visitRevokeUser((AuthorTreePlan) plan, context); + return visitRevokeUser((AuthorPlan) plan, context); case GrantRoleToUser: - return visitGrantRoleToUser((AuthorTreePlan) plan, context); + return visitGrantRoleToUser((AuthorPlan) plan, context); case RevokeRoleFromUser: - return visitRevokeRoleFromUser((AuthorTreePlan) plan, context); - case RCreateUser: - return visitRCreateUser((AuthorRelationalPlan) plan, context); - case RCreateRole: - return visitRCreateRole((AuthorRelationalPlan) plan, context); - case RUpdateUser: - return visitRUpdateUser((AuthorRelationalPlan) plan, context); - case RDropUser: - return visitRDropUserPlan((AuthorRelationalPlan) plan, context); - case RDropRole: - return visitRDropRolePlan((AuthorRelationalPlan) plan, context); - case RGrantUserRole: - return visitRGrantUserRole((AuthorRelationalPlan) plan, context); - case RRevokeUserRole: - return visitRRevokeUserRole((AuthorRelationalPlan) plan, context); - case RGrantUserAny: - return visitRGrantUserAny((AuthorRelationalPlan) plan, context); - case RGrantRoleAny: - return visitRGrantRoleAny((AuthorRelationalPlan) plan, context); - case RGrantUserAll: - return visitRGrantUserAll((AuthorRelationalPlan) plan, context); - case RGrantRoleAll: - return visitRGrantRoleAll((AuthorRelationalPlan) plan, context); - case RGrantUserDBPriv: - return visitRGrantUserDB((AuthorRelationalPlan) plan, context); - case RGrantUserTBPriv: - return visitRGrantUserTB((AuthorRelationalPlan) plan, context); - case RGrantRoleDBPriv: - return visitRGrantRoleDB((AuthorRelationalPlan) plan, context); - case RGrantRoleTBPriv: - return visitRGrantRoleTB((AuthorRelationalPlan) plan, context); - case RRevokeUserAny: - return visitRRevokeUserAny((AuthorRelationalPlan) plan, context); - case RRevokeRoleAny: - return visitRRevokeRoleAny((AuthorRelationalPlan) plan, context); - case RRevokeUserAll: - return visitRRevokeUserAll((AuthorRelationalPlan) plan, context); - case RRevokeRoleAll: - return visitRRevokeRoleAll((AuthorRelationalPlan) plan, context); - case RRevokeUserDBPriv: - return visitRRevokeUserDBPrivilege((AuthorRelationalPlan) plan, context); - case RRevokeUserTBPriv: - return visitRRevokeUserTBPrivilege((AuthorRelationalPlan) plan, context); - case RRevokeRoleDBPriv: - return visitRRevokeRoleDBPrivilege((AuthorRelationalPlan) plan, context); - case RRevokeRoleTBPriv: - return visitRRevokeRoleTBPrivilege((AuthorRelationalPlan) plan, context); - case RGrantUserSysPri: - return visitRGrantUserSysPrivilege((AuthorRelationalPlan) plan, context); - case RGrantRoleSysPri: - return visitRGrantRoleSysPrivilege((AuthorRelationalPlan) plan, context); - case RRevokeUserSysPri: - return visitRRevokeUserSysPrivilege((AuthorRelationalPlan) plan, context); - case RRevokeRoleSysPri: - return visitRRevokeRoleSysPrivilege((AuthorRelationalPlan) plan, context); + return visitRevokeRoleFromUser((AuthorPlan) plan, context); case SetTTL: return visitTTL((SetTTLPlan) plan, context); case PipeCreateTable: @@ -219,170 +164,54 @@ public R visitPipeDeactivateTemplate( return visitPlan(pipeDeactivateTemplatePlan, context); } - public R visitCreateUser(final AuthorTreePlan createUserPlan, final C context) { + public R visitCreateUser(final AuthorPlan createUserPlan, final C context) { return visitPlan(createUserPlan, context); } - public R visitCreateRawUser(final AuthorTreePlan createRawUserPlan, final C context) { + public R visitCreateRawUser(final AuthorPlan createRawUserPlan, final C context) { return visitPlan(createRawUserPlan, context); } - public R visitUpdateUser(final AuthorTreePlan updateUserPlan, final C context) { + public R visitUpdateUser(final AuthorPlan updateUserPlan, final C context) { return visitPlan(updateUserPlan, context); } - public R visitDropUser(final AuthorTreePlan dropUserPlan, final C context) { + public R visitDropUser(final AuthorPlan dropUserPlan, final C context) { return visitPlan(dropUserPlan, context); } - public R visitGrantUser(final AuthorTreePlan grantUserPlan, final C context) { + public R visitGrantUser(final AuthorPlan grantUserPlan, final C context) { return visitPlan(grantUserPlan, context); } - public R visitRevokeUser(final AuthorTreePlan revokeUserPlan, final C context) { + public R visitRevokeUser(final AuthorPlan revokeUserPlan, final C context) { return visitPlan(revokeUserPlan, context); } - public R visitCreateRole(final AuthorTreePlan createRolePlan, final C context) { + public R visitCreateRole(final AuthorPlan createRolePlan, final C context) { return visitPlan(createRolePlan, context); } - public R visitDropRole(final AuthorTreePlan dropRolePlan, final C context) { + public R visitDropRole(final AuthorPlan dropRolePlan, final C context) { return visitPlan(dropRolePlan, context); } - public R visitGrantRole(final AuthorTreePlan grantRolePlan, final C context) { + public R visitGrantRole(final AuthorPlan grantRolePlan, final C context) { return visitPlan(grantRolePlan, context); } - public R visitRevokeRole(final AuthorTreePlan revokeRolePlan, final C context) { + public R visitRevokeRole(final AuthorPlan revokeRolePlan, final C context) { return visitPlan(revokeRolePlan, context); } - public R visitGrantRoleToUser(final AuthorTreePlan grantRoleToUserPlan, final C context) { + public R visitGrantRoleToUser(final AuthorPlan grantRoleToUserPlan, final C context) { return visitPlan(grantRoleToUserPlan, context); } - public R visitRevokeRoleFromUser(final AuthorTreePlan revokeRoleFromUserPlan, final C context) { + public R visitRevokeRoleFromUser(final AuthorPlan revokeRoleFromUserPlan, final C context) { return visitPlan(revokeRoleFromUserPlan, context); } - public R visitRCreateUser(final AuthorRelationalPlan rCreateUserPlan, final C context) { - return visitPlan(rCreateUserPlan, context); - } - - public R visitRCreateRole(final AuthorRelationalPlan rCreateRolePlan, final C context) { - return visitPlan(rCreateRolePlan, context); - } - - public R visitRUpdateUser(final AuthorRelationalPlan rUpdateUserPlan, final C context) { - return visitPlan(rUpdateUserPlan, context); - } - - public R visitRDropUserPlan(final AuthorRelationalPlan rDropUserPlan, final C context) { - return visitPlan(rDropUserPlan, context); - } - - public R visitRDropRolePlan(final AuthorRelationalPlan rDropRolePlan, final C context) { - return visitPlan(rDropRolePlan, context); - } - - public R visitRGrantUserRole(final AuthorRelationalPlan rGrantUserRolePlan, final C context) { - return visitPlan(rGrantUserRolePlan, context); - } - - public R visitRRevokeUserRole(final AuthorRelationalPlan rRevokeUserRolePlan, final C context) { - return visitPlan(rRevokeUserRolePlan, context); - } - - public R visitRGrantUserAny(final AuthorRelationalPlan rGrantUserAnyPlan, final C context) { - return visitPlan(rGrantUserAnyPlan, context); - } - - public R visitRGrantRoleAny(final AuthorRelationalPlan rGrantRoleAnyPlan, final C context) { - return visitPlan(rGrantRoleAnyPlan, context); - } - - public R visitRGrantUserAll(final AuthorRelationalPlan rGrantUserAllPlan, final C context) { - return visitPlan(rGrantUserAllPlan, context); - } - - public R visitRGrantRoleAll(final AuthorRelationalPlan rGrantRoleAllPlan, final C context) { - return visitPlan(rGrantRoleAllPlan, context); - } - - public R visitRGrantUserDB(final AuthorRelationalPlan rGrantUserDBPlan, final C context) { - return visitPlan(rGrantUserDBPlan, context); - } - - public R visitRGrantUserTB(final AuthorRelationalPlan rGrantUserTBPlan, final C context) { - return visitPlan(rGrantUserTBPlan, context); - } - - public R visitRGrantRoleDB(final AuthorRelationalPlan rGrantRoleDBPlan, final C context) { - return visitPlan(rGrantRoleDBPlan, context); - } - - public R visitRGrantRoleTB(final AuthorRelationalPlan rGrantRoleTBPlan, final C context) { - return visitPlan(rGrantRoleTBPlan, context); - } - - public R visitRRevokeUserAny(final AuthorRelationalPlan rRevokeUserAnyPlan, final C context) { - return visitPlan(rRevokeUserAnyPlan, context); - } - - public R visitRRevokeRoleAny(final AuthorRelationalPlan rRevokeRoleAnyPlan, final C context) { - return visitPlan(rRevokeRoleAnyPlan, context); - } - - public R visitRRevokeUserAll(final AuthorRelationalPlan rRevokeUserAllPlan, final C context) { - return visitPlan(rRevokeUserAllPlan, context); - } - - public R visitRRevokeRoleAll(final AuthorRelationalPlan rRevokeRoleAllPlan, final C context) { - return visitPlan(rRevokeRoleAllPlan, context); - } - - public R visitRRevokeUserDBPrivilege( - final AuthorRelationalPlan rRevokeUserDBPrivilegePlan, final C context) { - return visitPlan(rRevokeUserDBPrivilegePlan, context); - } - - public R visitRRevokeUserTBPrivilege( - final AuthorRelationalPlan rRevokeUserTBPrivilegePlan, final C context) { - return visitPlan(rRevokeUserTBPrivilegePlan, context); - } - - public R visitRRevokeRoleDBPrivilege( - final AuthorRelationalPlan rRevokeRoleTBPrivilegePlan, final C context) { - return visitPlan(rRevokeRoleTBPrivilegePlan, context); - } - - public R visitRRevokeRoleTBPrivilege( - final AuthorRelationalPlan rRevokeRoleTBPrivilegePlan, final C context) { - return visitPlan(rRevokeRoleTBPrivilegePlan, context); - } - - public R visitRGrantUserSysPrivilege( - final AuthorRelationalPlan rGrantUserSysPrivilegePlan, final C context) { - return visitPlan(rGrantUserSysPrivilegePlan, context); - } - - public R visitRGrantRoleSysPrivilege( - final AuthorRelationalPlan rGrantRoleSysPrivilegePlan, final C context) { - return visitPlan(rGrantRoleSysPrivilegePlan, context); - } - - public R visitRRevokeUserSysPrivilege( - final AuthorRelationalPlan rRevokeUserSysPrivilegePlan, final C context) { - return visitPlan(rRevokeUserSysPrivilegePlan, context); - } - - public R visitRRevokeRoleSysPrivilege( - final AuthorRelationalPlan rRevokeRoleSysPrivilegePlan, final C context) { - return visitPlan(rRevokeRoleSysPrivilegePlan, context); - } - public R visitTTL(final SetTTLPlan setTTLPlan, final C context) { return visitPlan(setTTLPlan, context); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/PermissionManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/PermissionManager.java index bab7b08c25e4..4d31ae8961e8 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/PermissionManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/PermissionManager.java @@ -25,7 +25,6 @@ import org.apache.iotdb.commons.auth.entity.PrivilegeUnion; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; import org.apache.iotdb.confignode.consensus.request.write.auth.AuthorPlan; -import org.apache.iotdb.confignode.consensus.request.write.pipe.payload.PipeEnrichedPlan; import org.apache.iotdb.confignode.consensus.response.auth.PermissionInfoResp; import org.apache.iotdb.confignode.manager.consensus.ConsensusManager; import org.apache.iotdb.confignode.persistence.AuthorInfo; @@ -67,9 +66,7 @@ public TSStatus operatePermission(AuthorPlan authorPlan, boolean isGeneratedByPi if (authorPlan.getAuthorType() == ConfigPhysicalPlanType.CreateUser || authorPlan.getAuthorType() == ConfigPhysicalPlanType.CreateRole || authorPlan.getAuthorType() == ConfigPhysicalPlanType.CreateUserWithRawPassword) { - tsStatus = - getConsensusManager() - .write(isGeneratedByPipe ? new PipeEnrichedPlan(authorPlan) : authorPlan); + tsStatus = getConsensusManager().write(authorPlan); } else { List allDataNodes = configManager.getNodeManager().getRegisteredDataNodes(); @@ -79,7 +76,7 @@ public TSStatus operatePermission(AuthorPlan authorPlan, boolean isGeneratedByPi .operateAuthPlan(authorPlan, allDataNodes, isGeneratedByPipe); } return tsStatus; - } catch (final ConsensusException e) { + } catch (ConsensusException e) { LOGGER.warn("Failed in the write API executing the consensus layer due to: ", e); TSStatus res = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); res.setMessage(e.getMessage()); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java index 002fcf859ffc..2b2acd14019b 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java @@ -19,7 +19,6 @@ package org.apache.iotdb.confignode.manager; -import org.apache.iotdb.common.rpc.thrift.Model; import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId; import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType; import org.apache.iotdb.common.rpc.thrift.TDataNodeConfiguration; @@ -141,7 +140,6 @@ import org.apache.iotdb.consensus.ConsensusFactory; import org.apache.iotdb.db.exception.BatchProcessException; import org.apache.iotdb.db.schemaengine.template.Template; -import org.apache.iotdb.db.utils.constant.SqlConstant; import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.rpc.TSStatusCode; @@ -716,8 +714,7 @@ private TSStatus checkMigrateRegion( Arrays.asList( new Pair<>("Original DataNode", originalDataNode), new Pair<>("Destination DataNode", destDataNode), - new Pair<>("Coordinator for add peer", coordinatorForAddPeer)), - migrateRegionReq.getModel()); + new Pair<>("Coordinator for add peer", coordinatorForAddPeer))); if (configManager .getPartitionManager() .getAllReplicaSets(originalDataNode.getDataNodeId()) @@ -758,8 +755,7 @@ private TSStatus checkReconstructRegion( targetDataNode, Arrays.asList( new Pair<>("Target DataNode", targetDataNode), - new Pair<>("Coordinator", coordinator)), - req.getModel()); + new Pair<>("Coordinator", coordinator))); ConfigNodeConfig conf = ConfigNodeDescriptor.getInstance().getConf(); if (configManager @@ -800,8 +796,7 @@ private TSStatus checkExtendRegion( targetDataNode, Arrays.asList( new Pair<>("Target DataNode", targetDataNode), - new Pair<>("Coordinator", coordinator)), - req.getModel()); + new Pair<>("Coordinator", coordinator))); if (configManager .getPartitionManager() .getAllReplicaSets(targetDataNode.getDataNodeId()) @@ -833,9 +828,9 @@ private TSStatus checkRemoveRegion( targetDataNode, Arrays.asList( new Pair<>("Target DataNode", targetDataNode), - new Pair<>("Coordinator", coordinator)), - req.getModel()); + new Pair<>("Coordinator", coordinator))); + ConfigNodeConfig conf = ConfigNodeDescriptor.getInstance().getConf(); if (configManager .getPartitionManager() .getAllReplicaSetsMap(regionId.getType()) @@ -874,11 +869,10 @@ private TSStatus checkRemoveRegion( private String regionOperationCommonCheck( TConsensusGroupId regionId, TDataNodeLocation targetDataNode, - List> relatedDataNodes, - Model model) { + List> relatedDataNodes) { String failMessage; - ConfigNodeConfig conf = ConfigNodeDescriptor.getInstance().getConf(); + ConfigNodeConfig conf = ConfigNodeDescriptor.getInstance().getConf(); if (TConsensusGroupType.DataRegion == regionId.getType() && ConsensusFactory.SIMPLE_CONSENSUS.equals(conf.getDataRegionConsensusProtocolClass())) { failMessage = "SimpleConsensus not supports region operation."; @@ -905,8 +899,6 @@ private String regionOperationCommonCheck( } else if ((failMessage = checkRegionOperationWithRemoveDataNode(regionId, targetDataNode)) != null) { // need to do nothing more - } else if ((failMessage = checkRegionOperationModelCorrectness(regionId, model)) != null) { - // need to do nothing more } return failMessage; @@ -971,20 +963,6 @@ private String checkRegionOperationDuplication(TConsensusGroupId regionId) { return null; } - private String checkRegionOperationModelCorrectness(TConsensusGroupId regionId, Model model) { - String databaseName = configManager.getPartitionManager().getRegionDatabase(regionId); - boolean isTreeModelDatabase = databaseName.startsWith(SqlConstant.TREE_MODEL_DATABASE_PREFIX); - if (Model.TREE == model && isTreeModelDatabase - || Model.TABLE == model && !isTreeModelDatabase) { - return null; - } - return String.format( - "The region's database %s is belong to %s model, but the model you are operating is %s", - databaseName, - isTreeModelDatabase ? Model.TREE.toString() : Model.TABLE.toString(), - model.toString()); - } - // end region public TSStatus migrateRegion(TMigrateRegionReq migrateRegionReq) { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/event/PipeConfigRegionSnapshotEvent.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/event/PipeConfigRegionSnapshotEvent.java index eda703f6e512..b37bfde0baa0 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/event/PipeConfigRegionSnapshotEvent.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/event/PipeConfigRegionSnapshotEvent.java @@ -67,24 +67,14 @@ public class PipeConfigRegionSnapshotEvent extends PipeSnapshotEvent new HashSet<>( Arrays.asList( ConfigPhysicalPlanType.CreateRole.getPlanType(), - ConfigPhysicalPlanType.GrantRole.getPlanType(), - ConfigPhysicalPlanType.RGrantRoleDBPriv.getPlanType(), - ConfigPhysicalPlanType.RGrantRoleTBPriv.getPlanType(), - ConfigPhysicalPlanType.RGrantRoleAny.getPlanType(), - ConfigPhysicalPlanType.RGrantRoleSysPri.getPlanType(), - ConfigPhysicalPlanType.RGrantRoleAll.getPlanType())))); + ConfigPhysicalPlanType.GrantRole.getPlanType())))); SNAPSHOT_FILE_TYPE_2_CONFIG_PHYSICAL_PLAN_TYPE_MAP.put( CNSnapshotFileType.USER, Collections.unmodifiableSet( new HashSet<>( Arrays.asList( ConfigPhysicalPlanType.CreateUserWithRawPassword.getPlanType(), - ConfigPhysicalPlanType.GrantUser.getPlanType(), - ConfigPhysicalPlanType.RGrantUserDBPriv.getPlanType(), - ConfigPhysicalPlanType.RGrantUserTBPriv.getPlanType(), - ConfigPhysicalPlanType.RGrantUserAny.getPlanType(), - ConfigPhysicalPlanType.RGrantUserSysPri.getPlanType(), - ConfigPhysicalPlanType.RGrantUserAll.getPlanType())))); + ConfigPhysicalPlanType.GrantUser.getPlanType())))); SNAPSHOT_FILE_TYPE_2_CONFIG_PHYSICAL_PLAN_TYPE_MAP.put( CNSnapshotFileType.USER_ROLE, Collections.singleton(ConfigPhysicalPlanType.GrantRoleToUser.getPlanType())); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/extractor/ConfigRegionListeningFilter.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/extractor/ConfigRegionListeningFilter.java index 5f0e2889d9fc..0c87b8d60e74 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/extractor/ConfigRegionListeningFilter.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/extractor/ConfigRegionListeningFilter.java @@ -41,8 +41,8 @@ import static org.apache.iotdb.commons.pipe.datastructure.options.PipeInclusionOptions.getExclusionString; import static org.apache.iotdb.commons.pipe.datastructure.options.PipeInclusionOptions.getInclusionString; import static org.apache.iotdb.commons.pipe.datastructure.options.PipeInclusionOptions.parseOptions; -import static org.apache.iotdb.commons.pipe.datastructure.options.PipeInclusionOptions.tableOnlySyncPrefixes; -import static org.apache.iotdb.commons.pipe.datastructure.options.PipeInclusionOptions.treeOnlySyncPrefixes; +import static org.apache.iotdb.commons.pipe.datastructure.options.PipeInclusionOptions.tableOnlySyncPrefix; +import static org.apache.iotdb.commons.pipe.datastructure.options.PipeInclusionOptions.treeOnlySyncPrefix; /** * {@link ConfigRegionListeningFilter} is to classify the {@link ConfigPhysicalPlan}s to help {@link @@ -122,45 +122,16 @@ public class ConfigRegionListeningFilter { OPTION_PLAN_MAP.put( new PartialPath("auth.role.create"), - Collections.unmodifiableList( - Arrays.asList( - ConfigPhysicalPlanType.CreateRole, ConfigPhysicalPlanType.RCreateRole))); + Collections.singletonList(ConfigPhysicalPlanType.CreateRole)); OPTION_PLAN_MAP.put( new PartialPath("auth.role.drop"), - Collections.unmodifiableList( - Arrays.asList(ConfigPhysicalPlanType.DropRole, ConfigPhysicalPlanType.RDropRole))); - - // Both + Collections.singletonList(ConfigPhysicalPlanType.DropRole)); OPTION_PLAN_MAP.put( new PartialPath("auth.role.grant"), - Collections.unmodifiableList( - Arrays.asList( - ConfigPhysicalPlanType.GrantRole, - ConfigPhysicalPlanType.RGrantRoleAll, - ConfigPhysicalPlanType.RGrantRoleSysPri))); + Collections.singletonList(ConfigPhysicalPlanType.GrantRole)); OPTION_PLAN_MAP.put( new PartialPath("auth.role.revoke"), - Collections.unmodifiableList( - Arrays.asList( - ConfigPhysicalPlanType.RevokeRole, - ConfigPhysicalPlanType.RRevokeRoleAll, - ConfigPhysicalPlanType.RRevokeRoleSysPri))); - - // Table - OPTION_PLAN_MAP.put( - new PartialPath("auth.role.grant.table"), - Collections.unmodifiableList( - Arrays.asList( - ConfigPhysicalPlanType.RGrantRoleAny, - ConfigPhysicalPlanType.RGrantRoleDBPriv, - ConfigPhysicalPlanType.RGrantRoleTBPriv))); - OPTION_PLAN_MAP.put( - new PartialPath("auth.role.revoke.table"), - Collections.unmodifiableList( - Arrays.asList( - ConfigPhysicalPlanType.RRevokeRoleAny, - ConfigPhysicalPlanType.RRevokeRoleDBPriv, - ConfigPhysicalPlanType.RRevokeRoleTBPriv))); + Collections.singletonList(ConfigPhysicalPlanType.RevokeRole)); OPTION_PLAN_MAP.put( new PartialPath("auth.user.create"), @@ -170,49 +141,20 @@ public class ConfigRegionListeningFilter { ConfigPhysicalPlanType.CreateUserWithRawPassword))); OPTION_PLAN_MAP.put( new PartialPath("auth.user.alter"), - Collections.unmodifiableList( - Arrays.asList( - ConfigPhysicalPlanType.UpdateUser, ConfigPhysicalPlanType.RUpdateUser))); + Collections.singletonList(ConfigPhysicalPlanType.UpdateUser)); OPTION_PLAN_MAP.put( new PartialPath("auth.user.drop"), - Collections.unmodifiableList( - Arrays.asList(ConfigPhysicalPlanType.DropUser, ConfigPhysicalPlanType.RUpdateUser))); - - // Both + Collections.singletonList(ConfigPhysicalPlanType.DropUser)); OPTION_PLAN_MAP.put( new PartialPath("auth.user.grant"), Collections.unmodifiableList( Arrays.asList( - ConfigPhysicalPlanType.GrantUser, - ConfigPhysicalPlanType.GrantRoleToUser, - ConfigPhysicalPlanType.RGrantUserRole, - ConfigPhysicalPlanType.RGrantUserAll, - ConfigPhysicalPlanType.RGrantUserSysPri))); + ConfigPhysicalPlanType.GrantUser, ConfigPhysicalPlanType.GrantRoleToUser))); OPTION_PLAN_MAP.put( new PartialPath("auth.user.revoke"), Collections.unmodifiableList( Arrays.asList( - ConfigPhysicalPlanType.RevokeUser, - ConfigPhysicalPlanType.RevokeRoleFromUser, - ConfigPhysicalPlanType.RRevokeUserRole, - ConfigPhysicalPlanType.RGrantUserAll, - ConfigPhysicalPlanType.RGrantUserSysPri))); - - // Table - OPTION_PLAN_MAP.put( - new PartialPath("auth.user.grant.table"), - Collections.unmodifiableList( - Arrays.asList( - ConfigPhysicalPlanType.RGrantUserAny, - ConfigPhysicalPlanType.RGrantUserDBPriv, - ConfigPhysicalPlanType.RGrantUserTBPriv))); - OPTION_PLAN_MAP.put( - new PartialPath("auth.user.revoke.table"), - Collections.unmodifiableList( - Arrays.asList( - ConfigPhysicalPlanType.RRevokeUserAny, - ConfigPhysicalPlanType.RRevokeUserDBPriv, - ConfigPhysicalPlanType.RRevokeUserTBPriv))); + ConfigPhysicalPlanType.RevokeUser, ConfigPhysicalPlanType.RevokeRoleFromUser))); } catch (final IllegalPathException ignore) { // There won't be any exceptions here } @@ -243,11 +185,11 @@ public static Set parseListeningPlanTypeSet( exclusionOptions.forEach(exclusion -> planTypes.removeAll(getOptionsByPrefix(exclusion))); if (!TreePattern.isTreeModelDataAllowToBeCaptured(parameters)) { - treeOnlySyncPrefixes.forEach(prefix -> planTypes.removeAll(getOptionsByPrefix(prefix))); + planTypes.removeAll(getOptionsByPrefix(treeOnlySyncPrefix)); } if (!TablePattern.isTableModelDataAllowToBeCaptured(parameters)) { - tableOnlySyncPrefixes.forEach(prefix -> planTypes.removeAll(getOptionsByPrefix(prefix))); + planTypes.removeAll(getOptionsByPrefix(tableOnlySyncPrefix)); } return planTypes; @@ -255,7 +197,7 @@ public static Set parseListeningPlanTypeSet( private static Set getOptionsByPrefix(final PartialPath prefix) { return OPTION_PLAN_MAP.keySet().stream() - .filter(path -> path.matchPrefixPath(prefix)) + .filter(path -> path.overlapWithFullPathPrefix(prefix)) .map(OPTION_PLAN_MAP::get) .flatMap(Collection::stream) .collect(Collectors.toSet()); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/extractor/IoTDBConfigRegionExtractor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/extractor/IoTDBConfigRegionExtractor.java index 2795879dcce8..9387d67bebef 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/extractor/IoTDBConfigRegionExtractor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/extractor/IoTDBConfigRegionExtractor.java @@ -62,10 +62,6 @@ public class IoTDBConfigRegionExtractor extends IoTDBNonDataRegionExtractor { new PipeConfigPhysicalPlanTreePatternParseVisitor(); public static final PipeConfigPhysicalPlanTablePatternParseVisitor TABLE_PATTERN_PARSE_VISITOR = new PipeConfigPhysicalPlanTablePatternParseVisitor(); - public static final PipeConfigPhysicalPlanTreeScopeParseVisitor TREE_SCOPE_PARSE_VISITOR = - new PipeConfigPhysicalPlanTreeScopeParseVisitor(); - public static final PipeConfigPhysicalPlanTableScopeParseVisitor TABLE_SCOPE_PARSE_VISITOR = - new PipeConfigPhysicalPlanTableScopeParseVisitor(); private Set listenedTypeSet = new HashSet<>(); @@ -153,23 +149,8 @@ public static Optional parseConfigPlan( return result; } } - if (!Boolean.FALSE.equals(isTableDatabasePlan) && result.isPresent()) { - result = TABLE_PATTERN_PARSE_VISITOR.process(result.get(), tablePattern); - if (!result.isPresent()) { - return result; - } - } - if (!treePattern.isTreeModelDataAllowedToBeCaptured() && result.isPresent()) { - result = TREE_SCOPE_PARSE_VISITOR.process(result.get(), null); - if (!result.isPresent()) { - return result; - } - } - if (!tablePattern.isTableModelDataAllowedToBeCaptured() && result.isPresent()) { - result = TABLE_SCOPE_PARSE_VISITOR.process(result.get(), null); - if (!result.isPresent()) { - return result; - } + if (!Boolean.FALSE.equals(isTableDatabasePlan)) { + result = TABLE_PATTERN_PARSE_VISITOR.process(plan, tablePattern); } return result; } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/extractor/PipeConfigPhysicalPlanTablePatternParseVisitor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/extractor/PipeConfigPhysicalPlanTablePatternParseVisitor.java index 679614ac458c..91be75a3bb5b 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/extractor/PipeConfigPhysicalPlanTablePatternParseVisitor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/extractor/PipeConfigPhysicalPlanTablePatternParseVisitor.java @@ -23,7 +23,6 @@ import org.apache.iotdb.commons.utils.PathUtils; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanVisitor; -import org.apache.iotdb.confignode.consensus.request.write.auth.AuthorRelationalPlan; import org.apache.iotdb.confignode.consensus.request.write.database.DatabaseSchemaPlan; import org.apache.iotdb.confignode.consensus.request.write.database.DeleteDatabasePlan; import org.apache.iotdb.confignode.consensus.request.write.pipe.payload.PipeCreateTablePlan; @@ -133,65 +132,4 @@ private boolean matchDatabaseAndTableName( return pattern.matchesDatabase(PathUtils.unQualifyDatabaseName(database)) && pattern.matchesTable(tableName); } - - @Override - public Optional visitRGrantUserDB( - final AuthorRelationalPlan plan, final TablePattern pattern) { - return visitAuthorDBPlan(plan, pattern); - } - - @Override - public Optional visitRGrantRoleDB( - final AuthorRelationalPlan plan, final TablePattern pattern) { - return visitAuthorDBPlan(plan, pattern); - } - - @Override - public Optional visitRRevokeUserDBPrivilege( - final AuthorRelationalPlan plan, final TablePattern pattern) { - return visitAuthorDBPlan(plan, pattern); - } - - @Override - public Optional visitRRevokeRoleDBPrivilege( - final AuthorRelationalPlan plan, final TablePattern pattern) { - return visitAuthorDBPlan(plan, pattern); - } - - private Optional visitAuthorDBPlan( - final AuthorRelationalPlan plan, final TablePattern pattern) { - return pattern.matchesDatabase(plan.getDatabaseName()) ? Optional.of(plan) : Optional.empty(); - } - - @Override - public Optional visitRGrantUserTB( - final AuthorRelationalPlan plan, final TablePattern pattern) { - return visitAuthorTBPlan(plan, pattern); - } - - @Override - public Optional visitRGrantRoleTB( - final AuthorRelationalPlan plan, final TablePattern pattern) { - return visitAuthorTBPlan(plan, pattern); - } - - @Override - public Optional visitRRevokeUserTBPrivilege( - final AuthorRelationalPlan plan, final TablePattern pattern) { - return visitAuthorTBPlan(plan, pattern); - } - - @Override - public Optional visitRRevokeRoleTBPrivilege( - final AuthorRelationalPlan plan, final TablePattern pattern) { - return visitAuthorTBPlan(plan, pattern); - } - - private Optional visitAuthorTBPlan( - final AuthorRelationalPlan plan, final TablePattern pattern) { - return pattern.matchesDatabase(plan.getDatabaseName()) - && pattern.matchesTable(plan.getTableName()) - ? Optional.of(plan) - : Optional.empty(); - } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/extractor/PipeConfigPhysicalPlanTableScopeParseVisitor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/extractor/PipeConfigPhysicalPlanTableScopeParseVisitor.java deleted file mode 100644 index e5673c0dafd4..000000000000 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/extractor/PipeConfigPhysicalPlanTableScopeParseVisitor.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.confignode.manager.pipe.extractor; - -import org.apache.iotdb.commons.auth.entity.PrivilegeType; -import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; -import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; -import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanVisitor; -import org.apache.iotdb.confignode.consensus.request.write.auth.AuthorRelationalPlan; -import org.apache.iotdb.confignode.consensus.request.write.auth.AuthorTreePlan; - -import java.util.Arrays; -import java.util.Collections; -import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; - -public class PipeConfigPhysicalPlanTableScopeParseVisitor - extends ConfigPhysicalPlanVisitor, Void> { - @Override - public Optional visitPlan(final ConfigPhysicalPlan plan, final Void context) { - return Optional.of(plan); - } - - @Override - public Optional visitRGrantUserAll( - final AuthorRelationalPlan plan, final Void context) { - return visitTableAuthorPlan(plan, ConfigPhysicalPlanType.GrantUser); - } - - @Override - public Optional visitRGrantRoleAll( - final AuthorRelationalPlan plan, final Void context) { - return visitTableAuthorPlan(plan, ConfigPhysicalPlanType.GrantRole); - } - - @Override - public Optional visitRRevokeUserAll( - final AuthorRelationalPlan plan, final Void context) { - return visitTableAuthorPlan(plan, ConfigPhysicalPlanType.RevokeUser); - } - - @Override - public Optional visitRRevokeRoleAll( - final AuthorRelationalPlan plan, final Void context) { - return visitTableAuthorPlan(plan, ConfigPhysicalPlanType.RevokeRole); - } - - private Optional visitTableAuthorPlan( - final AuthorRelationalPlan authorRelationalPlan, final ConfigPhysicalPlanType type) { - final Set permissions = - Arrays.stream(PrivilegeType.values()) - .filter(PrivilegeType::forRelationalSys) - .map(Enum::ordinal) - .collect(Collectors.toSet()); - return !permissions.isEmpty() - ? Optional.of( - new AuthorTreePlan( - type, - authorRelationalPlan.getUserName(), - authorRelationalPlan.getRoleName(), - authorRelationalPlan.getPassword(), - authorRelationalPlan.getNewPassword(), - permissions, - authorRelationalPlan.getGrantOpt(), - Collections.emptyList())) - : Optional.empty(); - } -} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/extractor/PipeConfigPhysicalPlanTreePatternParseVisitor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/extractor/PipeConfigPhysicalPlanTreePatternParseVisitor.java index 344a61c2304f..f5e5888ed7d5 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/extractor/PipeConfigPhysicalPlanTreePatternParseVisitor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/extractor/PipeConfigPhysicalPlanTreePatternParseVisitor.java @@ -25,6 +25,7 @@ import org.apache.iotdb.commons.pipe.datastructure.pattern.IoTDBTreePattern; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanVisitor; +import org.apache.iotdb.confignode.consensus.request.write.auth.AuthorPlan; import org.apache.iotdb.confignode.consensus.request.write.auth.AuthorTreePlan; import org.apache.iotdb.confignode.consensus.request.write.database.DatabaseSchemaPlan; import org.apache.iotdb.confignode.consensus.request.write.database.DeleteDatabasePlan; @@ -165,51 +166,52 @@ public Optional visitExtendSchemaTemplate( @Override public Optional visitGrantUser( - final AuthorTreePlan grantUserPlan, final IoTDBTreePattern pattern) { + final AuthorPlan grantUserPlan, final IoTDBTreePattern pattern) { return visitTreeAuthorPlan(grantUserPlan, pattern); } @Override public Optional visitRevokeUser( - final AuthorTreePlan revokeUserPlan, final IoTDBTreePattern pattern) { + final AuthorPlan revokeUserPlan, final IoTDBTreePattern pattern) { return visitTreeAuthorPlan(revokeUserPlan, pattern); } @Override public Optional visitGrantRole( - final AuthorTreePlan revokeUserPlan, final IoTDBTreePattern pattern) { + final AuthorPlan revokeUserPlan, final IoTDBTreePattern pattern) { return visitTreeAuthorPlan(revokeUserPlan, pattern); } @Override public Optional visitRevokeRole( - final AuthorTreePlan revokeUserPlan, final IoTDBTreePattern pattern) { + final AuthorPlan revokeUserPlan, final IoTDBTreePattern pattern) { return visitTreeAuthorPlan(revokeUserPlan, pattern); } private Optional visitTreeAuthorPlan( - final AuthorTreePlan pathRelatedAuthorTreePlan, final IoTDBTreePattern pattern) { + final AuthorPlan pathRelatedAuthorPlan, final IoTDBTreePattern pattern) { + AuthorTreePlan plan = (AuthorTreePlan) pathRelatedAuthorPlan; final List intersectedPaths = - pathRelatedAuthorTreePlan.getNodeNameList().stream() + plan.getNodeNameList().stream() .map(pattern::getIntersection) .flatMap(Collection::stream) .collect(Collectors.toList()); final Set permissions = !intersectedPaths.isEmpty() - ? pathRelatedAuthorTreePlan.getPermissions() - : pathRelatedAuthorTreePlan.getPermissions().stream() + ? plan.getPermissions() + : plan.getPermissions().stream() .filter(permission -> !PrivilegeType.values()[permission].isPathPrivilege()) .collect(Collectors.toSet()); return !permissions.isEmpty() ? Optional.of( new AuthorTreePlan( - pathRelatedAuthorTreePlan.getAuthorType(), - pathRelatedAuthorTreePlan.getUserName(), - pathRelatedAuthorTreePlan.getRoleName(), - pathRelatedAuthorTreePlan.getPassword(), - pathRelatedAuthorTreePlan.getNewPassword(), + plan.getAuthorType(), + plan.getUserName(), + plan.getRoleName(), + plan.getPassword(), + plan.getNewPassword(), permissions, - pathRelatedAuthorTreePlan.getGrantOpt(), + plan.getGrantOpt(), intersectedPaths)) : Optional.empty(); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/extractor/PipeConfigPhysicalPlanTreeScopeParseVisitor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/extractor/PipeConfigPhysicalPlanTreeScopeParseVisitor.java deleted file mode 100644 index 175b836532f7..000000000000 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/extractor/PipeConfigPhysicalPlanTreeScopeParseVisitor.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.confignode.manager.pipe.extractor; - -import org.apache.iotdb.commons.auth.entity.PrivilegeType; -import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; -import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanVisitor; -import org.apache.iotdb.confignode.consensus.request.write.auth.AuthorTreePlan; - -import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; - -public class PipeConfigPhysicalPlanTreeScopeParseVisitor - extends ConfigPhysicalPlanVisitor, Void> { - @Override - public Optional visitPlan(final ConfigPhysicalPlan plan, final Void context) { - return Optional.of(plan); - } - - @Override - public Optional visitGrantRole( - final AuthorTreePlan grantRolePlan, final Void context) { - return visitTreeAuthorPlan(grantRolePlan); - } - - @Override - public Optional visitGrantUser( - final AuthorTreePlan grantUserPlan, final Void context) { - return visitTreeAuthorPlan(grantUserPlan); - } - - @Override - public Optional visitRevokeUser( - final AuthorTreePlan revokeUserPlan, final Void context) { - return visitTreeAuthorPlan(revokeUserPlan); - } - - @Override - public Optional visitRevokeRole( - final AuthorTreePlan revokeRolePlan, final Void context) { - return visitTreeAuthorPlan(revokeRolePlan); - } - - private Optional visitTreeAuthorPlan(final AuthorTreePlan authorTreePlan) { - final Set permissions = - authorTreePlan.getPermissions().stream() - .filter(permission -> PrivilegeType.values()[permission].forRelationalSys()) - .collect(Collectors.toSet()); - return !permissions.isEmpty() - ? Optional.of( - new AuthorTreePlan( - authorTreePlan.getAuthorType(), - authorTreePlan.getUserName(), - authorTreePlan.getRoleName(), - authorTreePlan.getPassword(), - authorTreePlan.getNewPassword(), - permissions, - authorTreePlan.getGrantOpt(), - authorTreePlan.getNodeNameList())) - : Optional.empty(); - } -} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/receiver/protocol/IoTDBConfigNodeReceiver.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/receiver/protocol/IoTDBConfigNodeReceiver.java index 15da365f57ee..48d9a6f2f167 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/receiver/protocol/IoTDBConfigNodeReceiver.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/receiver/protocol/IoTDBConfigNodeReceiver.java @@ -41,7 +41,6 @@ import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; import org.apache.iotdb.confignode.consensus.request.write.auth.AuthorPlan; -import org.apache.iotdb.confignode.consensus.request.write.auth.AuthorRelationalPlan; import org.apache.iotdb.confignode.consensus.request.write.auth.AuthorTreePlan; import org.apache.iotdb.confignode.consensus.request.write.database.DatabaseSchemaPlan; import org.apache.iotdb.confignode.consensus.request.write.database.DeleteDatabasePlan; @@ -318,35 +317,6 @@ private TSStatus checkPermission(final ConfigPhysicalPlan plan) { return configManager .checkUserPrivileges(username, new PrivilegeUnion(PrivilegeType.USE_TRIGGER)) .getStatus(); - case PipeCreateTable: - return configManager - .checkUserPrivileges( - username, - new PrivilegeUnion( - ((PipeCreateTablePlan) plan).getDatabase(), - ((PipeCreateTablePlan) plan).getTable().getTableName(), - PrivilegeType.CREATE)) - .getStatus(); - case AddTableColumn: - case SetTableProperties: - case CommitDeleteColumn: - return configManager - .checkUserPrivileges( - username, - new PrivilegeUnion( - ((PipeCreateTablePlan) plan).getDatabase(), - ((PipeCreateTablePlan) plan).getTable().getTableName(), - PrivilegeType.ALTER)) - .getStatus(); - case CommitDeleteTable: - return configManager - .checkUserPrivileges( - username, - new PrivilegeUnion( - ((PipeCreateTablePlan) plan).getDatabase(), - ((PipeCreateTablePlan) plan).getTable().getTableName(), - PrivilegeType.DROP)) - .getStatus(); case GrantRole: case GrantUser: case RevokeUser: @@ -368,122 +338,22 @@ private TSStatus checkPermission(final ConfigPhysicalPlan plan) { } } return StatusUtils.OK; - case RGrantUserAny: - case RGrantRoleAny: - case RRevokeUserAny: - case RRevokeRoleAny: - for (final int permission : ((AuthorRelationalPlan) plan).getPermissions()) { - final TSStatus status = - configManager - .checkUserPrivileges( - username, new PrivilegeUnion(PrivilegeType.values()[permission], true, true)) - .getStatus(); - if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - return status; - } - } - return StatusUtils.OK; - case RGrantUserAll: - case RGrantRoleAll: - case RRevokeUserAll: - case RRevokeRoleAll: - for (PrivilegeType privilegeType : PrivilegeType.values()) { - final TSStatus status; - if (privilegeType.isRelationalPrivilege()) { - status = - configManager - .checkUserPrivileges(username, new PrivilegeUnion(privilegeType, true, true)) - .getStatus(); - } else if (privilegeType.forRelationalSys()) { - status = - configManager - .checkUserPrivileges(username, new PrivilegeUnion(privilegeType, true)) - .getStatus(); - } else { - continue; - } - if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - return status; - } - } - return StatusUtils.OK; - case RGrantUserDBPriv: - case RGrantRoleDBPriv: - case RRevokeUserDBPriv: - case RRevokeRoleDBPriv: - for (final int permission : ((AuthorRelationalPlan) plan).getPermissions()) { - final TSStatus status = - configManager - .checkUserPrivileges( - username, - new PrivilegeUnion( - ((AuthorRelationalPlan) plan).getDatabaseName(), - PrivilegeType.values()[permission], - true)) - .getStatus(); - if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - return status; - } - } - return StatusUtils.OK; - case RGrantUserTBPriv: - case RGrantRoleTBPriv: - case RRevokeUserTBPriv: - case RRevokeRoleTBPriv: - for (final int permission : ((AuthorRelationalPlan) plan).getPermissions()) { - final TSStatus status = - configManager - .checkUserPrivileges( - username, - new PrivilegeUnion( - ((AuthorRelationalPlan) plan).getDatabaseName(), - ((AuthorRelationalPlan) plan).getTableName(), - PrivilegeType.values()[permission], - true)) - .getStatus(); - if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - return status; - } - } - return StatusUtils.OK; - case RGrantUserSysPri: - case RGrantRoleSysPri: - case RRevokeUserSysPri: - case RRevokeRoleSysPri: - for (final int permission : ((AuthorRelationalPlan) plan).getPermissions()) { - final TSStatus status = - configManager - .checkUserPrivileges( - username, new PrivilegeUnion(PrivilegeType.values()[permission], true)) - .getStatus(); - if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - return status; - } - } - return StatusUtils.OK; case UpdateUser: - case RUpdateUser: return ((AuthorPlan) plan).getUserName().equals(username) ? StatusUtils.OK : configManager .checkUserPrivileges(username, new PrivilegeUnion(PrivilegeType.MANAGE_USER)) .getStatus(); case CreateUser: - case RCreateUser: case CreateUserWithRawPassword: case DropUser: - case RDropUser: return configManager .checkUserPrivileges(username, new PrivilegeUnion(PrivilegeType.MANAGE_USER)) .getStatus(); case CreateRole: - case RCreateRole: case DropRole: - case RDropRole: case GrantRoleToUser: - case RGrantUserRole: case RevokeRoleFromUser: - case RRevokeUserRole: return configManager .checkUserPrivileges(username, new PrivilegeUnion(PrivilegeType.MANAGE_ROLE)) .getStatus(); @@ -682,9 +552,6 @@ private TSStatus executePlan(final ConfigPhysicalPlan plan) throws ConsensusExce ByteBuffer.wrap(((PipeDeleteDevicesPlan) plan).getModBytes())), true) .getStatus(); - case CreateUser: - case CreateUserWithRawPassword: - case CreateRole: case DropUser: case DropRole: case GrantRole: @@ -694,34 +561,11 @@ private TSStatus executePlan(final ConfigPhysicalPlan plan) throws ConsensusExce case RevokeRole: case RevokeRoleFromUser: case UpdateUser: - case RCreateUser: - case RCreateRole: - case RDropUser: - case RDropRole: - case RGrantRoleAll: - case RGrantUserAll: - case RRevokeRoleAll: - case RRevokeUserAll: - case RGrantRoleAny: - case RGrantUserAny: - case RRevokeUserAny: - case RRevokeRoleAny: - case RGrantRoleDBPriv: - case RGrantUserDBPriv: - case RRevokeRoleDBPriv: - case RRevokeUserDBPriv: - case RGrantRoleTBPriv: - case RGrantUserTBPriv: - case RRevokeRoleTBPriv: - case RRevokeUserTBPriv: - case RGrantRoleSysPri: - case RGrantUserSysPri: - case RRevokeRoleSysPri: - case RRevokeUserSysPri: - case RGrantUserRole: - case RRevokeUserRole: return configManager.getPermissionManager().operatePermission((AuthorPlan) plan, true); case CreateSchemaTemplate: + case CreateUser: + case CreateRole: + case CreateUserWithRawPassword: default: return configManager.getConsensusManager().write(new PipeEnrichedPlan(plan)); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/receiver/visitor/PipeConfigPhysicalPlanTSStatusVisitor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/receiver/visitor/PipeConfigPhysicalPlanTSStatusVisitor.java index 653ef4339ea9..0583e32e2801 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/receiver/visitor/PipeConfigPhysicalPlanTSStatusVisitor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/receiver/visitor/PipeConfigPhysicalPlanTSStatusVisitor.java @@ -22,8 +22,7 @@ import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanVisitor; -import org.apache.iotdb.confignode.consensus.request.write.auth.AuthorRelationalPlan; -import org.apache.iotdb.confignode.consensus.request.write.auth.AuthorTreePlan; +import org.apache.iotdb.confignode.consensus.request.write.auth.AuthorPlan; import org.apache.iotdb.confignode.consensus.request.write.database.DatabaseSchemaPlan; import org.apache.iotdb.confignode.consensus.request.write.database.DeleteDatabasePlan; import org.apache.iotdb.confignode.consensus.request.write.database.SetTTLPlan; @@ -192,7 +191,7 @@ public TSStatus visitPipeDeactivateTemplate( } @Override - public TSStatus visitCreateUser(final AuthorTreePlan plan, final TSStatus context) { + public TSStatus visitCreateUser(final AuthorPlan plan, final TSStatus context) { if (context.getCode() == TSStatusCode.USER_ALREADY_EXIST.getStatusCode()) { return new TSStatus(TSStatusCode.PIPE_RECEIVER_IDEMPOTENT_CONFLICT_EXCEPTION.getStatusCode()) .setMessage(context.getMessage()); @@ -201,7 +200,7 @@ public TSStatus visitCreateUser(final AuthorTreePlan plan, final TSStatus contex } @Override - public TSStatus visitCreateRawUser(final AuthorTreePlan plan, final TSStatus context) { + public TSStatus visitCreateRawUser(final AuthorPlan plan, final TSStatus context) { if (context.getCode() == TSStatusCode.USER_ALREADY_EXIST.getStatusCode()) { return new TSStatus(TSStatusCode.PIPE_RECEIVER_IDEMPOTENT_CONFLICT_EXCEPTION.getStatusCode()) .setMessage(context.getMessage()); @@ -210,7 +209,7 @@ public TSStatus visitCreateRawUser(final AuthorTreePlan plan, final TSStatus con } @Override - public TSStatus visitUpdateUser(final AuthorTreePlan plan, final TSStatus context) { + public TSStatus visitUpdateUser(final AuthorPlan plan, final TSStatus context) { if (context.getCode() == TSStatusCode.USER_NOT_EXIST.getStatusCode()) { return new TSStatus(TSStatusCode.PIPE_RECEIVER_USER_CONFLICT_EXCEPTION.getStatusCode()) .setMessage(context.getMessage()); @@ -219,7 +218,7 @@ public TSStatus visitUpdateUser(final AuthorTreePlan plan, final TSStatus contex } @Override - public TSStatus visitDropUser(final AuthorTreePlan plan, final TSStatus context) { + public TSStatus visitDropUser(final AuthorPlan plan, final TSStatus context) { if (context.getCode() == TSStatusCode.USER_NOT_EXIST.getStatusCode()) { return new TSStatus(TSStatusCode.PIPE_RECEIVER_IDEMPOTENT_CONFLICT_EXCEPTION.getStatusCode()) .setMessage(context.getMessage()); @@ -228,7 +227,7 @@ public TSStatus visitDropUser(final AuthorTreePlan plan, final TSStatus context) } @Override - public TSStatus visitGrantUser(final AuthorTreePlan plan, final TSStatus context) { + public TSStatus visitGrantUser(final AuthorPlan plan, final TSStatus context) { if (context.getCode() == TSStatusCode.NO_PERMISSION.getStatusCode()) { // Admin user return new TSStatus(TSStatusCode.PIPE_RECEIVER_IDEMPOTENT_CONFLICT_EXCEPTION.getStatusCode()) @@ -242,7 +241,7 @@ public TSStatus visitGrantUser(final AuthorTreePlan plan, final TSStatus context } @Override - public TSStatus visitRevokeUser(final AuthorTreePlan plan, final TSStatus context) { + public TSStatus visitRevokeUser(final AuthorPlan plan, final TSStatus context) { if (context.getCode() == TSStatusCode.NOT_HAS_PRIVILEGE.getStatusCode()) { return new TSStatus(TSStatusCode.PIPE_RECEIVER_IDEMPOTENT_CONFLICT_EXCEPTION.getStatusCode()) .setMessage(context.getMessage()); @@ -256,7 +255,7 @@ public TSStatus visitRevokeUser(final AuthorTreePlan plan, final TSStatus contex } @Override - public TSStatus visitCreateRole(final AuthorTreePlan plan, final TSStatus context) { + public TSStatus visitCreateRole(final AuthorPlan plan, final TSStatus context) { if (context.getCode() == TSStatusCode.ROLE_ALREADY_EXIST.getStatusCode()) { return new TSStatus(TSStatusCode.PIPE_RECEIVER_IDEMPOTENT_CONFLICT_EXCEPTION.getStatusCode()) .setMessage(context.getMessage()); @@ -265,7 +264,7 @@ public TSStatus visitCreateRole(final AuthorTreePlan plan, final TSStatus contex } @Override - public TSStatus visitDropRole(final AuthorTreePlan plan, final TSStatus context) { + public TSStatus visitDropRole(final AuthorPlan plan, final TSStatus context) { if (context.getCode() == TSStatusCode.ROLE_NOT_EXIST.getStatusCode()) { return new TSStatus(TSStatusCode.PIPE_RECEIVER_IDEMPOTENT_CONFLICT_EXCEPTION.getStatusCode()) .setMessage(context.getMessage()); @@ -274,7 +273,7 @@ public TSStatus visitDropRole(final AuthorTreePlan plan, final TSStatus context) } @Override - public TSStatus visitGrantRole(final AuthorTreePlan plan, final TSStatus context) { + public TSStatus visitGrantRole(final AuthorPlan plan, final TSStatus context) { if (context.getCode() == TSStatusCode.ROLE_NOT_EXIST.getStatusCode()) { return new TSStatus(TSStatusCode.PIPE_RECEIVER_USER_CONFLICT_EXCEPTION.getStatusCode()) .setMessage(context.getMessage()); @@ -283,7 +282,7 @@ public TSStatus visitGrantRole(final AuthorTreePlan plan, final TSStatus context } @Override - public TSStatus visitRevokeRole(final AuthorTreePlan plan, final TSStatus context) { + public TSStatus visitRevokeRole(final AuthorPlan plan, final TSStatus context) { if (context.getCode() == TSStatusCode.NOT_HAS_PRIVILEGE.getStatusCode()) { return new TSStatus(TSStatusCode.PIPE_RECEIVER_IDEMPOTENT_CONFLICT_EXCEPTION.getStatusCode()) .setMessage(context.getMessage()); @@ -295,7 +294,7 @@ public TSStatus visitRevokeRole(final AuthorTreePlan plan, final TSStatus contex } @Override - public TSStatus visitGrantRoleToUser(final AuthorTreePlan plan, final TSStatus context) { + public TSStatus visitGrantRoleToUser(final AuthorPlan plan, final TSStatus context) { if (context.getCode() == TSStatusCode.USER_ALREADY_HAS_ROLE.getStatusCode()) { return new TSStatus(TSStatusCode.PIPE_RECEIVER_IDEMPOTENT_CONFLICT_EXCEPTION.getStatusCode()) .setMessage(context.getMessage()); @@ -304,8 +303,7 @@ public TSStatus visitGrantRoleToUser(final AuthorTreePlan plan, final TSStatus c } @Override - public TSStatus visitRevokeRoleFromUser( - final AuthorTreePlan revokeRoleFromUserPlan, final TSStatus context) { + public TSStatus visitRevokeRoleFromUser(final AuthorPlan plan, final TSStatus context) { if (context.getCode() == TSStatusCode.USER_NOT_HAS_ROLE.getStatusCode()) { return new TSStatus(TSStatusCode.PIPE_RECEIVER_IDEMPOTENT_CONFLICT_EXCEPTION.getStatusCode()) .setMessage(context.getMessage()); @@ -313,184 +311,7 @@ public TSStatus visitRevokeRoleFromUser( return new TSStatus(TSStatusCode.PIPE_RECEIVER_USER_CONFLICT_EXCEPTION.getStatusCode()) .setMessage(context.getMessage()); } - return super.visitRevokeRoleFromUser(revokeRoleFromUserPlan, context); - } - - @Override - public TSStatus visitRCreateUser( - final AuthorRelationalPlan rCreateUserPlan, final TSStatus context) { - return visitAuthorRelationalPlan(rCreateUserPlan, context); - } - - @Override - public TSStatus visitRCreateRole( - final AuthorRelationalPlan rCreateRolePlan, final TSStatus context) { - return visitAuthorRelationalPlan(rCreateRolePlan, context); - } - - @Override - public TSStatus visitRUpdateUser( - final AuthorRelationalPlan rUpdateUserPlan, final TSStatus context) { - return visitAuthorRelationalPlan(rUpdateUserPlan, context); - } - - @Override - public TSStatus visitRDropUserPlan( - final AuthorRelationalPlan rDropUserPlan, final TSStatus context) { - return visitAuthorRelationalPlan(rDropUserPlan, context); - } - - @Override - public TSStatus visitRDropRolePlan( - final AuthorRelationalPlan rDropRolePlan, final TSStatus context) { - return visitAuthorRelationalPlan(rDropRolePlan, context); - } - - @Override - public TSStatus visitRGrantUserRole( - final AuthorRelationalPlan rGrantUserRolePlan, final TSStatus context) { - return visitAuthorRelationalPlan(rGrantUserRolePlan, context); - } - - @Override - public TSStatus visitRRevokeUserRole( - final AuthorRelationalPlan rRevokeUserRolePlan, final TSStatus context) { - return visitAuthorRelationalPlan(rRevokeUserRolePlan, context); - } - - @Override - public TSStatus visitRGrantUserAny( - final AuthorRelationalPlan rGrantUserAnyPlan, final TSStatus context) { - return visitAuthorRelationalPlan(rGrantUserAnyPlan, context); - } - - @Override - public TSStatus visitRGrantRoleAny( - final AuthorRelationalPlan rGrantRoleAnyPlan, final TSStatus context) { - return visitAuthorRelationalPlan(rGrantRoleAnyPlan, context); - } - - @Override - public TSStatus visitRGrantUserAll( - final AuthorRelationalPlan rGrantUserAllPlan, final TSStatus context) { - return visitAuthorRelationalPlan(rGrantUserAllPlan, context); - } - - @Override - public TSStatus visitRGrantRoleAll( - final AuthorRelationalPlan rGrantRoleAllPlan, final TSStatus context) { - return visitAuthorRelationalPlan(rGrantRoleAllPlan, context); - } - - @Override - public TSStatus visitRGrantUserDB( - final AuthorRelationalPlan rGrantUserDBPlan, final TSStatus context) { - return visitAuthorRelationalPlan(rGrantUserDBPlan, context); - } - - @Override - public TSStatus visitRGrantUserTB( - final AuthorRelationalPlan rGrantUserTBPlan, final TSStatus context) { - return visitAuthorRelationalPlan(rGrantUserTBPlan, context); - } - - @Override - public TSStatus visitRGrantRoleDB( - final AuthorRelationalPlan rGrantRoleDBPlan, final TSStatus context) { - return visitAuthorRelationalPlan(rGrantRoleDBPlan, context); - } - - @Override - public TSStatus visitRGrantRoleTB( - final AuthorRelationalPlan rGrantRoleTBPlan, final TSStatus context) { - return visitAuthorRelationalPlan(rGrantRoleTBPlan, context); - } - - @Override - public TSStatus visitRRevokeUserAny( - final AuthorRelationalPlan rRevokeUserAnyPlan, final TSStatus context) { - return visitAuthorRelationalPlan(rRevokeUserAnyPlan, context); - } - - @Override - public TSStatus visitRRevokeRoleAny( - final AuthorRelationalPlan rRevokeRoleAnyPlan, final TSStatus context) { - return visitAuthorRelationalPlan(rRevokeRoleAnyPlan, context); - } - - @Override - public TSStatus visitRRevokeUserAll( - final AuthorRelationalPlan rRevokeUserAllPlan, final TSStatus context) { - return visitAuthorRelationalPlan(rRevokeUserAllPlan, context); - } - - @Override - public TSStatus visitRRevokeRoleAll( - final AuthorRelationalPlan rRevokeRoleAllPlan, final TSStatus context) { - return visitAuthorRelationalPlan(rRevokeRoleAllPlan, context); - } - - @Override - public TSStatus visitRRevokeUserDBPrivilege( - final AuthorRelationalPlan rRevokeUserDBPrivilegePlan, final TSStatus context) { - return visitAuthorRelationalPlan(rRevokeUserDBPrivilegePlan, context); - } - - @Override - public TSStatus visitRRevokeUserTBPrivilege( - final AuthorRelationalPlan rRevokeUserTBPrivilegePlan, final TSStatus context) { - return visitAuthorRelationalPlan(rRevokeUserTBPrivilegePlan, context); - } - - @Override - public TSStatus visitRRevokeRoleDBPrivilege( - final AuthorRelationalPlan rRevokeRoleTBPrivilegePlan, final TSStatus context) { - return visitAuthorRelationalPlan(rRevokeRoleTBPrivilegePlan, context); - } - - @Override - public TSStatus visitRRevokeRoleTBPrivilege( - final AuthorRelationalPlan rRevokeRoleTBPrivilegePlan, final TSStatus context) { - return visitAuthorRelationalPlan(rRevokeRoleTBPrivilegePlan, context); - } - - @Override - public TSStatus visitRGrantUserSysPrivilege( - final AuthorRelationalPlan rGrantUserSysPrivilegePlan, final TSStatus context) { - return visitAuthorRelationalPlan(rGrantUserSysPrivilegePlan, context); - } - - @Override - public TSStatus visitRGrantRoleSysPrivilege( - final AuthorRelationalPlan rGrantRoleSysPrivilegePlan, final TSStatus context) { - return visitAuthorRelationalPlan(rGrantRoleSysPrivilegePlan, context); - } - - @Override - public TSStatus visitRRevokeUserSysPrivilege( - final AuthorRelationalPlan rRevokeUserSysPrivilegePlan, final TSStatus context) { - return visitAuthorRelationalPlan(rRevokeUserSysPrivilegePlan, context); - } - - @Override - public TSStatus visitRRevokeRoleSysPrivilege( - final AuthorRelationalPlan rRevokeRoleSysPrivilegePlan, final TSStatus context) { - return visitAuthorRelationalPlan(rRevokeRoleSysPrivilegePlan, context); - } - - private TSStatus visitAuthorRelationalPlan( - final AuthorRelationalPlan plan, final TSStatus context) { - if (context.getCode() == TSStatusCode.USER_NOT_EXIST.getStatusCode() - || context.getCode() == TSStatusCode.USER_ALREADY_EXIST.getStatusCode() - || context.getCode() == TSStatusCode.ROLE_NOT_EXIST.getStatusCode() - || context.getCode() == TSStatusCode.ROLE_ALREADY_EXIST.getStatusCode() - || context.getCode() == TSStatusCode.USER_ALREADY_HAS_ROLE.getStatusCode() - || context.getCode() == TSStatusCode.USER_NOT_HAS_ROLE.getStatusCode() - || context.getCode() == TSStatusCode.NO_PERMISSION.getStatusCode()) { - return new TSStatus(TSStatusCode.PIPE_RECEIVER_IDEMPOTENT_CONFLICT_EXCEPTION.getStatusCode()) - .setMessage(context.getMessage()); - } - return visitPlan(plan, context); + return super.visitRevokeRoleFromUser(plan, context); } @Override diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/AuthorInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/AuthorInfo.java index 88d173da0464..62abe8af8198 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/AuthorInfo.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/AuthorInfo.java @@ -320,21 +320,6 @@ public TSStatus authorNonQuery(AuthorRelationalPlan authorPlan) { } break; case RGrantUserAll: - // database scope and table scope all - if (!database.isEmpty()) { - for (PrivilegeType privilege : privileges) { - if (privilege.isRelationalPrivilege()) { - if (table.isEmpty()) { - authorizer.grantPrivilegeToUser( - userName, new PrivilegeUnion(database, privilege, grantOpt)); - } else { - authorizer.grantPrivilegeToUser( - userName, new PrivilegeUnion(database, table, privilege, grantOpt)); - } - } - } - break; - } for (PrivilegeType privilege : PrivilegeType.values()) { if (privilege.forRelationalSys()) { authorizer.grantPrivilegeToUser(userName, new PrivilegeUnion(privilege, grantOpt)); @@ -346,21 +331,6 @@ public TSStatus authorNonQuery(AuthorRelationalPlan authorPlan) { } break; case RGrantRoleAll: - // database scope and table scope all - if (!database.isEmpty()) { - for (PrivilegeType privilege : privileges) { - if (privilege.isRelationalPrivilege()) { - if (table.isEmpty()) { - authorizer.grantPrivilegeToRole( - roleName, new PrivilegeUnion(database, privilege, grantOpt)); - } else { - authorizer.grantPrivilegeToRole( - roleName, new PrivilegeUnion(database, table, privilege, grantOpt)); - } - } - } - break; - } for (PrivilegeType privilege : PrivilegeType.values()) { if (privilege.forRelationalSys()) { authorizer.grantPrivilegeToRole(roleName, new PrivilegeUnion(privilege, grantOpt)); @@ -408,38 +378,26 @@ public TSStatus authorNonQuery(AuthorRelationalPlan authorPlan) { } break; case RRevokeUserAll: - if (!database.isEmpty()) { - for (PrivilegeType privilege : PrivilegeType.values()) { - if (privilege.isRelationalPrivilege()) { - if (table.isEmpty()) { - authorizer.revokePrivilegeFromUser( - userName, new PrivilegeUnion(database, privilege, grantOpt)); - } else { - authorizer.revokePrivilegeFromUser( - userName, new PrivilegeUnion(database, table, privilege, grantOpt)); - } - } + for (PrivilegeType privilege : PrivilegeType.values()) { + if (privilege.forRelationalSys()) { + authorizer.revokePrivilegeFromUser(userName, new PrivilegeUnion(privilege, grantOpt)); + } + if (privilege.isRelationalPrivilege()) { + authorizer.revokePrivilegeFromUser( + userName, new PrivilegeUnion(privilege, grantOpt, true)); } - break; } - authorizer.revokeAllPrivilegeFromUser(userName); break; case RRevokeRoleAll: - if (!database.isEmpty()) { - for (PrivilegeType privilege : PrivilegeType.values()) { - if (privilege.isRelationalPrivilege()) { - if (table.isEmpty()) { - authorizer.revokePrivilegeFromRole( - roleName, new PrivilegeUnion(database, privilege, grantOpt)); - } else { - authorizer.revokePrivilegeFromRole( - roleName, new PrivilegeUnion(database, table, privilege, grantOpt)); - } - } + for (PrivilegeType privilege : PrivilegeType.values()) { + if (privilege.forRelationalSys()) { + authorizer.revokePrivilegeFromRole(roleName, new PrivilegeUnion(privilege, grantOpt)); + } + if (privilege.isRelationalPrivilege()) { + authorizer.revokePrivilegeFromRole( + roleName, new PrivilegeUnion(privilege, grantOpt, true)); } - break; } - authorizer.revokeAllPrivilegeFromRole(roleName); break; case RRevokeUserDBPriv: for (PrivilegeType privilege : privileges) { diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/extractor/PipeConfigPhysicalPlanScopeParseVisitorTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/extractor/PipeConfigPhysicalPlanScopeParseVisitorTest.java deleted file mode 100644 index 6e5e9870fbfa..000000000000 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/extractor/PipeConfigPhysicalPlanScopeParseVisitorTest.java +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.confignode.manager.pipe.extractor; - -import org.apache.iotdb.commons.auth.entity.PrivilegeType; -import org.apache.iotdb.commons.path.PartialPath; -import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; -import org.apache.iotdb.confignode.consensus.request.write.auth.AuthorRelationalPlan; -import org.apache.iotdb.confignode.consensus.request.write.auth.AuthorTreePlan; - -import org.junit.Assert; -import org.junit.Test; - -import java.util.Arrays; -import java.util.Collections; -import java.util.HashSet; -import java.util.stream.Collectors; - -public class PipeConfigPhysicalPlanScopeParseVisitorTest { - @Test - public void testTreeScopeParsing() { - testTreeScopeParsing(ConfigPhysicalPlanType.GrantRole, false); - testTreeScopeParsing(ConfigPhysicalPlanType.RevokeRole, false); - testTreeScopeParsing(ConfigPhysicalPlanType.GrantUser, true); - testTreeScopeParsing(ConfigPhysicalPlanType.RevokeUser, true); - } - - private void testTreeScopeParsing(final ConfigPhysicalPlanType type, final boolean isUser) { - Assert.assertEquals( - new AuthorTreePlan( - type, - isUser ? "user" : "", - isUser ? "" : "role", - "", - "", - new HashSet<>( - Arrays.stream(PrivilegeType.values()) - .filter(PrivilegeType::forRelationalSys) - .map(Enum::ordinal) - .collect(Collectors.toList())), - false, - Collections.singletonList(new PartialPath(new String[] {"root", "**"}))), - IoTDBConfigRegionExtractor.TREE_SCOPE_PARSE_VISITOR - .process( - new AuthorTreePlan( - type, - isUser ? "user" : "", - isUser ? "" : "role", - "", - "", - new HashSet<>( - Arrays.stream(PrivilegeType.values()) - .filter(privilegeType -> !privilegeType.isRelationalPrivilege()) - .map(Enum::ordinal) - .collect(Collectors.toList())), - false, - Collections.singletonList(new PartialPath(new String[] {"root", "**"}))), - null) - .orElseThrow(AssertionError::new)); - } - - @Test - public void testTableScopeParsing() { - testTableScopeParsing( - ConfigPhysicalPlanType.GrantRole, ConfigPhysicalPlanType.RGrantRoleAll, false); - testTableScopeParsing( - ConfigPhysicalPlanType.RevokeRole, ConfigPhysicalPlanType.RRevokeRoleAll, false); - testTableScopeParsing( - ConfigPhysicalPlanType.GrantUser, ConfigPhysicalPlanType.RGrantUserAll, true); - testTableScopeParsing( - ConfigPhysicalPlanType.RevokeUser, ConfigPhysicalPlanType.RRevokeUserAll, true); - } - - private void testTableScopeParsing( - final ConfigPhysicalPlanType outputType, - final ConfigPhysicalPlanType inputType, - final boolean isUser) { - Assert.assertEquals( - new AuthorTreePlan( - outputType, - isUser ? "user" : "", - isUser ? "" : "role", - "", - "", - new HashSet<>( - Arrays.stream(PrivilegeType.values()) - .filter(PrivilegeType::forRelationalSys) - .map(Enum::ordinal) - .collect(Collectors.toList())), - true, - Collections.emptyList()), - IoTDBConfigRegionExtractor.TABLE_SCOPE_PARSE_VISITOR - .process( - new AuthorRelationalPlan( - inputType, isUser ? "user" : "", isUser ? "" : "role", "", "", -1, true), - null) - .orElseThrow(AssertionError::new)); - } -} diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/extractor/PipeConfigPhysicalPlanTablePatternParseVisitorTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/extractor/PipeConfigPhysicalPlanTablePatternParseVisitorTest.java index 64ec14be5d84..0681dc5aaca6 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/extractor/PipeConfigPhysicalPlanTablePatternParseVisitorTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/extractor/PipeConfigPhysicalPlanTablePatternParseVisitorTest.java @@ -19,12 +19,10 @@ package org.apache.iotdb.confignode.manager.pipe.extractor; -import org.apache.iotdb.commons.auth.entity.PrivilegeType; import org.apache.iotdb.commons.pipe.datastructure.pattern.TablePattern; import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; -import org.apache.iotdb.confignode.consensus.request.write.auth.AuthorRelationalPlan; import org.apache.iotdb.confignode.consensus.request.write.database.DatabaseSchemaPlan; import org.apache.iotdb.confignode.consensus.request.write.database.DeleteDatabasePlan; import org.apache.iotdb.confignode.consensus.request.write.pipe.payload.PipeCreateTablePlan; @@ -114,29 +112,6 @@ public void testCommitDeleteTable() { new CommitDeleteTablePlan("da", "ac")); } - @Test - public void testAuth() { - testInput( - new AuthorRelationalPlan( - ConfigPhysicalPlanType.RGrantRoleAll, "", "role", "", "", -1, false), - new AuthorRelationalPlan( - ConfigPhysicalPlanType.RGrantUserDBPriv, - "user", - "", - "da", - "", - PrivilegeType.SELECT.ordinal(), - false), - new AuthorRelationalPlan( - ConfigPhysicalPlanType.RGrantUserTBPriv, - "user", - "", - "db1", - "ac", - PrivilegeType.DROP.ordinal(), - false)); - } - private void testInput( final ConfigPhysicalPlan trueInput, final ConfigPhysicalPlan falseInput1, diff --git a/iotdb-core/consensus/pom.xml b/iotdb-core/consensus/pom.xml index 00295dcbfd27..bbf4e2e34592 100644 --- a/iotdb-core/consensus/pom.xml +++ b/iotdb-core/consensus/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-core - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT iotdb-consensus IoTDB: Core: Consensus @@ -39,32 +39,32 @@ org.apache.iotdb node-commons - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.apache.iotdb metrics-interface - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.apache.iotdb iotdb-thrift-commons - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.apache.iotdb iotdb-thrift-consensus - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.apache.iotdb service-rpc - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.apache.iotdb pipe-api - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.apache.ratis diff --git a/iotdb-core/datanode/pom.xml b/iotdb-core/datanode/pom.xml index ad5c6aaced52..8e8e5f6e2325 100644 --- a/iotdb-core/datanode/pom.xml +++ b/iotdb-core/datanode/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-core - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT iotdb-server IoTDB: Core: Data-Node (Server) @@ -37,12 +37,12 @@ org.apache.iotdb service-rpc - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.apache.iotdb iotdb-consensus - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.apache.tsfile @@ -57,82 +57,82 @@ org.apache.iotdb external-api - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.apache.iotdb openapi - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.apache.iotdb node-commons - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.apache.iotdb isession - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.apache.iotdb iotdb-antlr - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.apache.iotdb iotdb-relational-grammar - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.apache.iotdb iotdb-thrift-commons - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.apache.iotdb iotdb-thrift-consensus - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.apache.iotdb udf-api - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.apache.iotdb trigger-api - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.apache.iotdb metrics-interface - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.apache.iotdb iotdb-thrift - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.apache.iotdb iotdb-thrift-confignode - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.apache.iotdb iotdb-thrift-ainode - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.apache.iotdb pipe-api - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.apache.iotdb iotdb-session - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.apache.commons @@ -314,7 +314,7 @@ org.apache.iotdb metrics-core - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.mockito diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/auth/AuthorityChecker.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/auth/AuthorityChecker.java index d084e17123fb..0d6d2727250c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/auth/AuthorityChecker.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/auth/AuthorityChecker.java @@ -66,9 +66,6 @@ public class AuthorityChecker { public static final TSStatus SUCCEED = new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); - public static final String ONLY_ADMIN_ALLOWED = - "No permissions for this operation, only root user is allowed"; - private static final String NO_PERMISSION_PROMOTION = "No permissions for this operation, please add privilege "; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/connector/PipeRealtimePriorityBlockingQueue.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/connector/PipeRealtimePriorityBlockingQueue.java index 3edbcef539c5..185b60c29e98 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/connector/PipeRealtimePriorityBlockingQueue.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/connector/PipeRealtimePriorityBlockingQueue.java @@ -51,8 +51,6 @@ public PipeRealtimePriorityBlockingQueue() { @Override public boolean directOffer(final Event event) { - checkBeforeOffer(event); - if (event instanceof TsFileInsertionEvent) { tsfileInsertEventDeque.add((TsFileInsertionEvent) event); return true; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/processor/PipeProcessorSubtask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/processor/PipeProcessorSubtask.java index b4bd024bfec4..7f29b3c55f99 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/processor/PipeProcessorSubtask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/processor/PipeProcessorSubtask.java @@ -228,9 +228,10 @@ public void close() { PipeProcessorMetrics.getInstance().deregister(taskID); try { isClosed.set(true); + + // pipeProcessor closes first, then no more events will be added into outputEventCollector. + // only after that, outputEventCollector can be closed. pipeProcessor.close(); - // It is important to note that even if the subtask and its corresponding processor are - // closed, the execution thread may still deliver events downstream. } catch (final Exception e) { LOGGER.info( "Exception occurred when closing pipe processor subtask {}, root cause: {}", diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/processor/PipeProcessorSubtaskWorker.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/processor/PipeProcessorSubtaskWorker.java index 813d07317212..44fd578c84db 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/processor/PipeProcessorSubtaskWorker.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/processor/PipeProcessorSubtaskWorker.java @@ -33,6 +33,9 @@ public class PipeProcessorSubtaskWorker extends WrappedRunnable { private static final Logger LOGGER = LoggerFactory.getLogger(PipeProcessorSubtaskWorker.class); + private static final long CLOSED_SUBTASK_CLEANUP_ROUND_INTERVAL = 1000; + private long closedSubtaskCleanupRoundCounter = 0; + private static final int SLEEP_INTERVAL_ADJUSTMENT_ROUND_INTERVAL = 100; private int totalRoundInAdjustmentInterval = 0; private int workingRoundInAdjustmentInterval = 0; @@ -53,10 +56,12 @@ public void runMayThrow() { } private void cleanupClosedSubtasksIfNecessary() { - subtasks.stream() - .filter(PipeProcessorSubtask::isClosed) - .collect(Collectors.toList()) - .forEach(subtasks::remove); + if (++closedSubtaskCleanupRoundCounter % CLOSED_SUBTASK_CLEANUP_ROUND_INTERVAL == 0) { + subtasks.stream() + .filter(PipeProcessorSubtask::isClosed) + .collect(Collectors.toList()) + .forEach(subtasks::remove); + } } private boolean runSubtasks() { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeTsFileInsertionEvent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeTsFileInsertionEvent.java index 7b8e9a69b73c..78b67fc701e6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeTsFileInsertionEvent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeTsFileInsertionEvent.java @@ -56,7 +56,6 @@ import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicReference; import static org.apache.tsfile.common.constant.TsFileConstant.PATH_ROOT; import static org.apache.tsfile.common.constant.TsFileConstant.PATH_SEPARATOR; @@ -82,7 +81,7 @@ public class PipeTsFileInsertionEvent extends PipeInsertionEvent private final boolean isGeneratedByHistoricalExtractor; private final AtomicBoolean isClosed; - private final AtomicReference eventParser; + private TsFileInsertionEventParser eventParser; // The point count of the TsFile. Used for metrics on PipeConsensus' receiver side. // May be updated after it is flushed. Should be negative if not set. @@ -189,8 +188,6 @@ public PipeTsFileInsertionEvent( // If the status is "closed", then the resource status is "closed", the tsFile won't be altered // and can be sent. isClosed.set(resource.isClosed()); - - this.eventParser = new AtomicReference<>(null); } /** @@ -573,12 +570,13 @@ public boolean isGeneratedByHistoricalExtractor() { private TsFileInsertionEventParser initEventParser() { try { - eventParser.compareAndSet( - null, - new TsFileInsertionEventParserProvider( - tsFile, treePattern, tablePattern, startTime, endTime, pipeTaskMeta, this) - .provide()); - return eventParser.get(); + if (eventParser == null) { + eventParser = + new TsFileInsertionEventParserProvider( + tsFile, treePattern, tablePattern, startTime, endTime, pipeTaskMeta, this) + .provide(); + } + return eventParser; } catch (final IOException e) { close(); @@ -615,13 +613,10 @@ public long count(final boolean skipReportOnCommit) throws IOException { /** Release the resource of {@link TsFileInsertionEventParser}. */ @Override public void close() { - eventParser.getAndUpdate( - parser -> { - if (Objects.nonNull(parser)) { - parser.close(); - } - return null; - }); + if (eventParser != null) { + eventParser.close(); + eventParser = null; + } } /////////////////////////// Object /////////////////////////// @@ -659,8 +654,7 @@ public PipeEventResource eventResourceBuilder() { this.tsFile, this.isWithMod, this.modFile, - this.sharedModFile, - this.eventParser); + this.sharedModFile); } private static class PipeTsFileInsertionEventResource extends PipeEventResource { @@ -668,8 +662,7 @@ private static class PipeTsFileInsertionEventResource extends PipeEventResource private final File tsFile; private final boolean isWithMod; private final File modFile; - private final File sharedModFile; // unused now - private final AtomicReference eventParser; + private final File sharedModFile; private PipeTsFileInsertionEventResource( final AtomicBoolean isReleased, @@ -677,33 +670,21 @@ private PipeTsFileInsertionEventResource( final File tsFile, final boolean isWithMod, final File modFile, - final File sharedModFile, - final AtomicReference eventParser) { + File sharedModFile) { super(isReleased, referenceCount); this.tsFile = tsFile; this.isWithMod = isWithMod; this.modFile = modFile; this.sharedModFile = sharedModFile; - this.eventParser = eventParser; } @Override protected void finalizeResource() { try { - // decrease reference count PipeDataNodeResourceManager.tsfile().decreaseFileReference(tsFile); if (isWithMod) { PipeDataNodeResourceManager.tsfile().decreaseFileReference(modFile); } - - // close event parser - eventParser.getAndUpdate( - parser -> { - if (Objects.nonNull(parser)) { - parser.close(); - } - return null; - }); } catch (final Exception e) { LOGGER.warn( String.format("Decrease reference count for TsFile %s error.", tsFile.getPath()), e); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/schemaregion/SchemaRegionListeningFilter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/schemaregion/SchemaRegionListeningFilter.java index 86e985f330c2..cb64d208b382 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/schemaregion/SchemaRegionListeningFilter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/schemaregion/SchemaRegionListeningFilter.java @@ -41,8 +41,8 @@ import static org.apache.iotdb.commons.pipe.datastructure.options.PipeInclusionOptions.getExclusionString; import static org.apache.iotdb.commons.pipe.datastructure.options.PipeInclusionOptions.getInclusionString; import static org.apache.iotdb.commons.pipe.datastructure.options.PipeInclusionOptions.parseOptions; -import static org.apache.iotdb.commons.pipe.datastructure.options.PipeInclusionOptions.tableOnlySyncPrefixes; -import static org.apache.iotdb.commons.pipe.datastructure.options.PipeInclusionOptions.treeOnlySyncPrefixes; +import static org.apache.iotdb.commons.pipe.datastructure.options.PipeInclusionOptions.tableOnlySyncPrefix; +import static org.apache.iotdb.commons.pipe.datastructure.options.PipeInclusionOptions.treeOnlySyncPrefix; /** * {@link SchemaRegionListeningFilter} is to classify the {@link PlanNode}s to help {@link @@ -118,11 +118,11 @@ public static Set parseListeningPlanTypeSet(final PipeParameters p exclusionOptions.forEach(exclusion -> planTypes.removeAll(getOptionsByPrefix(exclusion))); if (!TreePattern.isTreeModelDataAllowToBeCaptured(parameters)) { - treeOnlySyncPrefixes.forEach(prefix -> planTypes.removeAll(getOptionsByPrefix(prefix))); + planTypes.removeAll(getOptionsByPrefix(treeOnlySyncPrefix)); } if (!TablePattern.isTableModelDataAllowToBeCaptured(parameters)) { - tableOnlySyncPrefixes.forEach(prefix -> planTypes.removeAll(getOptionsByPrefix(prefix))); + planTypes.removeAll(getOptionsByPrefix(tableOnlySyncPrefix)); } return planTypes; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java index 121f37bf2ea3..c5db184030db 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java @@ -90,7 +90,6 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.SeriesScanOptions; import org.apache.iotdb.db.queryengine.plan.relational.metadata.Metadata; import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache.TreeDeviceSchemaCacheManager; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SetSqlDialect; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Use; import org.apache.iotdb.db.queryengine.plan.relational.sql.parser.ParsingException; import org.apache.iotdb.db.queryengine.plan.relational.sql.parser.SqlParser; @@ -114,7 +113,6 @@ import org.apache.iotdb.db.queryengine.plan.statement.metadata.template.DropSchemaTemplateStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.template.SetSchemaTemplateStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.template.UnsetSchemaTemplateStatement; -import org.apache.iotdb.db.queryengine.plan.statement.sys.SetSqlDialectStatement; import org.apache.iotdb.db.schemaengine.template.TemplateQueryType; import org.apache.iotdb.db.storageengine.StorageEngine; import org.apache.iotdb.db.storageengine.dataregion.DataRegion; @@ -314,17 +312,12 @@ private TSExecuteStatementResp executeStatementInternal( StatementType statementType = null; Throwable t = null; boolean useDatabase = false; - boolean setSqlDialect = false; try { // create and cache dataset ExecutionResult result; if (clientSession.getSqlDialect() == IClientSession.SqlDialect.TREE) { Statement s = StatementGenerator.createStatement(statement, clientSession.getZoneId()); - if (s instanceof SetSqlDialectStatement) { - setSqlDialect = true; - } - if (s == null) { return RpcUtils.getTSExecuteStatementResp( RpcUtils.getStatus( @@ -364,10 +357,6 @@ private TSExecuteStatementResp executeStatementInternal( useDatabase = true; } - if (s instanceof SetSqlDialect) { - setSqlDialect = true; - } - if (s == null) { return RpcUtils.getTSExecuteStatementResp( RpcUtils.getStatus( @@ -415,12 +404,6 @@ private TSExecuteStatementResp executeStatementInternal( if (useDatabase) { resp.setDatabase(clientSession.getDatabaseName()); } - - if (setSqlDialect) { - resp.setTableModel( - SESSION_MANAGER.getCurrSessionAndUpdateIdleTime().getSqlDialect() - == IClientSession.SqlDialect.TABLE); - } } return resp; } @@ -2247,12 +2230,10 @@ public TSStatus insertTablet(TSInsertTabletReq req) { return RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS); } - // permission check for tree model, table model does this in the analysis stage - if (!req.isWriteToTable()) { - TSStatus status = AuthorityChecker.checkAuthority(statement, clientSession); - if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - return status; - } + // permission check + TSStatus status = AuthorityChecker.checkAuthority(statement, clientSession); + if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + return status; } quota = diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/SharedTsBlockQueue.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/SharedTsBlockQueue.java index f38104263980..b3b94f66282f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/SharedTsBlockQueue.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/SharedTsBlockQueue.java @@ -196,8 +196,8 @@ public TsBlock remove() { localFragmentInstanceId.getQueryId(), fullFragmentInstanceId, localPlanNodeId, - tsBlock.getSizeInBytes()); - bufferRetainedSizeInBytes -= tsBlock.getSizeInBytes(); + tsBlock.getRetainedSizeInBytes()); + bufferRetainedSizeInBytes -= tsBlock.getRetainedSizeInBytes(); // Every time LocalSourceHandle consumes a TsBlock, it needs to send the event to // corresponding LocalSinkChannel. if (sinkChannel != null) { @@ -236,10 +236,10 @@ public ListenableFuture add(TsBlock tsBlock) { localFragmentInstanceId.getQueryId(), fullFragmentInstanceId, localPlanNodeId, - tsBlock.getSizeInBytes(), + tsBlock.getRetainedSizeInBytes(), maxBytesCanReserve); blockedOnMemory = pair.left; - bufferRetainedSizeInBytes += tsBlock.getSizeInBytes(); + bufferRetainedSizeInBytes += tsBlock.getRetainedSizeInBytes(); // reserve memory failed, we should wait until there is enough memory if (!Boolean.TRUE.equals(pair.right)) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/sink/SinkChannel.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/sink/SinkChannel.java index 8915938eb03c..9f29a8a753fc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/sink/SinkChannel.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/sink/SinkChannel.java @@ -201,7 +201,7 @@ public synchronized void send(TsBlock tsBlock) { if (noMoreTsBlocks) { return; } - long sizeInBytes = tsBlock.getSizeInBytes(); + long retainedSizeInBytes = tsBlock.getRetainedSizeInBytes(); int startSequenceId; startSequenceId = nextSequenceId; blocked = @@ -211,16 +211,17 @@ public synchronized void send(TsBlock tsBlock) { localFragmentInstanceId.getQueryId(), fullFragmentInstanceId, localPlanNodeId, - sizeInBytes, + retainedSizeInBytes, maxBytesCanReserve) .left; - bufferRetainedSizeInBytes += sizeInBytes; + bufferRetainedSizeInBytes += retainedSizeInBytes; sequenceIdToTsBlock.put(nextSequenceId, new Pair<>(tsBlock, currentTsBlockSize)); nextSequenceId += 1; - currentTsBlockSize = sizeInBytes; + currentTsBlockSize = retainedSizeInBytes; - submitSendNewDataBlockEventTask(startSequenceId, ImmutableList.of(sizeInBytes)); + // TODO: consider merge multiple NewDataBlockEvent for less network traffic. + submitSendNewDataBlockEventTask(startSequenceId, ImmutableList.of(retainedSizeInBytes)); } finally { DATA_EXCHANGE_COST_METRIC_SET.recordDataExchangeCost( SINK_HANDLE_SEND_TSBLOCK_REMOTE, System.nanoTime() - startTime); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/source/LocalSourceHandle.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/source/LocalSourceHandle.java index 63feb571cb09..4da89f72d07a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/source/LocalSourceHandle.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/source/LocalSourceHandle.java @@ -124,7 +124,9 @@ public TsBlock receive() { if (tsBlock != null) { if (LOGGER.isDebugEnabled()) { LOGGER.debug( - "[GetTsBlockFromQueue] TsBlock:{} size:{}", currSequenceId, tsBlock.getSizeInBytes()); + "[GetTsBlockFromQueue] TsBlock:{} size:{}", + currSequenceId, + tsBlock.getRetainedSizeInBytes()); } currSequenceId++; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/executor/RegionWriteExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/executor/RegionWriteExecutor.java index e9fac5c1d32a..079238309c75 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/executor/RegionWriteExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/executor/RegionWriteExecutor.java @@ -85,7 +85,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.concurrent.locks.ReentrantReadWriteLock; @@ -486,11 +485,11 @@ private RegionExecutionResult executeCreateMultiTimeSeries( measurementGroupMap.remove(emptyDevice); } - final RegionExecutionResult executionResult = + final RegionExecutionResult failingResult = registerTimeSeries(measurementGroupMap, node, context, failingStatus); - if (executionResult != null) { - return executionResult; + if (failingResult != null) { + return failingResult; } final TSStatus status = RpcUtils.getStatus(failingStatus); @@ -617,12 +616,7 @@ private RegionExecutionResult executeInternalCreateTimeSeries( measurementGroup.removeMeasurements(failingMeasurementMap.keySet()); return processExecutionResultOfInternalCreateSchema( - !measurementGroup.isEmpty() - ? super.visitInternalCreateTimeSeries(node, context) - : RegionExecutionResult.create( - true, - "Execute successfully", - RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS, "Execute successfully")), + super.visitInternalCreateTimeSeries(node, context), failingStatus, alreadyExistingStatus); } finally { @@ -667,13 +661,8 @@ private RegionExecutionResult executeInternalCreateMultiTimeSeries( MeasurementGroup measurementGroup; Map failingMeasurementMap; MetadataException metadataException; - - final Iterator>> iterator = - node.getDeviceMap().entrySet().iterator(); - - while (iterator.hasNext()) { - final Map.Entry> deviceEntry = - iterator.next(); + for (final Map.Entry> deviceEntry : + node.getDeviceMap().entrySet()) { measurementGroup = deviceEntry.getValue().right; failingMeasurementMap = schemaRegion.checkMeasurementExistence( @@ -701,18 +690,10 @@ private RegionExecutionResult executeInternalCreateMultiTimeSeries( } } measurementGroup.removeMeasurements(failingMeasurementMap.keySet()); - if (measurementGroup.isEmpty()) { - iterator.remove(); - } } return processExecutionResultOfInternalCreateSchema( - !node.getDeviceMap().isEmpty() - ? super.visitInternalCreateMultiTimeSeries(node, context) - : RegionExecutionResult.create( - true, - "Execute successfully", - RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS, "Execute successfully")), + super.visitInternalCreateMultiTimeSeries(node, context), failingStatus, alreadyExistingStatus); } finally { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java index 14629fa37fa1..6d035e1dc3c6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java @@ -29,7 +29,6 @@ import org.apache.iotdb.db.queryengine.common.FragmentInstanceId; import org.apache.iotdb.db.queryengine.common.QueryId; import org.apache.iotdb.db.queryengine.common.SessionInfo; -import org.apache.iotdb.db.queryengine.metric.DriverSchedulerMetricSet; import org.apache.iotdb.db.queryengine.metric.QueryRelatedResourceMetricSet; import org.apache.iotdb.db.queryengine.metric.SeriesScanCostMetricSet; import org.apache.iotdb.db.queryengine.plan.planner.memory.MemoryReservationManager; @@ -58,14 +57,10 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.CountDownLatch; -import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; -import static org.apache.iotdb.db.queryengine.metric.DriverSchedulerMetricSet.BLOCK_QUEUED_TIME; -import static org.apache.iotdb.db.queryengine.metric.DriverSchedulerMetricSet.READY_QUEUED_TIME; - public class FragmentInstanceContext extends QueryContext { private static final Logger LOGGER = LoggerFactory.getLogger(FragmentInstanceContext.class); @@ -376,32 +371,19 @@ public void setDataNodeQueryContext(DataNodeQueryContext dataNodeQueryContext) { } public FragmentInstanceInfo getInstanceInfo() { - FragmentInstanceState state = stateMachine.getState(); - long endTime = getEndTime(); - LinkedBlockingQueue failures = stateMachine.getFailureCauses(); - String failureCause = ""; - List failureInfoList = new ArrayList<>(); - TSStatus status = null; - - for (Throwable failure : failures) { - if (failureCause.isEmpty()) { - failureCause = failure.getMessage(); - } - failureInfoList.add(FragmentInstanceFailureInfo.toFragmentInstanceFailureInfo(failure)); - if (failure instanceof IoTDBException) { - status = new TSStatus(((IoTDBException) failure).getErrorCode()); - status.setMessage(failure.getMessage()); - } else if (failure instanceof IoTDBRuntimeException) { - status = new TSStatus(((IoTDBRuntimeException) failure).getErrorCode()); - status.setMessage(failure.getMessage()); - } - } - - if (status == null) { - return new FragmentInstanceInfo(state, endTime, failureCause, failureInfoList); - } else { - return new FragmentInstanceInfo(state, endTime, failureCause, failureInfoList, status); - } + return getErrorCode() + .map( + s -> + new FragmentInstanceInfo( + stateMachine.getState(), + getEndTime(), + getFailedCause(), + getFailureInfoList(), + s)) + .orElseGet( + () -> + new FragmentInstanceInfo( + stateMachine.getState(), getEndTime(), getFailedCause(), getFailureInfoList())); } public FragmentInstanceStateMachine getStateMachine() { @@ -693,11 +675,6 @@ public synchronized void releaseResource() { // record fragment instance execution time and metadata get time to metrics long durationTime = System.currentTimeMillis() - executionStartTime.get(); - DriverSchedulerMetricSet.getInstance() - .recordTaskQueueTime(BLOCK_QUEUED_TIME, blockQueueTime.get()); - DriverSchedulerMetricSet.getInstance() - .recordTaskQueueTime(READY_QUEUED_TIME, readyQueueTime.get()); - QueryRelatedResourceMetricSet.getInstance().updateFragmentInstanceTime(durationTime); SeriesScanCostMetricSet.getInstance() diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/AbstractOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/AbstractOperator.java index 5c6d731f9a04..1278dc15d8dc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/AbstractOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/AbstractOperator.java @@ -44,7 +44,7 @@ public void initializeMaxTsBlockLength(TsBlock tsBlock) { long oneTupleSize = Math.max( 1, - (tsBlock.getSizeInBytes() - tsBlock.getTotalInstanceSize()) + (tsBlock.getRetainedSizeInBytes() - tsBlock.getTotalInstanceSize()) / tsBlock.getPositionCount()); if (oneTupleSize > maxReturnSize) { // make sure at least one-tuple-at-a-time diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/AbstractSortOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/AbstractSortOperator.java index 0bfa1d3e7673..c017cded6b74 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/AbstractSortOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/AbstractSortOperator.java @@ -162,7 +162,7 @@ private void prepareSortReaders() throws IoTDBException { } protected void cacheTsBlock(TsBlock tsBlock) throws IoTDBException { - long bytesSize = tsBlock.getSizeInBytes(); + long bytesSize = tsBlock.getRetainedSizeInBytes(); if (bytesSize + cachedBytes < sortBufferManager.getSortBufferSize()) { cachedBytes += bytesSize; for (int i = 0; i < tsBlock.getPositionCount(); i++) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/SortOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/SortOperator.java index 31029a30da64..eb556e1cfac0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/SortOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/SortOperator.java @@ -62,7 +62,7 @@ public TsBlock next() throws Exception { if (tsBlock == null) { return null; } - dataSize += tsBlock.getSizeInBytes(); + dataSize += tsBlock.getRetainedSizeInBytes(); cacheTsBlock(tsBlock); } catch (IoTDBException e) { clear(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/TableStreamSortOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/TableStreamSortOperator.java index 90565a858759..71e938f4f309 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/TableStreamSortOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/TableStreamSortOperator.java @@ -119,7 +119,7 @@ public TsBlock next() throws Exception { return null; } // record total sorted data size - dataSize += currentTsBlock.getSizeInBytes(); + dataSize += currentTsBlock.getRetainedSizeInBytes(); // if currentTsBlock line count + remainingCount is still less than minLinesToOutput, just // cache it diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java index 213e4990a5f4..cd00aa5e4196 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java @@ -41,7 +41,7 @@ import org.apache.tsfile.file.metadata.IChunkMetadata; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.ITimeSeriesMetadata; -import org.apache.tsfile.file.metadata.TableDeviceTimeSeriesMetadata; +import org.apache.tsfile.file.metadata.TableDeviceMetadata; import org.apache.tsfile.file.metadata.TimeseriesMetadata; import org.apache.tsfile.read.controller.IChunkLoader; import org.apache.tsfile.read.filter.basic.Filter; @@ -395,7 +395,7 @@ private static AbstractAlignedTimeSeriesMetadata setModifications( AbstractAlignedTimeSeriesMetadata alignedTimeSeriesMetadata = ignoreAllNullRows ? new AlignedTimeSeriesMetadata(timeColumnMetadata, valueColumnMetadataList) - : new TableDeviceTimeSeriesMetadata(timeColumnMetadata, valueColumnMetadataList); + : new TableDeviceMetadata(timeColumnMetadata, valueColumnMetadataList); alignedTimeSeriesMetadata.setChunkMetadataLoader( new DiskAlignedChunkMetadataLoader( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/InformationSchemaContentSupplierFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/InformationSchemaContentSupplierFactory.java index d199fb4c9b84..a798a1879c15 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/InformationSchemaContentSupplierFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/InformationSchemaContentSupplierFactory.java @@ -19,10 +19,8 @@ package org.apache.iotdb.db.queryengine.execution.operator.source.relational; -import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType; import org.apache.iotdb.commons.conf.IoTDBConstant; -import org.apache.iotdb.commons.pipe.agent.plugin.builtin.BuiltinPipePlugin; -import org.apache.iotdb.commons.pipe.agent.plugin.meta.PipePluginMeta; +import org.apache.iotdb.commons.exception.IoTDBException; import org.apache.iotdb.commons.schema.table.InformationSchema; import org.apache.iotdb.commons.schema.table.TableNodeStatus; import org.apache.iotdb.commons.schema.table.TsTable; @@ -31,16 +29,8 @@ import org.apache.iotdb.confignode.rpc.thrift.TDatabaseInfo; import org.apache.iotdb.confignode.rpc.thrift.TDescTable4InformationSchemaResp; import org.apache.iotdb.confignode.rpc.thrift.TGetDatabaseReq; -import org.apache.iotdb.confignode.rpc.thrift.TRegionInfo; -import org.apache.iotdb.confignode.rpc.thrift.TShowPipeInfo; -import org.apache.iotdb.confignode.rpc.thrift.TShowPipeReq; -import org.apache.iotdb.confignode.rpc.thrift.TShowRegionReq; -import org.apache.iotdb.confignode.rpc.thrift.TShowSubscriptionInfo; -import org.apache.iotdb.confignode.rpc.thrift.TShowSubscriptionReq; -import org.apache.iotdb.confignode.rpc.thrift.TShowTopicInfo; -import org.apache.iotdb.confignode.rpc.thrift.TShowTopicReq; +import org.apache.iotdb.confignode.rpc.thrift.TShowDatabaseResp; import org.apache.iotdb.confignode.rpc.thrift.TTableInfo; -import org.apache.iotdb.db.pipe.metric.PipeDataNodeRemainingEventAndTimeMetrics; import org.apache.iotdb.db.protocol.client.ConfigNodeClient; import org.apache.iotdb.db.protocol.client.ConfigNodeClientManager; import org.apache.iotdb.db.protocol.client.ConfigNodeInfo; @@ -48,7 +38,6 @@ import org.apache.iotdb.db.queryengine.plan.Coordinator; import org.apache.iotdb.db.queryengine.plan.execution.IQueryExecution; import org.apache.iotdb.db.schemaengine.table.InformationSchemaUtils; -import org.apache.iotdb.db.utils.TimestampPrecisionUtils; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.common.conf.TSFileConfig; @@ -68,40 +57,27 @@ import java.util.NoSuchElementException; import java.util.Objects; import java.util.Set; -import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import static org.apache.iotdb.commons.conf.IoTDBConstant.TTL_INFINITE; import static org.apache.iotdb.commons.schema.SchemaConstant.ALL_MATCH_SCOPE; import static org.apache.iotdb.commons.schema.SchemaConstant.ALL_RESULT_NODES; import static org.apache.iotdb.commons.schema.table.TsTable.TTL_PROPERTY; -import static org.apache.iotdb.db.queryengine.plan.execution.config.metadata.ShowPipePluginsTask.PIPE_PLUGIN_TYPE_BUILTIN; -import static org.apache.iotdb.db.queryengine.plan.execution.config.metadata.ShowPipePluginsTask.PIPE_PLUGIN_TYPE_EXTERNAL; public class InformationSchemaContentSupplierFactory { private InformationSchemaContentSupplierFactory() {} public static Iterator getSupplier( - final String tableName, final List dataTypes) { + final String tableName, final List dataTypes, final String userName) { switch (tableName) { case InformationSchema.QUERIES: return new QueriesSupplier(dataTypes); case InformationSchema.DATABASES: - return new DatabaseSupplier(dataTypes); + return new DatabaseSupplier(dataTypes, userName); case InformationSchema.TABLES: - return new TableSupplier(dataTypes); + return new TableSupplier(dataTypes, userName); case InformationSchema.COLUMNS: - return new ColumnSupplier(dataTypes); - case InformationSchema.REGIONS: - return new RegionSupplier(dataTypes); - case InformationSchema.PIPES: - return new PipeSupplier(dataTypes); - case InformationSchema.PIPE_PLUGINS: - return new PipePluginSupplier(dataTypes); - case InformationSchema.TOPICS: - return new TopicSupplier(dataTypes); - case InformationSchema.SUBSCRIPTIONS: - return new SubscriptionSupplier(dataTypes); + return new ColumnSupplier(dataTypes, userName); default: throw new UnsupportedOperationException("Unknown table: " + tableName); } @@ -110,32 +86,31 @@ public static Iterator getSupplier( private static class QueriesSupplier extends TsBlockSupplier { private final long currTime = System.currentTimeMillis(); // We initialize it later for the convenience of data preparation + protected int totalSize; protected int nextConsumedIndex; - private final List queryExecutions; + private final List queryExecutions = + Coordinator.getInstance().getAllQueryExecutions(); private QueriesSupplier(final List dataTypes) { super(dataTypes); - queryExecutions = Coordinator.getInstance().getAllQueryExecutions(); + this.totalSize = queryExecutions.size(); } @Override protected void constructLine() { - final IQueryExecution queryExecution = queryExecutions.get(nextConsumedIndex); + IQueryExecution queryExecution = queryExecutions.get(nextConsumedIndex); if (queryExecution.getSQLDialect().equals(IClientSession.SqlDialect.TABLE)) { - final String[] splits = queryExecution.getQueryId().split("_"); - final int dataNodeId = Integer.parseInt(splits[splits.length - 1]); + String[] splits = queryExecution.getQueryId().split("_"); + int dataNodeId = Integer.parseInt(splits[splits.length - 1]); columnBuilders[0].writeBinary(BytesUtils.valueOf(queryExecution.getQueryId())); - columnBuilders[1].writeLong( - TimestampPrecisionUtils.convertToCurrPrecision( - queryExecution.getStartExecutionTime(), TimeUnit.MILLISECONDS)); + columnBuilders[1].writeLong(queryExecution.getStartExecutionTime()); columnBuilders[2].writeInt(dataNodeId); columnBuilders[3].writeFloat( (float) (currTime - queryExecution.getStartExecutionTime()) / 1000); columnBuilders[4].writeBinary( BytesUtils.valueOf(queryExecution.getExecuteSQL().orElse("UNKNOWN"))); - columnBuilders[5].writeBinary(BytesUtils.valueOf(queryExecution.getUser())); resultBuilder.declarePosition(); } nextConsumedIndex++; @@ -143,28 +118,26 @@ protected void constructLine() { @Override public boolean hasNext() { - return nextConsumedIndex < queryExecutions.size(); + return nextConsumedIndex < totalSize; } } private static class DatabaseSupplier extends TsBlockSupplier { private Iterator> iterator; private TDatabaseInfo currentDatabase; + private final String userName; private boolean hasShownInformationSchema; - private DatabaseSupplier(final List dataTypes) { + private DatabaseSupplier(final List dataTypes, final String userName) { super(dataTypes); + this.userName = userName; try (final ConfigNodeClient client = ConfigNodeClientManager.getInstance().borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { - iterator = - client - .showDatabase( - new TGetDatabaseReq( - Arrays.asList(ALL_RESULT_NODES), ALL_MATCH_SCOPE.serialize()) - .setIsTableModel(true)) - .getDatabaseInfoMap() - .entrySet() - .iterator(); + final TShowDatabaseResp resp = + client.showDatabase( + new TGetDatabaseReq(Arrays.asList(ALL_RESULT_NODES), ALL_MATCH_SCOPE.serialize()) + .setIsTableModel(true)); + iterator = resp.getDatabaseInfoMap().entrySet().iterator(); } catch (final Exception e) { lastException = e; } @@ -199,10 +172,19 @@ protected void constructLine() { @Override public boolean hasNext() { if (!hasShownInformationSchema) { - return true; + if (!canShowDB(userName, InformationSchema.INFORMATION_DATABASE)) { + hasShownInformationSchema = true; + } else { + return true; + } } - if (iterator.hasNext()) { - currentDatabase = iterator.next().getValue(); + while (iterator.hasNext()) { + final Map.Entry result = iterator.next(); + if (!canShowDB(userName, result.getKey())) { + continue; + } + currentDatabase = result.getValue(); + break; } return Objects.nonNull(currentDatabase); } @@ -211,11 +193,12 @@ public boolean hasNext() { private static class TableSupplier extends TsBlockSupplier { private Iterator>> dbIterator; private Iterator tableInfoIterator = null; - private TTableInfo currentTable; private String dbName; + private final String userName; - private TableSupplier(final List dataTypes) { + private TableSupplier(final List dataTypes, final String userName) { super(dataTypes); + this.userName = userName; try (final ConfigNodeClient client = ConfigNodeClientManager.getInstance().borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { final Map> databaseTableInfoMap = @@ -241,31 +224,28 @@ private TableSupplier(final List dataTypes) { @Override protected void constructLine() { + final TTableInfo info = tableInfoIterator.next(); columnBuilders[0].writeBinary(new Binary(dbName, TSFileConfig.STRING_CHARSET)); - columnBuilders[1].writeBinary( - new Binary(currentTable.getTableName(), TSFileConfig.STRING_CHARSET)); - columnBuilders[2].writeBinary(new Binary(currentTable.getTTL(), TSFileConfig.STRING_CHARSET)); + columnBuilders[1].writeBinary(new Binary(info.getTableName(), TSFileConfig.STRING_CHARSET)); + columnBuilders[2].writeBinary(new Binary(info.getTTL(), TSFileConfig.STRING_CHARSET)); columnBuilders[3].writeBinary( new Binary( - TableNodeStatus.values()[currentTable.getState()].toString(), - TSFileConfig.STRING_CHARSET)); + TableNodeStatus.values()[info.getState()].toString(), TSFileConfig.STRING_CHARSET)); resultBuilder.declarePosition(); - currentTable = null; } @Override public boolean hasNext() { // Get next table info iterator - while (Objects.isNull(currentTable)) { - if (Objects.nonNull(tableInfoIterator) && tableInfoIterator.hasNext()) { - currentTable = tableInfoIterator.next(); - return true; - } + while (Objects.isNull(tableInfoIterator) || !tableInfoIterator.hasNext()) { if (!dbIterator.hasNext()) { return false; } final Map.Entry> entry = dbIterator.next(); dbName = entry.getKey(); + if (!canShowDB(userName, dbName)) { + continue; + } tableInfoIterator = entry.getValue().iterator(); } return true; @@ -279,9 +259,11 @@ private static class ColumnSupplier extends TsBlockSupplier { private String dbName; private String tableName; private Set preDeletedColumns; + private final String userName; - private ColumnSupplier(final List dataTypes) { + private ColumnSupplier(final List dataTypes, final String userName) { super(dataTypes); + this.userName = userName; try (final ConfigNodeClient client = ConfigNodeClientManager.getInstance().borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { final TDescTable4InformationSchemaResp resp = client.descTables4InformationSchema(); @@ -341,232 +323,31 @@ public boolean hasNext() { final Map.Entry>>> entry = dbIterator.next(); dbName = entry.getKey(); + if (!canShowDB(userName, dbName)) { + continue; + } tableInfoIterator = entry.getValue().entrySet().iterator(); } - Map.Entry>> tableEntry = tableInfoIterator.next(); + final Map.Entry>> tableEntry = tableInfoIterator.next(); tableName = tableEntry.getKey(); preDeletedColumns = tableEntry.getValue().getRight(); columnSchemaIterator = tableEntry.getValue().getLeft().getColumnList().iterator(); - break; } return true; } } - private static class RegionSupplier extends TsBlockSupplier { - private Iterator iterator; - - private RegionSupplier(final List dataTypes) { - super(dataTypes); - try (final ConfigNodeClient client = - ConfigNodeClientManager.getInstance().borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { - iterator = - client - .showRegion(new TShowRegionReq().setIsTableModel(true).setDatabases(null)) - .getRegionInfoListIterator(); - } catch (final Exception e) { - lastException = e; - } - } - - @Override - protected void constructLine() { - final TRegionInfo regionInfo = iterator.next(); - columnBuilders[0].writeInt(regionInfo.getConsensusGroupId().getId()); - columnBuilders[1].writeInt(regionInfo.getDataNodeId()); - if (regionInfo.getConsensusGroupId().getType().ordinal() - == TConsensusGroupType.SchemaRegion.ordinal()) { - columnBuilders[2].writeBinary( - BytesUtils.valueOf(String.valueOf(TConsensusGroupType.SchemaRegion))); - } else if (regionInfo.getConsensusGroupId().getType().ordinal() - == TConsensusGroupType.DataRegion.ordinal()) { - columnBuilders[2].writeBinary( - BytesUtils.valueOf(String.valueOf(TConsensusGroupType.DataRegion))); - } - columnBuilders[3].writeBinary( - BytesUtils.valueOf(regionInfo.getStatus() == null ? "" : regionInfo.getStatus())); - columnBuilders[4].writeBinary(BytesUtils.valueOf(regionInfo.getDatabase())); - columnBuilders[5].writeInt(regionInfo.getSeriesSlots()); - columnBuilders[6].writeLong(regionInfo.getTimeSlots()); - columnBuilders[7].writeBinary(BytesUtils.valueOf(regionInfo.getClientRpcIp())); - columnBuilders[8].writeInt(regionInfo.getClientRpcPort()); - columnBuilders[9].writeBinary(BytesUtils.valueOf(regionInfo.getInternalAddress())); - columnBuilders[10].writeBinary(BytesUtils.valueOf(regionInfo.getRoleType())); - columnBuilders[11].writeLong(regionInfo.getCreateTime()); - if (regionInfo.getConsensusGroupId().getType().ordinal() - == TConsensusGroupType.DataRegion.ordinal()) { - columnBuilders[12].writeLong(regionInfo.getTsFileSize()); - } else { - columnBuilders[12].appendNull(); - } - resultBuilder.declarePosition(); - } - - @Override - public boolean hasNext() { - return iterator.hasNext(); - } - } - - private static class PipeSupplier extends TsBlockSupplier { - private Iterator iterator; - - private PipeSupplier(final List dataTypes) { - super(dataTypes); - try (final ConfigNodeClient client = - ConfigNodeClientManager.getInstance().borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { - iterator = - client.showPipe(new TShowPipeReq().setIsTableModel(true)).getPipeInfoListIterator(); - } catch (final Exception e) { - lastException = e; - } - } - - @Override - protected void constructLine() { - final TShowPipeInfo tPipeInfo = iterator.next(); - columnBuilders[0].writeBinary(new Binary(tPipeInfo.getId(), TSFileConfig.STRING_CHARSET)); - columnBuilders[1].writeLong( - TimestampPrecisionUtils.convertToCurrPrecision( - tPipeInfo.getCreationTime(), TimeUnit.MILLISECONDS)); - columnBuilders[2].writeBinary(new Binary(tPipeInfo.getState(), TSFileConfig.STRING_CHARSET)); - columnBuilders[3].writeBinary( - new Binary(tPipeInfo.getPipeExtractor(), TSFileConfig.STRING_CHARSET)); - columnBuilders[4].writeBinary( - new Binary(tPipeInfo.getPipeProcessor(), TSFileConfig.STRING_CHARSET)); - columnBuilders[5].writeBinary( - new Binary(tPipeInfo.getPipeConnector(), TSFileConfig.STRING_CHARSET)); - columnBuilders[6].writeBinary( - new Binary(tPipeInfo.getExceptionMessage(), TSFileConfig.STRING_CHARSET)); - - // Optional, default 0/0.0 - long remainingEventCount = tPipeInfo.getRemainingEventCount(); - double remainingTime = tPipeInfo.getEstimatedRemainingTime(); - - if (remainingEventCount == -1 && remainingTime == -1) { - final Pair remainingEventAndTime = - PipeDataNodeRemainingEventAndTimeMetrics.getInstance() - .getRemainingEventAndTime(tPipeInfo.getId(), tPipeInfo.getCreationTime()); - remainingEventCount = remainingEventAndTime.getLeft(); - remainingTime = remainingEventAndTime.getRight(); - } - - columnBuilders[7].writeLong(tPipeInfo.isSetRemainingEventCount() ? remainingEventCount : -1); - columnBuilders[8].writeDouble(tPipeInfo.isSetEstimatedRemainingTime() ? remainingTime : -1); - - resultBuilder.declarePosition(); - } - - @Override - public boolean hasNext() { - return iterator.hasNext(); - } - } - - private static class PipePluginSupplier extends TsBlockSupplier { - private Iterator iterator; - - private PipePluginSupplier(final List dataTypes) { - super(dataTypes); - try (final ConfigNodeClient client = - ConfigNodeClientManager.getInstance().borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { - iterator = - client.getPipePluginTable().getAllPipePluginMeta().stream() - .map(PipePluginMeta::deserialize) - .filter( - pipePluginMeta -> - !BuiltinPipePlugin.SHOW_PIPE_PLUGINS_BLACKLIST.contains( - pipePluginMeta.getPluginName())) - .iterator(); - } catch (final Exception e) { - lastException = e; - } - } - - @Override - protected void constructLine() { - final PipePluginMeta pipePluginMeta = iterator.next(); - columnBuilders[0].writeBinary(BytesUtils.valueOf(pipePluginMeta.getPluginName())); - columnBuilders[1].writeBinary( - pipePluginMeta.isBuiltin() ? PIPE_PLUGIN_TYPE_BUILTIN : PIPE_PLUGIN_TYPE_EXTERNAL); - columnBuilders[2].writeBinary(BytesUtils.valueOf(pipePluginMeta.getClassName())); - if (Objects.nonNull(pipePluginMeta.getJarName())) { - columnBuilders[3].writeBinary(BytesUtils.valueOf(pipePluginMeta.getJarName())); - } else { - columnBuilders[3].appendNull(); - } - resultBuilder.declarePosition(); - } - - @Override - public boolean hasNext() { - return iterator.hasNext(); - } - } - - private static class TopicSupplier extends TsBlockSupplier { - private Iterator iterator; - - private TopicSupplier(final List dataTypes) { - super(dataTypes); - try (final ConfigNodeClient client = - ConfigNodeClientManager.getInstance().borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { - iterator = client.showTopic(new TShowTopicReq()).getTopicInfoList().iterator(); - } catch (final Exception e) { - lastException = e; - } - } - - @Override - protected void constructLine() { - final TShowTopicInfo topicInfo = iterator.next(); - columnBuilders[0].writeBinary( - new Binary(topicInfo.getTopicName(), TSFileConfig.STRING_CHARSET)); - columnBuilders[1].writeBinary( - new Binary(topicInfo.getTopicAttributes(), TSFileConfig.STRING_CHARSET)); - resultBuilder.declarePosition(); - } - - @Override - public boolean hasNext() { - return iterator.hasNext(); - } - } - - private static class SubscriptionSupplier extends TsBlockSupplier { - private Iterator iterator; - - private SubscriptionSupplier(final List dataTypes) { - super(dataTypes); - try (final ConfigNodeClient client = - ConfigNodeClientManager.getInstance().borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { - iterator = - client - .showSubscription(new TShowSubscriptionReq()) - .getSubscriptionInfoList() - .iterator(); - } catch (final Exception e) { - lastException = e; + private static boolean canShowDB(final String userName, final String dbName) { + try { + Coordinator.getInstance().getAccessControl().checkCanShowOrUseDatabase(userName, dbName); + } catch (final RuntimeException e) { + if (e.getCause() instanceof IoTDBException) { + return false; } + throw e; } - - @Override - protected void constructLine() { - final TShowSubscriptionInfo tSubscriptionInfo = iterator.next(); - columnBuilders[0].writeBinary( - new Binary(tSubscriptionInfo.getTopicName(), TSFileConfig.STRING_CHARSET)); - columnBuilders[1].writeBinary( - new Binary(tSubscriptionInfo.getConsumerGroupId(), TSFileConfig.STRING_CHARSET)); - columnBuilders[2].writeBinary( - new Binary(tSubscriptionInfo.getConsumerIds().toString(), TSFileConfig.STRING_CHARSET)); - resultBuilder.declarePosition(); - } - - @Override - public boolean hasNext() { - return iterator.hasNext(); - } + return true; } private abstract static class TsBlockSupplier implements Iterator { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/HashAggregationOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/HashAggregationOperator.java index ce7e91f84485..31ddcbe293ba 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/HashAggregationOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/HashAggregationOperator.java @@ -45,6 +45,8 @@ public class HashAggregationOperator extends AbstractOperator { private static final long INSTANCE_SIZE = RamUsageEstimator.shallowSizeOfInstance(HashAggregationOperator.class); + private final OperatorContext operatorContext; + private final Operator child; private final List groupByTypes; @@ -79,7 +81,7 @@ public HashAggregationOperator( long maxPartialMemory, boolean spillEnabled, long unspillMemoryLimit) { - super.operatorContext = operatorContext; + this.operatorContext = operatorContext; this.child = child; this.groupByTypes = ImmutableList.copyOf(groupByTypes); this.groupByChannels = ImmutableList.copyOf(groupByChannels); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/StreamingAggregationOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/StreamingAggregationOperator.java index 3ed2bbb1bded..7b0d37c365f3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/StreamingAggregationOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/StreamingAggregationOperator.java @@ -23,7 +23,7 @@ import org.apache.iotdb.db.queryengine.execution.operator.AbstractOperator; import org.apache.iotdb.db.queryengine.execution.operator.Operator; import org.apache.iotdb.db.queryengine.execution.operator.OperatorContext; -import org.apache.iotdb.db.queryengine.execution.operator.source.relational.AbstractTableScanOperator; +import org.apache.iotdb.db.queryengine.execution.operator.source.relational.TableScanOperator; import org.apache.iotdb.db.queryengine.execution.operator.source.relational.aggregation.TableAggregator; import org.apache.iotdb.db.queryengine.plan.relational.type.InternalTypeManager; import org.apache.iotdb.db.utils.datastructure.SortKey; @@ -52,6 +52,8 @@ public class StreamingAggregationOperator extends AbstractOperator { private static final long INSTANCE_SIZE = RamUsageEstimator.shallowSizeOfInstance(StreamingAggregationOperator.class); + private final OperatorContext operatorContext; + private final Operator child; private final List aggregators; @@ -83,7 +85,7 @@ public StreamingAggregationOperator( long maxPartialMemory, boolean spillEnabled, long unSpillMemoryLimit) { - super.operatorContext = operatorContext; + this.operatorContext = operatorContext; this.child = child; this.groupByChannels = Ints.toArray(groupByChannels); this.groupKeyComparator = groupKeyComparator; @@ -206,8 +208,7 @@ private void evaluateAndFlushGroup(TsBlock page, int position, boolean lastCalcu outputs.add( resultBuilder.build( new RunLengthEncodedColumn( - AbstractTableScanOperator.TIME_COLUMN_TEMPLATE, - resultBuilder.getPositionCount()))); + TableScanOperator.TIME_COLUMN_TEMPLATE, resultBuilder.getPositionCount()))); resultBuilder.reset(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/StreamingHashAggregationOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/StreamingHashAggregationOperator.java index 8862a8137186..374a6964de22 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/StreamingHashAggregationOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/StreamingHashAggregationOperator.java @@ -53,6 +53,8 @@ public class StreamingHashAggregationOperator extends AbstractOperator { private static final long INSTANCE_SIZE = RamUsageEstimator.shallowSizeOfInstance(StreamingHashAggregationOperator.class); + private final OperatorContext operatorContext; + private final Operator child; private final int[] preGroupedChannels; @@ -101,7 +103,7 @@ public StreamingHashAggregationOperator( long maxPartialMemory, boolean spillEnabled, long unSpillMemoryLimit) { - super.operatorContext = operatorContext; + this.operatorContext = operatorContext; this.child = child; this.preGroupedChannels = Ints.toArray(preGroupedChannels); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/relational/ColumnTransformerBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/relational/ColumnTransformerBuilder.java index badd487b5fe2..ce69b23c3bcf 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/relational/ColumnTransformerBuilder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/relational/ColumnTransformerBuilder.java @@ -86,7 +86,6 @@ import org.apache.iotdb.db.queryengine.transformation.dag.column.leaf.LeafColumnTransformer; import org.apache.iotdb.db.queryengine.transformation.dag.column.leaf.NullColumnTransformer; import org.apache.iotdb.db.queryengine.transformation.dag.column.leaf.TimeColumnTransformer; -import org.apache.iotdb.db.queryengine.transformation.dag.column.multi.AbstractGreatestLeastColumnTransformer; import org.apache.iotdb.db.queryengine.transformation.dag.column.multi.CoalesceColumnTransformer; import org.apache.iotdb.db.queryengine.transformation.dag.column.multi.InBinaryMultiColumnTransformer; import org.apache.iotdb.db.queryengine.transformation.dag.column.multi.InBooleanMultiColumnTransformer; @@ -978,20 +977,6 @@ private ColumnTransformer getFunctionColumnTransformer( } return new FormatColumnTransformer( STRING, columnTransformers, context.sessionInfo.getZoneId()); - } else if (TableBuiltinScalarFunction.GREATEST - .getFunctionName() - .equalsIgnoreCase(functionName)) { - List columnTransformers = - children.stream().map(child -> this.process(child, context)).collect(Collectors.toList()); - Type returnType = columnTransformers.get(0).getType(); - return AbstractGreatestLeastColumnTransformer.getGreatestColumnTransformer( - returnType, columnTransformers); - } else if (TableBuiltinScalarFunction.LEAST.getFunctionName().equalsIgnoreCase(functionName)) { - List columnTransformers = - children.stream().map(child -> this.process(child, context)).collect(Collectors.toList()); - Type returnType = columnTransformers.get(0).getType(); - return AbstractGreatestLeastColumnTransformer.getLeastColumnTransformer( - returnType, columnTransformers); } else { // user defined function if (TableUDFUtils.isScalarFunction(functionName)) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/schedule/DriverScheduler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/schedule/DriverScheduler.java index 80dec27b4cf5..dfe4fc9a742e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/schedule/DriverScheduler.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/schedule/DriverScheduler.java @@ -43,6 +43,7 @@ import org.apache.iotdb.db.queryengine.execution.schedule.queue.multilevelqueue.MultilevelPriorityQueue; import org.apache.iotdb.db.queryengine.execution.schedule.task.DriverTask; import org.apache.iotdb.db.queryengine.execution.schedule.task.DriverTaskStatus; +import org.apache.iotdb.db.queryengine.metric.DriverSchedulerMetricSet; import org.apache.iotdb.db.storageengine.rescon.quotas.DataNodeThrottleQuotaManager; import org.apache.iotdb.db.utils.SetThreadName; import org.apache.iotdb.mpp.rpc.thrift.TFragmentInstanceId; @@ -65,10 +66,15 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; +import static org.apache.iotdb.db.queryengine.metric.DriverSchedulerMetricSet.BLOCK_QUEUED_TIME; +import static org.apache.iotdb.db.queryengine.metric.DriverSchedulerMetricSet.READY_QUEUED_TIME; + /** The manager of fragment instances scheduling. */ public class DriverScheduler implements IDriverScheduler, IService { private static final Logger logger = LoggerFactory.getLogger(DriverScheduler.class); + private static final DriverSchedulerMetricSet DRIVER_SCHEDULER_METRIC_SET = + DriverSchedulerMetricSet.getInstance(); private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); private static final double LEVEL_TIME_MULTIPLIER = 2; @@ -479,6 +485,7 @@ public void blockedToReady(DriverTask task) { .getDriverContext() .getFragmentInstanceContext() .addBlockQueuedTime(blockQueuedTime); + DRIVER_SCHEDULER_METRIC_SET.recordTaskQueueTime(BLOCK_QUEUED_TIME, blockQueuedTime); task.setLastEnterReadyQueueTime(currentTime); task.resetLevelScheduledTime(); readyQueue.repush(task); @@ -502,6 +509,7 @@ public boolean readyToRunning(DriverTask task) { .getDriverContext() .getFragmentInstanceContext() .addReadyQueuedTime(readyQueuedTime); + DRIVER_SCHEDULER_METRIC_SET.recordTaskQueueTime(READY_QUEUED_TIME, readyQueuedTime); } finally { task.unlock(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/Coordinator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/Coordinator.java index dd56b601f7e3..a0602bec83dc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/Coordinator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/Coordinator.java @@ -70,20 +70,13 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropDB; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropFunction; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropTable; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ExtendRegion; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Flush; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.KillQuery; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LoadConfiguration; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.MigrateRegion; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.PipeStatement; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ReconstructRegion; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.RelationalAuthorStatement; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.RemoveDataNode; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.RemoveRegion; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SetConfiguration; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SetProperties; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SetSqlDialect; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SetSystemStatus; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowAINodes; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowCluster; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowClusterId; @@ -414,15 +407,12 @@ private IQueryExecution createQueryExecutionForTableModel( || statement instanceof Flush || statement instanceof ClearCache || statement instanceof SetConfiguration - || statement instanceof LoadConfiguration - || statement instanceof SetSystemStatus || statement instanceof StartRepairData || statement instanceof StopRepairData || statement instanceof PipeStatement || statement instanceof RemoveDataNode || statement instanceof SubscriptionStatement || statement instanceof ShowCurrentSqlDialect - || statement instanceof SetSqlDialect || statement instanceof ShowCurrentUser || statement instanceof ShowCurrentDatabase || statement instanceof ShowVersion @@ -433,11 +423,7 @@ private IQueryExecution createQueryExecutionForTableModel( || statement instanceof CreateFunction || statement instanceof DropFunction || statement instanceof ShowFunctions - || statement instanceof RelationalAuthorStatement - || statement instanceof MigrateRegion - || statement instanceof ReconstructRegion - || statement instanceof ExtendRegion - || statement instanceof RemoveRegion) { + || statement instanceof RelationalAuthorStatement) { return new ConfigExecution( queryContext, null, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeUtils.java index 99c1dad2c536..ffc6442b89f2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeUtils.java @@ -329,7 +329,8 @@ public static void analyzeDelete(final Delete node, final MPPQueryContext queryC } @SuppressWarnings("java:S3655") // optional is checked - public static String getDatabaseName(final Delete node, final MPPQueryContext queryContext) { + private static void validateSchema(final Delete node, final MPPQueryContext queryContext) { + final String tableName = node.getTable().getName().getSuffix(); final String databaseName; if (node.getTable().getName().getPrefix().isPresent()) { databaseName = node.getTable().getName().getPrefix().get().toString(); @@ -338,12 +339,6 @@ public static String getDatabaseName(final Delete node, final MPPQueryContext qu } else { throw new SemanticException(DATABASE_NOT_SPECIFIED); } - return databaseName; - } - - private static void validateSchema(final Delete node, final MPPQueryContext queryContext) { - final String tableName = node.getTable().getName().getSuffix(); - final String databaseName = getDatabaseName(node, queryContext); InformationSchemaUtils.checkDBNameInWrite(databaseName); node.setDatabaseName(databaseName); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java index 27111794cc9c..f7d0f4c8f59f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java @@ -220,7 +220,7 @@ import static org.apache.iotdb.db.schemaengine.schemaregion.view.visitor.GetSourcePathsVisitor.getSourcePaths; import static org.apache.iotdb.db.storageengine.load.metrics.LoadTsFileCostMetricsSet.ANALYSIS; import static org.apache.iotdb.db.utils.constant.SqlConstant.COUNT_TIME_HEADER; -import static org.apache.iotdb.db.utils.constant.SqlConstant.TREE_MODEL_DATABASE_PREFIX; +import static org.apache.iotdb.db.utils.constant.SqlConstant.ROOT_DOT; /** This visitor is used to analyze each type of Statement and returns the {@link Analysis}. */ public class AnalyzeVisitor extends StatementVisitor { @@ -2430,7 +2430,7 @@ private void analyzeInto( if (viewPath != null) { try { // if it's really view path, it should start with root. - if (viewPath.startsWith(TREE_MODEL_DATABASE_PREFIX)) { + if (viewPath.startsWith(ROOT_DOT)) { sourcePath = new MeasurementPath(viewPath); } else { // otherwise it should just be an alias diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/IDualKeyCacheStats.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/IDualKeyCacheStats.java index 9247e20a9c48..9ed842cc602d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/IDualKeyCacheStats.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/IDualKeyCacheStats.java @@ -39,10 +39,4 @@ public interface IDualKeyCacheStats { /** Return current memory usage of dual key cache. */ long memoryUsage(); - - /** Return capacity of dual key cache. */ - long capacity(); - - /** Return entries num of dual key cache */ - long entriesCount(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/CacheStats.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/CacheStats.java index 99c331962a20..817092d3eb90 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/CacheStats.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/CacheStats.java @@ -31,7 +31,6 @@ class CacheStats implements IDualKeyCacheStats { private final long memoryThreshold; private final AtomicLong memoryUsage = new AtomicLong(0); - private final AtomicLong entriesCount = new AtomicLong(0); private final AtomicLong requestCount = new AtomicLong(0); private final AtomicLong hitCount = new AtomicLong(0); @@ -69,14 +68,6 @@ void recordMiss(int num) { requestCount.getAndAdd(num); } - void increaseEntryCount() { - entriesCount.incrementAndGet(); - } - - void decreaseEntryCount() { - entriesCount.decrementAndGet(); - } - @Override public long requestCount() { return requestCount.get(); @@ -105,24 +96,13 @@ public long memoryUsage() { return memoryUsage.get(); } - @Override - public long capacity() { - return memoryThreshold; - } - - @Override - public long entriesCount() { - return entriesCount.get(); - } - void reset() { - resetMemoryUsageAndEntriesCount(); + resetMemoryUsage(); hitCount.set(0); requestCount.set(0); } - void resetMemoryUsageAndEntriesCount() { + void resetMemoryUsage() { memoryUsage.set(0); - entriesCount.set(0); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/DualKeyCacheImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/DualKeyCacheImpl.java index 9ce72932733e..126f5314495d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/DualKeyCacheImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/dualkeycache/impl/DualKeyCacheImpl.java @@ -160,7 +160,6 @@ public void put(final FK firstKey, final SK secondKey, final V value) { cacheEntry = cacheEntryManager.createCacheEntry(secondKey, value, finalCacheEntryGroup); cacheEntryManager.put(cacheEntry); - cacheStats.increaseEntryCount(); usedMemorySize.getAndAdd(sizeComputer.computeSecondKeySize(sk)); } else { final V existingValue = cacheEntry.getValue(); @@ -209,7 +208,6 @@ public void update( cacheEntryManager.createCacheEntry( secondKey, value, finalCacheEntryGroup); cacheEntryManager.put(entry); - cacheStats.increaseEntryCount(); usedMemorySize.getAndAdd( sizeComputer.computeSecondKeySize(sk) + sizeComputer.computeValueSize(entry.getValue())); @@ -268,25 +266,21 @@ public void update( if (!firstKeyChecker.test(firstKey)) { continue; } - firstKeyMap.compute( - firstKey, - (fk, entryGroup) -> { - if (Objects.nonNull(entryGroup)) { - entryGroup - .getAllCacheEntries() - .forEachRemaining( - entry -> { - if (!secondKeyChecker.test(entry.getKey())) { - return; - } - final int result = updater.applyAsInt(entry.getValue().getValue()); - if (Objects.nonNull(entryGroup.getCacheEntry(entry.getKey()))) { - usedMemorySize.getAndAdd(result); - } - }); - } - return entryGroup; - }); + final ICacheEntryGroup entryGroup = firstKeyMap.get(firstKey); + if (Objects.nonNull(entryGroup)) { + entryGroup + .getAllCacheEntries() + .forEachRemaining( + entry -> { + if (!secondKeyChecker.test(entry.getKey())) { + return; + } + final int result = updater.applyAsInt(entry.getValue().getValue()); + if (Objects.nonNull(entryGroup.getCacheEntry(entry.getKey()))) { + usedMemorySize.getAndAdd(result); + } + }); + } } increaseMemoryUsageAndMayEvict(usedMemorySize.get()); } @@ -305,33 +299,30 @@ private int evictOneCacheEntry() { } final AtomicInteger evictedSize = new AtomicInteger(0); + evictedSize.getAndAdd(sizeComputer.computeValueSize(evictCacheEntry.getValue())); final ICacheEntryGroup belongedGroup = evictCacheEntry.getBelongedGroup(); evictCacheEntry.setBelongedGroup(null); + belongedGroup.removeCacheEntry(evictCacheEntry.getSecondKey()); + evictedSize.getAndAdd(sizeComputer.computeSecondKeySize(evictCacheEntry.getSecondKey())); - firstKeyMap.compute( - belongedGroup.getFirstKey(), - (firstKey, cacheEntryGroup) -> { - belongedGroup.removeCacheEntry(evictCacheEntry.getSecondKey()); - cacheStats.decreaseEntryCount(); - evictedSize.getAndAdd( - sizeComputer.computeValueSize(evictCacheEntry.getValue()) - + sizeComputer.computeSecondKeySize(evictCacheEntry.getSecondKey())); - - if (cacheEntryGroup == null) { - // has been removed by other threads - return null; - } - - if (cacheEntryGroup.isEmpty()) { - evictedSize.getAndAdd(sizeComputer.computeFirstKeySize(firstKey)); - return null; - } - - // some other thread has put value to it - return cacheEntryGroup; - }); + if (belongedGroup.isEmpty()) { + firstKeyMap.compute( + belongedGroup.getFirstKey(), + (firstKey, cacheEntryGroup) -> { + if (cacheEntryGroup == null) { + // has been removed by other threads + return null; + } + if (cacheEntryGroup.isEmpty()) { + evictedSize.getAndAdd(sizeComputer.computeFirstKeySize(firstKey)); + return null; + } + // some other thread has put value to it + return cacheEntryGroup; + }); + } return evictedSize.get(); } @@ -343,7 +334,7 @@ public void invalidateAll() { private void executeInvalidateAll() { firstKeyMap.clear(); cacheEntryManager.cleanUp(); - cacheStats.resetMemoryUsageAndEntriesCount(); + cacheStats.resetMemoryUsage(); } @Override @@ -373,7 +364,6 @@ public void invalidate(final FK firstKey) { it.hasNext(); ) { final Map.Entry entry = it.next(); if (cacheEntryManager.invalidate(entry.getValue())) { - cacheStats.decreaseEntryCount(); estimateSize += sizeComputer.computeSecondKeySize(entry.getKey()) + sizeComputer.computeValueSize(entry.getValue().getValue()); @@ -397,7 +387,6 @@ public void invalidate(final FK firstKey, final SK secondKey) { final T entry = cacheEntryGroup.getCacheEntry(secondKey); if (Objects.nonNull(entry) && cacheEntryManager.invalidate(entry)) { - cacheStats.decreaseEntryCount(); usedMemorySize.getAndAdd( sizeComputer.computeSecondKeySize(entry.getSecondKey()) + sizeComputer.computeValueSize(entry.getValue())); @@ -429,7 +418,6 @@ public void invalidate(final FK firstKey, final Predicate secondKeyChecker) it.hasNext(); ) { final Map.Entry entry = it.next(); if (cacheEntryManager.invalidate(entry.getValue())) { - cacheStats.decreaseEntryCount(); cacheEntryGroup.removeCacheEntry(entry.getKey()); estimateSize.addAndGet( sizeComputer.computeSecondKeySize(entry.getKey()) @@ -456,36 +444,28 @@ public void invalidate( if (!firstKeyChecker.test(firstKey)) { continue; } + final ICacheEntryGroup entryGroup = firstKeyMap.get(firstKey); + for (final Iterator> it = entryGroup.getAllCacheEntries(); it.hasNext(); ) { + final Map.Entry entry = it.next(); + if (!secondKeyChecker.test(entry.getKey())) { + continue; + } + if (cacheEntryManager.invalidate(entry.getValue())) { + entryGroup.removeCacheEntry(entry.getKey()); + estimateSize.addAndGet( + sizeComputer.computeSecondKeySize(entry.getKey()) + + sizeComputer.computeValueSize(entry.getValue().getValue())); + } + } firstKeyMap.compute( firstKey, - (fk, cacheEntryGroup) -> { - if (cacheEntryGroup == null) { - // has been removed by other threads - return null; - } - - for (final Iterator> it = cacheEntryGroup.getAllCacheEntries(); - it.hasNext(); ) { - final Map.Entry entry = it.next(); - if (!secondKeyChecker.test(entry.getKey())) { - continue; - } - - if (cacheEntryManager.invalidate(entry.getValue())) { - cacheStats.decreaseEntryCount(); - cacheEntryGroup.removeCacheEntry(entry.getKey()); - estimateSize.addAndGet( - sizeComputer.computeSecondKeySize(entry.getKey()) - + sizeComputer.computeValueSize(entry.getValue().getValue())); - } - } - - if (cacheEntryGroup.isEmpty()) { + (fk, sk) -> { + if (sk.isEmpty()) { estimateSize.getAndAdd(sizeComputer.computeFirstKeySize(firstKey)); return null; } - return cacheEntryGroup; + return sk; }); } cacheStats.decreaseMemoryUsage(estimateSize.get()); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java index 010fdf84bf5a..92cd9584841c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileAnalyzer.java @@ -194,16 +194,9 @@ protected void executeTabletConversion(final IAnalysis analysis, final LoadAnaly : null; if (status == null) { - LOGGER.warn( - "Load: Failed to convert to tablets from statement {}. Status is null.", - isTableModelStatement ? loadTsFileTableStatement : loadTsFileTreeStatement); analysis.setFailStatus( new TSStatus(TSStatusCode.LOAD_FILE_ERROR.getStatusCode()).setMessage(e.getMessage())); } else if (!loadTsFileDataTypeConverter.isSuccessful(status)) { - LOGGER.warn( - "Load: Failed to convert to tablets from statement {}. Status: {}", - isTableModelStatement ? loadTsFileTableStatement : loadTsFileTreeStatement, - status); analysis.setFailStatus(status); } analysis.setFinishQueryAfterAnalyze(true); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/SchemaValidator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/SchemaValidator.java index 441e882242ad..aaf5d08f43c8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/SchemaValidator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/SchemaValidator.java @@ -25,8 +25,6 @@ import org.apache.iotdb.db.queryengine.common.MPPQueryContext; import org.apache.iotdb.db.queryengine.common.schematree.ISchemaTree; import org.apache.iotdb.db.queryengine.plan.relational.metadata.Metadata; -import org.apache.iotdb.db.queryengine.plan.relational.metadata.QualifiedObjectName; -import org.apache.iotdb.db.queryengine.plan.relational.security.AccessControl; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.WrappedInsertStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertBaseStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertMultiTabletsStatement; @@ -41,8 +39,6 @@ import java.util.List; -import static org.apache.iotdb.commons.utils.PathUtils.unQualifyDatabaseName; - public class SchemaValidator { private static final Logger LOGGER = LoggerFactory.getLogger(SchemaValidator.class); @@ -68,15 +64,10 @@ public static void validate( public static void validate( final Metadata metadata, final WrappedInsertStatement insertStatement, - final MPPQueryContext context, - AccessControl accessControl) { + final MPPQueryContext context) { try { + insertStatement.toLowerCase(); insertStatement.validateTableSchema(metadata, context); - accessControl.checkCanInsertIntoTable( - context.getSession().getUserName(), - new QualifiedObjectName( - unQualifyDatabaseName(insertStatement.getDatabase()), - insertStatement.getTableName())); insertStatement.updateAfterSchemaValidation(context); insertStatement.validateDeviceSchema(metadata, context); } catch (final QueryProcessException e) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/IQueryExecution.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/IQueryExecution.java index d5a35ff8ca8c..366af9fdfca8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/IQueryExecution.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/IQueryExecution.java @@ -72,6 +72,4 @@ public interface IQueryExecution { String getStatementType(); IClientSession.SqlDialect getSQLDialect(); - - String getUser(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/QueryExecution.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/QueryExecution.java index 84a757893bc2..c387e9267b55 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/QueryExecution.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/QueryExecution.java @@ -693,11 +693,6 @@ public IClientSession.SqlDialect getSQLDialect() { return context.getSession().getSqlDialect(); } - @Override - public String getUser() { - return context.getSession().getUserName(); - } - public MPPQueryContext getContext() { return context; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/ConfigExecution.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/ConfigExecution.java index 80d99236c39b..92a8a979d653 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/ConfigExecution.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/ConfigExecution.java @@ -93,9 +93,7 @@ public class ConfigExecution implements IQueryExecution { TSStatusCode.ROLE_NOT_EXIST.getStatusCode(), TSStatusCode.USER_ALREADY_HAS_ROLE.getStatusCode(), TSStatusCode.USER_NOT_HAS_ROLE.getStatusCode(), - TSStatusCode.NOT_HAS_PRIVILEGE_GRANTOPT.getStatusCode(), - TSStatusCode.SEMANTIC_ERROR.getStatusCode(), - TSStatusCode.NO_SUCH_QUERY.getStatusCode()))); + TSStatusCode.NOT_HAS_PRIVILEGE_GRANTOPT.getStatusCode()))); private final MPPQueryContext context; private final ExecutorService executor; @@ -310,9 +308,4 @@ public String getStatementType() { public IClientSession.SqlDialect getSQLDialect() { return context.getSession().getSqlDialect(); } - - @Override - public String getUser() { - return context.getSession().getUserName(); - } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java index db1c40390104..18dd15c3ae2f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java @@ -44,10 +44,6 @@ import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.ShowPipePluginsTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.ShowRegionTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.ShowVariablesTask; -import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.region.ExtendRegionTask; -import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.region.MigrateRegionTask; -import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.region.ReconstructRegionTask; -import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.region.RemoveRegionTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.AlterDBTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.AlterTableAddColumnTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.AlterTableDropColumnTask; @@ -70,7 +66,6 @@ import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.ShowTablesDetailsTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.ShowTablesTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.UseDBTask; -import org.apache.iotdb.db.queryengine.plan.execution.config.session.SetSqlDialectTask; import org.apache.iotdb.db.queryengine.plan.execution.config.session.ShowCurrentDatabaseTask; import org.apache.iotdb.db.queryengine.plan.execution.config.session.ShowCurrentSqlDialectTask; import org.apache.iotdb.db.queryengine.plan.execution.config.session.ShowCurrentTimestampTask; @@ -78,9 +73,7 @@ import org.apache.iotdb.db.queryengine.plan.execution.config.session.ShowVersionTask; import org.apache.iotdb.db.queryengine.plan.execution.config.sys.FlushTask; import org.apache.iotdb.db.queryengine.plan.execution.config.sys.KillQueryTask; -import org.apache.iotdb.db.queryengine.plan.execution.config.sys.LoadConfigurationTask; import org.apache.iotdb.db.queryengine.plan.execution.config.sys.SetConfigurationTask; -import org.apache.iotdb.db.queryengine.plan.execution.config.sys.SetSystemStatusTask; import org.apache.iotdb.db.queryengine.plan.execution.config.sys.StartRepairDataTask; import org.apache.iotdb.db.queryengine.plan.execution.config.sys.StopRepairDataTask; import org.apache.iotdb.db.queryengine.plan.execution.config.sys.pipe.AlterPipeTask; @@ -123,26 +116,19 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropTable; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropTopic; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ExtendRegion; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Flush; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.KillQuery; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Literal; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LoadConfiguration; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LongLiteral; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.MigrateRegion; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Node; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Property; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.QualifiedName; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ReconstructRegion; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.RelationalAuthorStatement; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.RemoveDataNode; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.RemoveRegion; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.RenameColumn; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.RenameTable; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SetConfiguration; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SetProperties; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SetSqlDialect; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SetSystemStatus; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowAINodes; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowCluster; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowClusterId; @@ -174,9 +160,7 @@ import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowClusterStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowRegionStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.FlushStatement; -import org.apache.iotdb.db.queryengine.plan.statement.sys.LoadConfigurationStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.SetConfigurationStatement; -import org.apache.iotdb.db.queryengine.plan.statement.sys.SetSystemStatusStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.StartRepairDataStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.StopRepairDataStatement; import org.apache.iotdb.db.schemaengine.table.InformationSchemaUtils; @@ -746,20 +730,6 @@ protected IConfigTask visitStopRepairData(StopRepairData node, MPPQueryContext c return new StopRepairDataTask(((StopRepairDataStatement) node.getInnerTreeStatement())); } - @Override - protected IConfigTask visitLoadConfiguration(LoadConfiguration node, MPPQueryContext context) { - context.setQueryType(QueryType.WRITE); - accessControl.checkUserHasMaintainPrivilege(context.getSession().getUserName()); - return new LoadConfigurationTask(((LoadConfigurationStatement) node.getInnerTreeStatement())); - } - - @Override - protected IConfigTask visitSetSystemStatus(SetSystemStatus node, MPPQueryContext context) { - context.setQueryType(QueryType.WRITE); - accessControl.checkUserHasMaintainPrivilege(context.getSession().getUserName()); - return new SetSystemStatusTask(((SetSystemStatusStatement) node.getInnerTreeStatement())); - } - private Optional parseStringFromLiteralIfBinary(final Object value) { return value instanceof Literal && ((Literal) value).getTsValue() instanceof Binary ? Optional.of( @@ -950,12 +920,6 @@ protected IConfigTask visitShowCurrentSqlDialect( return new ShowCurrentSqlDialectTask(context.getSession().getSqlDialect().name()); } - @Override - protected IConfigTask visitSetSqlDialect(SetSqlDialect node, MPPQueryContext context) { - context.setQueryType(QueryType.WRITE); - return new SetSqlDialectTask(node.getSqlDialect()); - } - @Override protected IConfigTask visitShowCurrentDatabase( ShowCurrentDatabase node, MPPQueryContext context) { @@ -1002,14 +966,12 @@ protected IConfigTask visitRelationalAuthorPlan( @Override protected IConfigTask visitKillQuery(KillQuery node, MPPQueryContext context) { context.setQueryType(QueryType.WRITE); - accessControl.checkUserHasMaintainPrivilege(context.getSession().getUserName()); return new KillQueryTask(node); } @Override protected IConfigTask visitCreateFunction(CreateFunction node, MPPQueryContext context) { context.setQueryType(QueryType.WRITE); - accessControl.checkUserIsAdmin(context.getSession().getUserName()); if (node.getUriString().map(ExecutableManager::isUriTrusted).orElse(true)) { // 1. user specified uri and that uri is trusted // 2. user doesn't specify uri @@ -1029,44 +991,6 @@ protected IConfigTask visitShowFunctions(ShowFunctions node, MPPQueryContext con @Override protected IConfigTask visitDropFunction(DropFunction node, MPPQueryContext context) { context.setQueryType(QueryType.WRITE); - accessControl.checkUserIsAdmin(context.getSession().getUserName()); return new DropFunctionTask(Model.TABLE, node.getUdfName()); } - - @Override - protected IConfigTask visitMigrateRegion(MigrateRegion migrateRegion, MPPQueryContext context) { - context.setQueryType(QueryType.WRITE); - accessControl.checkUserHasMaintainPrivilege(context.getSession().getUserName()); - // As the implementation is identical, we'll simply translate to the - // corresponding tree-model variant and execute that. - return new MigrateRegionTask(migrateRegion); - } - - @Override - protected IConfigTask visitReconstructRegion( - ReconstructRegion reconstructRegion, MPPQueryContext context) { - context.setQueryType(QueryType.WRITE); - accessControl.checkUserHasMaintainPrivilege(context.getSession().getUserName()); - // As the implementation is identical, we'll simply translate to the - // corresponding tree-model variant and execute that. - return new ReconstructRegionTask(reconstructRegion); - } - - @Override - protected IConfigTask visitExtendRegion(ExtendRegion extendRegion, MPPQueryContext context) { - context.setQueryType(QueryType.WRITE); - accessControl.checkUserHasMaintainPrivilege(context.getSession().getUserName()); - // As the implementation is identical, we'll simply translate to the - // corresponding tree-model variant and execute that. - return new ExtendRegionTask(extendRegion); - } - - @Override - protected IConfigTask visitRemoveRegion(RemoveRegion removeRegion, MPPQueryContext context) { - context.setQueryType(QueryType.WRITE); - accessControl.checkUserHasMaintainPrivilege(context.getSession().getUserName()); - // As the implementation is identical, we'll simply translate to the - // corresponding tree-model variant and execute that. - return new RemoveRegionTask(removeRegion); - } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TreeConfigTaskVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TreeConfigTaskVisitor.java index 39fba9d7b318..7ec00aacf58d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TreeConfigTaskVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TreeConfigTaskVisitor.java @@ -76,9 +76,6 @@ import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.view.AlterLogicalViewTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.view.DeleteLogicalViewTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.view.RenameLogicalViewTask; -import org.apache.iotdb.db.queryengine.plan.execution.config.session.SetSqlDialectTask; -import org.apache.iotdb.db.queryengine.plan.execution.config.session.ShowCurrentSqlDialectTask; -import org.apache.iotdb.db.queryengine.plan.execution.config.session.ShowCurrentUserTask; import org.apache.iotdb.db.queryengine.plan.execution.config.sys.AuthorizerTask; import org.apache.iotdb.db.queryengine.plan.execution.config.sys.ClearCacheTask; import org.apache.iotdb.db.queryengine.plan.execution.config.sys.FlushTask; @@ -175,10 +172,7 @@ import org.apache.iotdb.db.queryengine.plan.statement.sys.LoadConfigurationStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.MergeStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.SetConfigurationStatement; -import org.apache.iotdb.db.queryengine.plan.statement.sys.SetSqlDialectStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.SetSystemStatusStatement; -import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowCurrentSqlDialectStatement; -import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowCurrentUserStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.StartRepairDataStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.StopRepairDataStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.TestConnectionStatement; @@ -708,12 +702,6 @@ public IConfigTask visitShowThrottleQuota( return new ShowThrottleQuotaTask(showThrottleQuotaStatement); } - @Override - public IConfigTask visitSetSqlDialect( - SetSqlDialectStatement setSqlDialectStatement, MPPQueryContext context) { - return new SetSqlDialectTask(setSqlDialectStatement.getSqlDialect()); - } - /** AI Model Management */ @Override public IConfigTask visitCreateModel( @@ -739,15 +727,4 @@ public IConfigTask visitShowModels( ShowModelsStatement showModelsStatement, MPPQueryContext context) { return new ShowModelsTask(showModelsStatement.getModelName()); } - - @Override - public IConfigTask visitShowCurrentUser(ShowCurrentUserStatement node, MPPQueryContext context) { - return new ShowCurrentUserTask(context.getSession().getUserName()); - } - - @Override - public IConfigTask visitShowCurrentSqlDialect( - ShowCurrentSqlDialectStatement node, MPPQueryContext context) { - return new ShowCurrentSqlDialectTask(context.getSession().getSqlDialect().name()); - } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java index 0eac1e050c98..4a70c5991c98 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java @@ -150,13 +150,13 @@ import org.apache.iotdb.db.exception.StorageEngineException; import org.apache.iotdb.db.exception.metadata.PathNotExistException; import org.apache.iotdb.db.exception.metadata.SchemaQuotaExceededException; +import org.apache.iotdb.db.exception.sql.SemanticException; import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent; import org.apache.iotdb.db.protocol.client.ConfigNodeClient; import org.apache.iotdb.db.protocol.client.ConfigNodeClientManager; import org.apache.iotdb.db.protocol.client.ConfigNodeInfo; import org.apache.iotdb.db.protocol.client.DataNodeClientPoolFactory; import org.apache.iotdb.db.protocol.session.IClientSession; -import org.apache.iotdb.db.protocol.session.SessionManager; import org.apache.iotdb.db.queryengine.common.MPPQueryContext; import org.apache.iotdb.db.queryengine.common.SessionInfo; import org.apache.iotdb.db.queryengine.common.schematree.ISchemaTree; @@ -187,10 +187,6 @@ import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.ShowTriggersTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.ShowVariablesTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.model.ShowModelsTask; -import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.region.ExtendRegionTask; -import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.region.MigrateRegionTask; -import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.region.ReconstructRegionTask; -import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.region.RemoveRegionTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.DeleteDeviceTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.DescribeTableDetailsTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.DescribeTableTask; @@ -245,6 +241,10 @@ import org.apache.iotdb.db.queryengine.plan.statement.metadata.pipe.ShowPipesStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.pipe.StartPipeStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.pipe.StopPipeStatement; +import org.apache.iotdb.db.queryengine.plan.statement.metadata.region.ExtendRegionStatement; +import org.apache.iotdb.db.queryengine.plan.statement.metadata.region.MigrateRegionStatement; +import org.apache.iotdb.db.queryengine.plan.statement.metadata.region.ReconstructRegionStatement; +import org.apache.iotdb.db.queryengine.plan.statement.metadata.region.RemoveRegionStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.subscription.CreateTopicStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.subscription.DropTopicStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.subscription.ShowSubscriptionsStatement; @@ -1344,29 +1344,19 @@ public SettableFuture setSystemStatus(boolean onCluster, NodeS public SettableFuture killQuery(final KillQueryStatement killQueryStatement) { int dataNodeId = -1; String queryId = killQueryStatement.getQueryId(); - SettableFuture future = SettableFuture.create(); if (!killQueryStatement.isKillAll()) { String[] splits = queryId.split("_"); try { // We just judge the input queryId has three '_' and the DataNodeId from it is non-negative // here if (splits.length != 4 || ((dataNodeId = Integer.parseInt(splits[3])) < 0)) { - future.setException( - new IoTDBException( - "Please ensure your input is correct", - TSStatusCode.SEMANTIC_ERROR.getStatusCode(), - true)); - return future; + throw new SemanticException("Please ensure your input is correct"); } } catch (NumberFormatException e) { - future.setException( - new IoTDBException( - "Please ensure your input is correct", - TSStatusCode.SEMANTIC_ERROR.getStatusCode(), - true)); - return future; + throw new SemanticException("Please ensure your input is correct"); } } + SettableFuture future = SettableFuture.create(); try (ConfigNodeClient client = CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { final TSStatus executionStatus = client.killQuery(queryId, dataNodeId); @@ -1446,18 +1436,6 @@ public SettableFuture showCurrentSqlDialect(final String sqlDi return future; } - @Override - public SettableFuture setSqlDialect(IClientSession.SqlDialect sqlDialect) { - final SettableFuture future = SettableFuture.create(); - try { - SessionManager.getInstance().getCurrSession().setSqlDialect(sqlDialect); - future.set(new ConfigTaskResult(TSStatusCode.SUCCESS_STATUS)); - } catch (Exception e) { - future.setException(e); - } - return future; - } - @Override public SettableFuture showCurrentDatabase( @Nullable final String currentDatabase) { @@ -2802,16 +2780,16 @@ public SettableFuture countTimeSlotList( } @Override - public SettableFuture migrateRegion(final MigrateRegionTask migrateRegionTask) { + public SettableFuture migrateRegion( + final MigrateRegionStatement migrateRegionStatement) { final SettableFuture future = SettableFuture.create(); try (ConfigNodeClient configNodeClient = CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { final TMigrateRegionReq tMigrateRegionReq = new TMigrateRegionReq( - migrateRegionTask.getStatement().getRegionId(), - migrateRegionTask.getStatement().getFromId(), - migrateRegionTask.getStatement().getToId(), - migrateRegionTask.getModel()); + migrateRegionStatement.getRegionId(), + migrateRegionStatement.getFromId(), + migrateRegionStatement.getToId()); final TSStatus status = configNodeClient.migrateRegion(tMigrateRegionReq); if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { future.setException(new IoTDBException(status.message, status.code)); @@ -2891,15 +2869,14 @@ public SettableFuture removeDataNode( @Override public SettableFuture reconstructRegion( - ReconstructRegionTask reconstructRegionTask) { + ReconstructRegionStatement reconstructRegionStatement) { final SettableFuture future = SettableFuture.create(); try (ConfigNodeClient configNodeClient = CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { final TReconstructRegionReq req = new TReconstructRegionReq( - reconstructRegionTask.getStatement().getRegionIds(), - reconstructRegionTask.getStatement().getDataNodeId(), - reconstructRegionTask.getModel()); + reconstructRegionStatement.getRegionIds(), + reconstructRegionStatement.getDataNodeId()); final TSStatus status = configNodeClient.reconstructRegion(req); if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { future.setException(new IoTDBException(status.message, status.code)); @@ -2914,15 +2891,14 @@ public SettableFuture reconstructRegion( } @Override - public SettableFuture extendRegion(ExtendRegionTask extendRegionTask) { + public SettableFuture extendRegion( + ExtendRegionStatement extendRegionStatement) { final SettableFuture future = SettableFuture.create(); try (ConfigNodeClient configNodeClient = CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { final TExtendRegionReq req = new TExtendRegionReq( - extendRegionTask.getStatement().getRegionId(), - extendRegionTask.getStatement().getDataNodeId(), - extendRegionTask.getModel()); + extendRegionStatement.getRegionId(), extendRegionStatement.getDataNodeId()); final TSStatus status = configNodeClient.extendRegion(req); if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { future.setException(new IoTDBException(status.message, status.code)); @@ -2937,15 +2913,14 @@ public SettableFuture extendRegion(ExtendRegionTask extendRegi } @Override - public SettableFuture removeRegion(RemoveRegionTask removeRegionTask) { + public SettableFuture removeRegion( + RemoveRegionStatement removeRegionStatement) { final SettableFuture future = SettableFuture.create(); try (ConfigNodeClient configNodeClient = CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { final TRemoveRegionReq req = new TRemoveRegionReq( - removeRegionTask.getStatement().getRegionId(), - removeRegionTask.getStatement().getDataNodeId(), - removeRegionTask.getModel()); + removeRegionStatement.getRegionId(), removeRegionStatement.getDataNodeId()); final TSStatus status = configNodeClient.removeRegion(req); if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { future.setException(new IoTDBException(status.message, status.code)); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java index 1372598493a1..d34ae66ac16e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java @@ -35,10 +35,6 @@ import org.apache.iotdb.db.queryengine.common.MPPQueryContext; import org.apache.iotdb.db.queryengine.common.SessionInfo; import org.apache.iotdb.db.queryengine.plan.execution.config.ConfigTaskResult; -import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.region.ExtendRegionTask; -import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.region.MigrateRegionTask; -import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.region.ReconstructRegionTask; -import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.region.RemoveRegionTask; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write.view.AlterLogicalViewNode; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DeleteDevice; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropDB; @@ -71,6 +67,10 @@ import org.apache.iotdb.db.queryengine.plan.statement.metadata.pipe.ShowPipesStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.pipe.StartPipeStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.pipe.StopPipeStatement; +import org.apache.iotdb.db.queryengine.plan.statement.metadata.region.ExtendRegionStatement; +import org.apache.iotdb.db.queryengine.plan.statement.metadata.region.MigrateRegionStatement; +import org.apache.iotdb.db.queryengine.plan.statement.metadata.region.ReconstructRegionStatement; +import org.apache.iotdb.db.queryengine.plan.statement.metadata.region.RemoveRegionStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.subscription.CreateTopicStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.subscription.DropTopicStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.subscription.ShowSubscriptionsStatement; @@ -253,13 +253,14 @@ SettableFuture getTimeSlotList( SettableFuture countTimeSlotList( CountTimeSlotListStatement countTimeSlotListStatement); - SettableFuture migrateRegion(MigrateRegionTask migrateRegionTask); + SettableFuture migrateRegion(MigrateRegionStatement migrateRegionStatement); - SettableFuture reconstructRegion(ReconstructRegionTask reconstructRegionTask); + SettableFuture reconstructRegion( + ReconstructRegionStatement reconstructRegionStatement); - SettableFuture extendRegion(ExtendRegionTask extendRegionTask); + SettableFuture extendRegion(ExtendRegionStatement extendRegionStatement); - SettableFuture removeRegion(RemoveRegionTask removeRegionTask); + SettableFuture removeRegion(RemoveRegionStatement removeRegionStatement); SettableFuture removeDataNode(RemoveDataNodeStatement removeDataNodeStatement); @@ -372,8 +373,6 @@ SettableFuture deleteDevice( SettableFuture showCurrentSqlDialect(String sqlDialect); - SettableFuture setSqlDialect(IClientSession.SqlDialect sqlDialect); - SettableFuture showCurrentUser(String currentUser); SettableFuture showCurrentDatabase(@Nullable String currentDatabase); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/ShowPipePluginsTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/ShowPipePluginsTask.java index f186702d595c..3292d838e206 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/ShowPipePluginsTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/ShowPipePluginsTask.java @@ -46,8 +46,8 @@ public class ShowPipePluginsTask implements IConfigTask { - public static final Binary PIPE_PLUGIN_TYPE_BUILTIN = BytesUtils.valueOf("Builtin"); - public static final Binary PIPE_PLUGIN_TYPE_EXTERNAL = BytesUtils.valueOf("External"); + private static final Binary PIPE_PLUGIN_TYPE_BUILTIN = BytesUtils.valueOf("Builtin"); + private static final Binary PIPE_PLUGIN_TYPE_EXTERNAL = BytesUtils.valueOf("External"); private static final Binary PIPE_JAR_NAME_EMPTY_FIELD = BytesUtils.valueOf(""); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/region/ExtendRegionTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/region/ExtendRegionTask.java index 1ffce0109276..00dafd9d1549 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/region/ExtendRegionTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/region/ExtendRegionTask.java @@ -19,11 +19,9 @@ package org.apache.iotdb.db.queryengine.plan.execution.config.metadata.region; -import org.apache.iotdb.common.rpc.thrift.Model; import org.apache.iotdb.db.queryengine.plan.execution.config.ConfigTaskResult; import org.apache.iotdb.db.queryengine.plan.execution.config.IConfigTask; import org.apache.iotdb.db.queryengine.plan.execution.config.executor.IConfigTaskExecutor; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ExtendRegion; import org.apache.iotdb.db.queryengine.plan.statement.metadata.region.ExtendRegionStatement; import com.google.common.util.concurrent.ListenableFuture; @@ -31,30 +29,14 @@ public class ExtendRegionTask implements IConfigTask { protected final ExtendRegionStatement statement; - private final Model model; public ExtendRegionTask(ExtendRegionStatement statement) { this.statement = statement; - this.model = Model.TREE; - } - - public ExtendRegionTask(ExtendRegion extendRegion) { - this.statement = - new ExtendRegionStatement(extendRegion.getRegionId(), extendRegion.getDataNodeId()); - this.model = Model.TABLE; } @Override public ListenableFuture execute(IConfigTaskExecutor configTaskExecutor) throws InterruptedException { - return configTaskExecutor.extendRegion(this); - } - - public ExtendRegionStatement getStatement() { - return statement; - } - - public Model getModel() { - return model; + return configTaskExecutor.extendRegion(statement); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/region/MigrateRegionTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/region/MigrateRegionTask.java index 8c6f2bfe36ba..fd22585aa8cf 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/region/MigrateRegionTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/region/MigrateRegionTask.java @@ -19,11 +19,9 @@ package org.apache.iotdb.db.queryengine.plan.execution.config.metadata.region; -import org.apache.iotdb.common.rpc.thrift.Model; import org.apache.iotdb.db.queryengine.plan.execution.config.ConfigTaskResult; import org.apache.iotdb.db.queryengine.plan.execution.config.IConfigTask; import org.apache.iotdb.db.queryengine.plan.execution.config.executor.IConfigTaskExecutor; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.MigrateRegion; import org.apache.iotdb.db.queryengine.plan.statement.metadata.region.MigrateRegionStatement; import com.google.common.util.concurrent.ListenableFuture; @@ -31,32 +29,15 @@ public class MigrateRegionTask implements IConfigTask { protected final MigrateRegionStatement statement; - private final Model model; public MigrateRegionTask(MigrateRegionStatement migrateRegionStatement) { this.statement = migrateRegionStatement; - this.model = Model.TREE; - } - - public MigrateRegionTask(MigrateRegion migrateRegion) { - this.statement = - new MigrateRegionStatement( - migrateRegion.getRegionId(), migrateRegion.getFromId(), migrateRegion.getToId()); - this.model = Model.TABLE; } @Override public ListenableFuture execute(IConfigTaskExecutor configTaskExecutor) { // If the action is executed successfully, return the Future. // If your operation is async, you can return the corresponding future directly. - return configTaskExecutor.migrateRegion(this); - } - - public Model getModel() { - return this.model; - } - - public MigrateRegionStatement getStatement() { - return statement; + return configTaskExecutor.migrateRegion(statement); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/region/ReconstructRegionTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/region/ReconstructRegionTask.java index e1d7127f749e..f5276ed64e7c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/region/ReconstructRegionTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/region/ReconstructRegionTask.java @@ -19,43 +19,24 @@ package org.apache.iotdb.db.queryengine.plan.execution.config.metadata.region; -import org.apache.iotdb.common.rpc.thrift.Model; import org.apache.iotdb.db.queryengine.plan.execution.config.ConfigTaskResult; import org.apache.iotdb.db.queryengine.plan.execution.config.IConfigTask; import org.apache.iotdb.db.queryengine.plan.execution.config.executor.IConfigTaskExecutor; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ReconstructRegion; import org.apache.iotdb.db.queryengine.plan.statement.metadata.region.ReconstructRegionStatement; import com.google.common.util.concurrent.ListenableFuture; public class ReconstructRegionTask implements IConfigTask { - protected final ReconstructRegionStatement statement; - private final Model model; + protected final ReconstructRegionStatement reconstructRegionStatement; public ReconstructRegionTask(ReconstructRegionStatement reconstructRegionStatement) { - this.statement = reconstructRegionStatement; - this.model = Model.TREE; - } - - public ReconstructRegionTask(ReconstructRegion reconstructRegion) { - this.statement = - new ReconstructRegionStatement( - reconstructRegion.getDataNodeId(), reconstructRegion.getRegionIds()); - this.model = Model.TABLE; + this.reconstructRegionStatement = reconstructRegionStatement; } @Override public ListenableFuture execute(IConfigTaskExecutor configTaskExecutor) throws InterruptedException { - return configTaskExecutor.reconstructRegion(this); - } - - public Model getModel() { - return model; - } - - public ReconstructRegionStatement getStatement() { - return statement; + return configTaskExecutor.reconstructRegion(reconstructRegionStatement); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/region/RemoveRegionTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/region/RemoveRegionTask.java index 86d4bafc9db8..26ecea097f74 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/region/RemoveRegionTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/region/RemoveRegionTask.java @@ -19,11 +19,9 @@ package org.apache.iotdb.db.queryengine.plan.execution.config.metadata.region; -import org.apache.iotdb.common.rpc.thrift.Model; import org.apache.iotdb.db.queryengine.plan.execution.config.ConfigTaskResult; import org.apache.iotdb.db.queryengine.plan.execution.config.IConfigTask; import org.apache.iotdb.db.queryengine.plan.execution.config.executor.IConfigTaskExecutor; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.RemoveRegion; import org.apache.iotdb.db.queryengine.plan.statement.metadata.region.RemoveRegionStatement; import com.google.common.util.concurrent.ListenableFuture; @@ -31,30 +29,14 @@ public class RemoveRegionTask implements IConfigTask { protected final RemoveRegionStatement statement; - private final Model model; public RemoveRegionTask(RemoveRegionStatement statement) { this.statement = statement; - this.model = Model.TREE; - } - - public RemoveRegionTask(RemoveRegion removeRegion) { - this.statement = - new RemoveRegionStatement(removeRegion.getRegionId(), removeRegion.getDataNodeId()); - this.model = Model.TABLE; } @Override public ListenableFuture execute(IConfigTaskExecutor configTaskExecutor) throws InterruptedException { - return configTaskExecutor.removeRegion(this); - } - - public RemoveRegionStatement getStatement() { - return statement; - } - - public Model getModel() { - return model; + return configTaskExecutor.removeRegion(statement); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/session/SetSqlDialectTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/session/SetSqlDialectTask.java deleted file mode 100644 index df4e3d3c641f..000000000000 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/session/SetSqlDialectTask.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.queryengine.plan.execution.config.session; - -import org.apache.iotdb.db.protocol.session.IClientSession; -import org.apache.iotdb.db.queryengine.plan.execution.config.ConfigTaskResult; -import org.apache.iotdb.db.queryengine.plan.execution.config.IConfigTask; -import org.apache.iotdb.db.queryengine.plan.execution.config.executor.IConfigTaskExecutor; - -import com.google.common.util.concurrent.ListenableFuture; - -public class SetSqlDialectTask implements IConfigTask { - private final IClientSession.SqlDialect sqlDialect; - - public SetSqlDialectTask(IClientSession.SqlDialect sqlDialect) { - this.sqlDialect = sqlDialect; - } - - @Override - public ListenableFuture execute(IConfigTaskExecutor configTaskExecutor) - throws InterruptedException { - return configTaskExecutor.setSqlDialect(sqlDialect); - } -} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java index 3b41ae014e52..d7118d7e70ee 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java @@ -39,7 +39,6 @@ import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.exception.sql.SemanticException; -import org.apache.iotdb.db.protocol.session.IClientSession; import org.apache.iotdb.db.qp.sql.IoTDBSqlParser; import org.apache.iotdb.db.qp.sql.IoTDBSqlParser.ConnectorAttributeClauseContext; import org.apache.iotdb.db.qp.sql.IoTDBSqlParser.ConstantContext; @@ -214,10 +213,7 @@ import org.apache.iotdb.db.queryengine.plan.statement.sys.KillQueryStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.LoadConfigurationStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.SetConfigurationStatement; -import org.apache.iotdb.db.queryengine.plan.statement.sys.SetSqlDialectStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.SetSystemStatusStatement; -import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowCurrentSqlDialectStatement; -import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowCurrentUserStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowQueriesStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowVersionStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.StartRepairDataStatement; @@ -1526,17 +1522,11 @@ private SelectComponent parseSelectClause( Map aliasToColumnMap = new HashMap<>(); for (IoTDBSqlParser.ResultColumnContext resultColumnContext : ctx.resultColumn()) { ResultColumn resultColumn = parseResultColumn(resultColumnContext); - String columnName = resultColumn.getExpression().getExpressionString(); // __endTime shouldn't be included in resultColumns - if (columnName.equals(ColumnHeaderConstant.ENDTIME)) { + if (resultColumn.getExpression().getExpressionString().equals(ColumnHeaderConstant.ENDTIME)) { queryStatement.setOutputEndTime(true); continue; } - // don't support pure time in select - if (columnName.equals(ColumnHeaderConstant.TIME)) { - throw new SemanticException( - "Time column is no need to appear in SELECT Clause explicitly, it will always be returned if possible"); - } if (resultColumn.hasAlias()) { String alias = resultColumn.getAlias(); if (aliasToColumnMap.containsKey(alias)) { @@ -2525,9 +2515,6 @@ private String[] parsePrivilege(String[] privileges) { continue; } else if (priv.equalsIgnoreCase("ALL")) { for (PrivilegeType type : PrivilegeType.values()) { - if (type.isRelationalPrivilege()) { - continue; - } privSet.add(type.toString()); } continue; @@ -4553,20 +4540,4 @@ private void parseWindowFunctionInInference( public Statement visitShowCurrentTimestamp(IoTDBSqlParser.ShowCurrentTimestampContext ctx) { return new ShowCurrentTimestampStatement(); } - - @Override - public Statement visitSetSqlDialect(IoTDBSqlParser.SetSqlDialectContext ctx) { - return new SetSqlDialectStatement( - ctx.TABLE() == null ? IClientSession.SqlDialect.TREE : IClientSession.SqlDialect.TABLE); - } - - @Override - public Statement visitShowCurrentSqlDialect(IoTDBSqlParser.ShowCurrentSqlDialectContext ctx) { - return new ShowCurrentSqlDialectStatement(); - } - - @Override - public Statement visitShowCurrentUser(IoTDBSqlParser.ShowCurrentUserContext ctx) { - return new ShowCurrentUserStatement(); - } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java index 860005397dbd..967345567b92 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java @@ -635,7 +635,14 @@ public Operator visitInformationSchemaTableScan( return new InformationSchemaTableScanOperator( operatorContext, node.getPlanNodeId(), - getSupplier(node.getQualifiedObjectName().getObjectName(), dataTypes)); + getSupplier( + node.getQualifiedObjectName().getObjectName(), + dataTypes, + context + .getDriverContext() + .getFragmentInstanceContext() + .getSessionInfo() + .getUserName())); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadTsFilePieceNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadTsFilePieceNode.java index f578148e014b..5d4f02378fc2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadTsFilePieceNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadTsFilePieceNode.java @@ -155,9 +155,9 @@ public static PlanNode deserialize(ByteBuffer buffer) { InputStream stream = new ByteArrayInputStream(buffer.array()); try { ReadWriteIOUtils.readShort(stream); // read PlanNodeType - final File tsFile = new File(ReadWriteIOUtils.readString(stream)); - final LoadTsFilePieceNode pieceNode = new LoadTsFilePieceNode(new PlanNodeId(""), tsFile); - final int tsFileDataSize = ReadWriteIOUtils.readInt(stream); + File tsFile = new File(ReadWriteIOUtils.readString(stream)); + LoadTsFilePieceNode pieceNode = new LoadTsFilePieceNode(new PlanNodeId(""), tsFile); + int tsFileDataSize = ReadWriteIOUtils.readInt(stream); for (int i = 0; i < tsFileDataSize; i++) { TsFileData tsFileData = TsFileData.deserialize(stream); pieceNode.addTsFileData(tsFileData); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java index d4b34dd1eaa2..65ae1c148357 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java @@ -536,18 +536,15 @@ private Scope visitInsert(WrappedInsertStatement insert, Optional scope) InsertBaseStatement innerInsert = insert.getInnerTreeStatement(); innerInsert.semanticCheck(); - innerInsert.toLowerCase(); - innerInsert = AnalyzeUtils.analyzeInsert( context, innerInsert, - () -> SchemaValidator.validate(metadata, insert, context, accessControl), + () -> SchemaValidator.validate(metadata, insert, context), metadata::getOrCreateDataPartition, AnalyzeUtils::computeTableDataPartitionParams, analysis, false); - insert.setInnerTreeStatement(innerInsert); analysis.setScope(insert, ret); @@ -557,13 +554,7 @@ private Scope visitInsert(WrappedInsertStatement insert, Optional scope) @Override protected Scope visitDelete(Delete node, Optional scope) { final Scope ret = Scope.create(); - accessControl.checkCanDeleteFromTable( - sessionContext.getUserName(), - new QualifiedObjectName( - AnalyzeUtils.getDatabaseName(node, queryContext), - node.getTable().getName().getSuffix())); AnalyzeUtils.analyzeDelete(node, queryContext); - analysis.setScope(node, ret); return ret; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java index 8364ea489b36..607b1c371355 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java @@ -541,15 +541,6 @@ && isIntegerNumber(argumentTypes.get(2)))) { + " must have at least two arguments, and first argument must be char type."); } return STRING; - } else if (TableBuiltinScalarFunction.GREATEST.getFunctionName().equalsIgnoreCase(functionName) - || TableBuiltinScalarFunction.LEAST.getFunctionName().equalsIgnoreCase(functionName)) { - if (argumentTypes.size() < 2 || !areAllTypesSameAndComparable(argumentTypes)) { - throw new SemanticException( - "Scalar function " - + functionName.toLowerCase(Locale.ENGLISH) - + " must have at least two arguments, and all type must be the same."); - } - return argumentTypes.get(0); } // builtin aggregation function @@ -893,17 +884,6 @@ public static boolean isTwoTypeComparable(List argumentTypes) { || ((isNumericType(left) || isCharType(left)) && isUnknownType(right)); } - public static boolean areAllTypesSameAndComparable(List argumentTypes) { - if (argumentTypes == null || argumentTypes.isEmpty()) { - return true; - } - Type firstType = argumentTypes.get(0); - if (!firstType.isComparable()) { - return false; - } - return argumentTypes.stream().allMatch(type -> type.equals(firstType)); - } - public static boolean isArithmeticType(Type type) { return INT32.equals(type) || INT64.equals(type) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCache.java index 579f8192f4b6..195e6e896ced 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCache.java @@ -508,18 +508,6 @@ long getRequestCount() { return dualKeyCache.stats().requestCount(); } - long getMemoryUsage() { - return dualKeyCache.stats().memoryUsage(); - } - - long capacity() { - return dualKeyCache.stats().capacity(); - } - - long entriesCount() { - return dualKeyCache.stats().entriesCount(); - } - void invalidateLastCache(final @Nonnull String database) { readWriteLock.writeLock().lock(); @@ -680,11 +668,6 @@ public void invalidateTreeSchema() { } public void invalidateAll() { - readWriteLock.writeLock().lock(); - try { - dualKeyCache.invalidateAll(); - } finally { - readWriteLock.writeLock().unlock(); - } + dualKeyCache.invalidateAll(); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCacheMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCacheMetrics.java index e65c418c0c1d..56e1b23de456 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCacheMetrics.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCacheMetrics.java @@ -29,10 +29,6 @@ import java.util.Objects; public class TableDeviceSchemaCacheMetrics implements IMetricSet { - - private static final String SCHEMA_CACHE_TOTAL_USAGE = "schema_cache_total_usage"; - private static final String SCHEMA_CACHE_MEM_CAPACITY = "schema_cache_mem_capacity"; - private final TableDeviceSchemaCache tableDeviceSchemaCache; public TableDeviceSchemaCacheMetrics(final TableDeviceSchemaCache dataNodeSchemaCache) { @@ -59,25 +55,6 @@ public void bindTo(final AbstractMetricService metricService) { "SchemaCache", Tag.TYPE.toString(), "all"); - metricService.createAutoGauge( - Metric.MEM.toString(), - MetricLevel.IMPORTANT, - tableDeviceSchemaCache, - TableDeviceSchemaCache::getMemoryUsage, - Tag.NAME.toString(), - SCHEMA_CACHE_TOTAL_USAGE); - metricService.createAutoGauge( - Metric.MEM.toString(), - MetricLevel.IMPORTANT, - tableDeviceSchemaCache, - TableDeviceSchemaCache::capacity, - Tag.NAME.toString(), - SCHEMA_CACHE_MEM_CAPACITY); - metricService.createAutoGauge( - Metric.CACHE_ENTRIES_NUM.toString(), - MetricLevel.IMPORTANT, - tableDeviceSchemaCache, - TableDeviceSchemaCache::entriesCount); } @Override @@ -96,17 +73,6 @@ public void unbindFrom(final AbstractMetricService metricService) { "SchemaCache", Tag.TYPE.toString(), "all"); - metricService.remove( - MetricType.AUTO_GAUGE, - Metric.MEM.toString(), - Tag.NAME.toString(), - SCHEMA_CACHE_TOTAL_USAGE); - metricService.remove( - MetricType.AUTO_GAUGE, - Metric.MEM.toString(), - Tag.NAME.toString(), - SCHEMA_CACHE_MEM_CAPACITY); - metricService.remove(MetricType.AUTO_GAUGE, Metric.CACHE_ENTRIES_NUM.toString()); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/DataNodeLocationSupplierFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/DataNodeLocationSupplierFactory.java index c8d92110d7c5..b91c9eecaf42 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/DataNodeLocationSupplierFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/DataNodeLocationSupplierFactory.java @@ -87,20 +87,14 @@ private static InformationSchemaTableDataNodeLocationSupplier getInstance() { @Override public List getDataNodeLocations(final String tableName) { - switch (tableName) { - case InformationSchema.QUERIES: - return getReadableDataNodeLocations(); - case InformationSchema.DATABASES: - case InformationSchema.TABLES: - case InformationSchema.COLUMNS: - case InformationSchema.REGIONS: - case InformationSchema.PIPES: - case InformationSchema.PIPE_PLUGINS: - case InformationSchema.TOPICS: - case InformationSchema.SUBSCRIPTIONS: - return Collections.singletonList(DataNodeEndPoints.getLocalDataNodeLocation()); - default: - throw new UnsupportedOperationException("Unknown table: " + tableName); + if (tableName.equals(InformationSchema.QUERIES)) { + return getReadableDataNodeLocations(); + } else if (tableName.equals(InformationSchema.DATABASES) + || tableName.equals(InformationSchema.TABLES) + || tableName.equals(InformationSchema.COLUMNS)) { + return Collections.singletonList(DataNodeEndPoints.getLocalDataNodeLocation()); + } else { + throw new UnsupportedOperationException("Unknown table: " + tableName); } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/security/AccessControl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/security/AccessControl.java index a9724b2636cd..ad6494ab582b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/security/AccessControl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/security/AccessControl.java @@ -140,12 +140,4 @@ public interface AccessControl { */ void checkUserCanRunRelationalAuthorStatement( String userName, RelationalAuthorStatement statement); - - /** - * Check if user is admin user - * - * @param userName name of user - * @throws AccessDeniedException if not allowed - */ - void checkUserIsAdmin(String userName); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/security/AccessControlImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/security/AccessControlImpl.java index cafdd13b138f..fd9fd8012f25 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/security/AccessControlImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/security/AccessControlImpl.java @@ -21,13 +21,12 @@ import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.auth.entity.PrivilegeType; -import org.apache.iotdb.commons.exception.auth.AccessDeniedException; +import org.apache.iotdb.commons.exception.IoTDBException; import org.apache.iotdb.db.auth.AuthorityChecker; import org.apache.iotdb.db.queryengine.plan.relational.metadata.QualifiedObjectName; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.RelationalAuthorStatement; import org.apache.iotdb.db.queryengine.plan.relational.type.AuthorRType; - -import static org.apache.iotdb.db.auth.AuthorityChecker.ONLY_ADMIN_ALLOWED; +import org.apache.iotdb.rpc.TSStatusCode; public class AccessControlImpl implements AccessControl { @@ -106,7 +105,10 @@ public void checkUserCanRunRelationalAuthorStatement( case CREATE_USER: // admin cannot be created. if (AuthorityChecker.SUPER_USER.equals(statement.getUserName())) { - throw new AccessDeniedException("Cannot create user has same name with admin user"); + status = + AuthorityChecker.getTSStatus( + false, "Cannot create user has same name with admin user"); + throw new RuntimeException(new IoTDBException(status.getMessage(), status.getCode())); } if (AuthorityChecker.SUPER_USER.equals(userName)) { return; @@ -116,7 +118,8 @@ public void checkUserCanRunRelationalAuthorStatement( case DROP_USER: if (AuthorityChecker.SUPER_USER.equals(statement.getUserName()) || statement.getUserName().equals(userName)) { - throw new AccessDeniedException("Cannot drop admin user or yourself"); + status = AuthorityChecker.getTSStatus(false, "Cannot drop admin user or yourself"); + throw new RuntimeException(new IoTDBException(status.getMessage(), status.getCode())); } if (AuthorityChecker.SUPER_USER.equals(userName)) { return; @@ -139,7 +142,10 @@ public void checkUserCanRunRelationalAuthorStatement( return; case CREATE_ROLE: if (AuthorityChecker.SUPER_USER.equals(statement.getRoleName())) { - throw new AccessDeniedException("Cannot create role has same name with admin user"); + throw new RuntimeException( + new IoTDBException( + "Cannot create role has same name with admin user", + TSStatusCode.NO_PERMISSION.getStatusCode())); } if (AuthorityChecker.SUPER_USER.equals(userName)) { return; @@ -149,7 +155,9 @@ public void checkUserCanRunRelationalAuthorStatement( case DROP_ROLE: if (AuthorityChecker.SUPER_USER.equals(statement.getUserName())) { - throw new AccessDeniedException("Cannot drop role with admin name"); + throw new RuntimeException( + new IoTDBException( + "Cannot drop role with admin name", TSStatusCode.NO_PERMISSION.getStatusCode())); } if (AuthorityChecker.SUPER_USER.equals(userName)) { return; @@ -159,7 +167,9 @@ public void checkUserCanRunRelationalAuthorStatement( case GRANT_USER_ROLE: if (AuthorityChecker.SUPER_USER.equals(statement.getUserName())) { - throw new AccessDeniedException("Cannot grant role to admin"); + throw new RuntimeException( + new IoTDBException( + "Cannot grant role to admin", TSStatusCode.NO_PERMISSION.getStatusCode())); } if (AuthorityChecker.SUPER_USER.equals(userName)) { return; @@ -169,7 +179,9 @@ public void checkUserCanRunRelationalAuthorStatement( case REVOKE_USER_ROLE: if (AuthorityChecker.SUPER_USER.equals(statement.getUserName())) { - throw new AccessDeniedException("Cannot revoke role from admin"); + throw new RuntimeException( + new IoTDBException( + "Cannot revoke role from admin", TSStatusCode.NO_PERMISSION.getStatusCode())); } if (AuthorityChecker.SUPER_USER.equals(userName)) { return; @@ -196,7 +208,10 @@ public void checkUserCanRunRelationalAuthorStatement( case REVOKE_ROLE_ANY: case REVOKE_USER_ANY: if (AuthorityChecker.SUPER_USER.equals(statement.getUserName())) { - throw new AccessDeniedException("Cannot grant/revoke privileges of admin user"); + throw new RuntimeException( + new IoTDBException( + "Cannot grant/revoke privileges to/from admin", + TSStatusCode.NO_PERMISSION.getStatusCode())); } if (AuthorityChecker.SUPER_USER.equals(userName)) { return; @@ -211,7 +226,10 @@ public void checkUserCanRunRelationalAuthorStatement( case GRANT_USER_ALL: case REVOKE_USER_ALL: if (AuthorityChecker.SUPER_USER.equals(statement.getUserName())) { - throw new AccessDeniedException("Cannot grant/revoke all privileges of admin user"); + throw new RuntimeException( + new IoTDBException( + "Cannot grant/revoke all privileges to/from admin", + TSStatusCode.NO_PERMISSION.getStatusCode())); } if (AuthorityChecker.SUPER_USER.equals(userName)) { return; @@ -230,7 +248,10 @@ public void checkUserCanRunRelationalAuthorStatement( case REVOKE_USER_DB: case REVOKE_ROLE_DB: if (AuthorityChecker.SUPER_USER.equals(statement.getUserName())) { - throw new AccessDeniedException("Cannot grant/revoke privileges of admin user"); + throw new RuntimeException( + new IoTDBException( + "Cannot grant/revoke privileges of admin user", + TSStatusCode.NO_PERMISSION.getStatusCode())); } if (AuthorityChecker.SUPER_USER.equals(userName)) { return; @@ -247,7 +268,10 @@ public void checkUserCanRunRelationalAuthorStatement( case REVOKE_USER_TB: case REVOKE_ROLE_TB: if (AuthorityChecker.SUPER_USER.equals(statement.getUserName())) { - throw new AccessDeniedException("Cannot grant/revoke privileges of admin user"); + throw new RuntimeException( + new IoTDBException( + "Cannot grant/revoke privileges of admin user", + TSStatusCode.NO_PERMISSION.getStatusCode())); } if (AuthorityChecker.SUPER_USER.equals(userName)) { return; @@ -265,7 +289,10 @@ public void checkUserCanRunRelationalAuthorStatement( case REVOKE_USER_SYS: case REVOKE_ROLE_SYS: if (AuthorityChecker.SUPER_USER.equals(statement.getUserName())) { - throw new AccessDeniedException("Cannot grant/revoke privileges of admin user"); + throw new RuntimeException( + new IoTDBException( + "Cannot grant/revoke privileges of admin user", + TSStatusCode.NO_PERMISSION.getStatusCode())); } if (AuthorityChecker.SUPER_USER.equals(userName)) { return; @@ -278,11 +305,4 @@ public void checkUserCanRunRelationalAuthorStatement( break; } } - - @Override - public void checkUserIsAdmin(String userName) { - if (!AuthorityChecker.SUPER_USER.equals(userName)) { - throw new AccessDeniedException(ONLY_ADMIN_ALLOWED); - } - } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/security/AllowAllAccessControl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/security/AllowAllAccessControl.java index 7843e1601e51..cea0393c7dcc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/security/AllowAllAccessControl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/security/AllowAllAccessControl.java @@ -88,9 +88,4 @@ public void checkUserCanRunRelationalAuthorStatement( String userName, RelationalAuthorStatement statement) { // allow anything } - - @Override - public void checkUserIsAdmin(String userName) { - // allow anything - } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java index efd4e59fa141..ed67e02098de 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java @@ -433,14 +433,6 @@ protected R visitStopRepairData(StopRepairData node, C context) { return visitStatement(node, context); } - protected R visitLoadConfiguration(LoadConfiguration node, C context) { - return visitStatement(node, context); - } - - protected R visitSetSystemStatus(SetSystemStatus node, C context) { - return visitStatement(node, context); - } - protected R visitInsertRow(InsertRow node, C context) { return visitStatement(node, context); } @@ -644,24 +636,4 @@ protected R visitKillQuery(KillQuery node, C context) { protected R visitRelationalAuthorPlan(RelationalAuthorStatement node, C context) { return visitStatement(node, context); } - - protected R visitMigrateRegion(MigrateRegion node, C context) { - return visitStatement(node, context); - } - - protected R visitReconstructRegion(ReconstructRegion node, C context) { - return visitStatement(node, context); - } - - protected R visitExtendRegion(ExtendRegion node, C context) { - return visitStatement(node, context); - } - - protected R visitRemoveRegion(RemoveRegion node, C context) { - return visitStatement(node, context); - } - - protected R visitSetSqlDialect(SetSqlDialect node, C context) { - return visitStatement(node, context); - } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ExtendRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ExtendRegion.java deleted file mode 100644 index 687777c85588..000000000000 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ExtendRegion.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; - -import com.google.common.collect.ImmutableList; - -import java.util.List; -import java.util.Objects; - -public class ExtendRegion extends Statement { - - private final int regionId; - private final int dataNodeId; - - public ExtendRegion(int regionId, int dataNodeId) { - super(null); - this.regionId = regionId; - this.dataNodeId = dataNodeId; - } - - @Override - public List getChildren() { - return ImmutableList.of(); - } - - @Override - public int hashCode() { - return Objects.hash(ExtendRegion.class, regionId, dataNodeId); - } - - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (!(obj instanceof ExtendRegion)) { - return false; - } - ExtendRegion another = (ExtendRegion) obj; - return regionId == another.regionId && dataNodeId == another.dataNodeId; - } - - @Override - public String toString() { - return String.format("extend region %d to datanode %d", regionId, dataNodeId); - } - - @Override - public R accept(AstVisitor visitor, C context) { - return visitor.visitExtendRegion(this, context); - } - - public int getRegionId() { - return regionId; - } - - public int getDataNodeId() { - return dataNodeId; - } -} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertRows.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertRows.java index c55c02da0573..cdfb4c3c39f3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertRows.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertRows.java @@ -61,7 +61,7 @@ public void updateAfterSchemaValidation(MPPQueryContext context) throws QueryPro @Override public String getTableName() { - return getInnerTreeStatement().getInsertRowStatementList().get(0).getDevicePath().getFullPath(); + return getInnerTreeStatement().getDevicePath().getFullPath(); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/LoadConfiguration.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/LoadConfiguration.java deleted file mode 100644 index b67ed2b3acf9..000000000000 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/LoadConfiguration.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; - -import org.apache.iotdb.db.queryengine.common.MPPQueryContext; -import org.apache.iotdb.db.queryengine.plan.statement.Statement; - -public class LoadConfiguration extends WrappedStatement { - public LoadConfiguration(Statement innerTreeStatement, MPPQueryContext context) { - super(innerTreeStatement, context); - } - - @Override - public R accept(AstVisitor visitor, C context) { - return visitor.visitLoadConfiguration(this, context); - } -} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/MigrateRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/MigrateRegion.java deleted file mode 100644 index 4ec5497f471e..000000000000 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/MigrateRegion.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; - -import com.google.common.collect.ImmutableList; - -import javax.annotation.Nullable; - -import java.util.List; -import java.util.Objects; - -public class MigrateRegion extends Statement { - private final int regionId; - - private final int fromId; - - private final int toId; - - public MigrateRegion(int regionId, int fromId, int toId) { - this(null, regionId, fromId, toId); - } - - public MigrateRegion(@Nullable NodeLocation location, int regionId, int fromId, int toId) { - super(location); - this.regionId = regionId; - this.fromId = fromId; - this.toId = toId; - } - - @Override - public List getChildren() { - return ImmutableList.of(); - } - - @Override - public int hashCode() { - return Objects.hash(MigrateRegion.class, regionId, fromId, toId); - } - - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (!(obj instanceof MigrateRegion)) { - return false; - } - MigrateRegion another = (MigrateRegion) obj; - return regionId == another.regionId && fromId == another.fromId && toId == another.toId; - } - - @Override - public String toString() { - return String.format("migrate region %d from %d to %d", regionId, fromId, toId); - } - - @Override - public R accept(AstVisitor visitor, C context) { - return visitor.visitMigrateRegion(this, context); - } - - public int getRegionId() { - return regionId; - } - - public int getFromId() { - return fromId; - } - - public int getToId() { - return toId; - } -} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ReconstructRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ReconstructRegion.java deleted file mode 100644 index df4aad226064..000000000000 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ReconstructRegion.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; - -import com.google.common.collect.ImmutableList; - -import java.util.List; -import java.util.Objects; - -public class ReconstructRegion extends Statement { - final int dataNodeId; - final List regionIds; - - public ReconstructRegion(int dataNodeId, List regionIds) { - super(null); - this.dataNodeId = dataNodeId; - this.regionIds = regionIds; - } - - @Override - public List getChildren() { - return ImmutableList.of(); - } - - @Override - public int hashCode() { - return Objects.hash(ReconstructRegion.class, dataNodeId, regionIds); - } - - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (!(obj instanceof ReconstructRegion)) { - return false; - } - ReconstructRegion another = (ReconstructRegion) obj; - return dataNodeId == another.dataNodeId && regionIds.equals(another.regionIds); - } - - @Override - public String toString() { - return String.format("reconstruct region %s of datanode %d", regionIds, dataNodeId); - } - - @Override - public R accept(AstVisitor visitor, C context) { - return visitor.visitReconstructRegion(this, context); - } - - public int getDataNodeId() { - return dataNodeId; - } - - public List getRegionIds() { - return regionIds; - } -} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/RemoveRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/RemoveRegion.java deleted file mode 100644 index 8c14433167a8..000000000000 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/RemoveRegion.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; - -import com.google.common.collect.ImmutableList; - -import java.util.List; -import java.util.Objects; - -public class RemoveRegion extends Statement { - - private final int regionId; - private final int dataNodeId; - - public RemoveRegion(int regionId, int dataNodeId) { - super(null); - this.regionId = regionId; - this.dataNodeId = dataNodeId; - } - - @Override - public List getChildren() { - return ImmutableList.of(); - } - - @Override - public int hashCode() { - return Objects.hash(RemoveRegion.class, regionId, dataNodeId); - } - - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (!(obj instanceof RemoveRegion)) { - return false; - } - RemoveRegion another = (RemoveRegion) obj; - return regionId == another.regionId && dataNodeId == another.dataNodeId; - } - - @Override - public String toString() { - return String.format("remove region %d from %d", regionId, dataNodeId); - } - - @Override - public R accept(AstVisitor visitor, C context) { - return visitor.visitRemoveRegion(this, context); - } - - public int getRegionId() { - return regionId; - } - - public int getDataNodeId() { - return dataNodeId; - } -} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SetSqlDialect.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SetSqlDialect.java deleted file mode 100644 index 5f4092d5c31b..000000000000 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SetSqlDialect.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; - -import org.apache.iotdb.db.protocol.session.IClientSession; - -import com.google.common.collect.ImmutableList; - -import javax.annotation.Nullable; - -import java.util.List; -import java.util.Objects; - -public class SetSqlDialect extends Statement { - private final IClientSession.SqlDialect sqlDialect; - - public SetSqlDialect(IClientSession.SqlDialect sqlDialect, @Nullable NodeLocation location) { - super(location); - this.sqlDialect = sqlDialect; - } - - public IClientSession.SqlDialect getSqlDialect() { - return sqlDialect; - } - - @Override - public List getChildren() { - return ImmutableList.of(); - } - - @Override - public R accept(AstVisitor visitor, C context) { - return visitor.visitSetSqlDialect(this, context); - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - SetSqlDialect other = (SetSqlDialect) o; - return Objects.equals(sqlDialect, other.sqlDialect); - } - - @Override - public int hashCode() { - return Objects.hash(sqlDialect); - } - - @Override - public boolean shallowEquals(Node other) { - return sameClass(this, other); - } - - @Override - public String toString() { - return "SET SQL_DIALECT=" + sqlDialect; - } -} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SetSystemStatus.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SetSystemStatus.java deleted file mode 100644 index c767b67c96f9..000000000000 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SetSystemStatus.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; - -import org.apache.iotdb.db.queryengine.common.MPPQueryContext; -import org.apache.iotdb.db.queryengine.plan.statement.Statement; - -public class SetSystemStatus extends WrappedStatement { - public SetSystemStatus(Statement innerTreeStatement, MPPQueryContext context) { - super(innerTreeStatement, context); - } - - @Override - public R accept(AstVisitor visitor, C context) { - return visitor.visitSetSystemStatus(this, context); - } -} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java index 82ff69b564e1..27465268674c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java @@ -21,7 +21,6 @@ import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType; import org.apache.iotdb.commons.auth.entity.PrivilegeType; -import org.apache.iotdb.commons.cluster.NodeStatus; import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.schema.cache.CacheClearOptions; @@ -83,7 +82,6 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Explain; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ExplainAnalyze; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ExtendRegion; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Fill; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Flush; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FunctionCall; @@ -107,11 +105,9 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LikePredicate; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Limit; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Literal; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LoadConfiguration; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LoadTsFile; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LogicalExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LongLiteral; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.MigrateRegion; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.NaturalJoin; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Node; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.NodeLocation; @@ -128,11 +124,9 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Query; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.QueryBody; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.QuerySpecification; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ReconstructRegion; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Relation; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.RelationalAuthorStatement; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.RemoveDataNode; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.RemoveRegion; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.RenameColumn; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.RenameTable; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Row; @@ -141,8 +135,6 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SelectItem; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SetConfiguration; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SetProperties; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SetSqlDialect; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SetSystemStatus; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowAINodes; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowCluster; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowClusterId; @@ -195,9 +187,7 @@ import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowsStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.FlushStatement; -import org.apache.iotdb.db.queryengine.plan.statement.sys.LoadConfigurationStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.SetConfigurationStatement; -import org.apache.iotdb.db.queryengine.plan.statement.sys.SetSystemStatusStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.StartRepairDataStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.StopRepairDataStatement; import org.apache.iotdb.db.relational.grammar.sql.RelationalSqlBaseVisitor; @@ -1151,34 +1141,7 @@ public Node visitShowSeriesSlotListStatement( @Override public Node visitMigrateRegionStatement(RelationalSqlParser.MigrateRegionStatementContext ctx) { - return new MigrateRegion( - Integer.parseInt(ctx.regionId.getText()), - Integer.parseInt(ctx.fromId.getText()), - Integer.parseInt(ctx.toId.getText())); - } - - @Override - public Node visitReconstructRegionStatement( - RelationalSqlParser.ReconstructRegionStatementContext ctx) { - int dataNodeId = Integer.parseInt(ctx.targetDataNodeId.getText()); - List regionIds = - ctx.regionIds.stream() - .map(Token::getText) - .map(Integer::parseInt) - .collect(Collectors.toList()); - return new ReconstructRegion(dataNodeId, regionIds); - } - - @Override - public Node visitExtendRegionStatement(RelationalSqlParser.ExtendRegionStatementContext ctx) { - return new ExtendRegion( - Integer.parseInt(ctx.regionId.getText()), Integer.parseInt(ctx.targetDataNodeId.getText())); - } - - @Override - public Node visitRemoveRegionStatement(RelationalSqlParser.RemoveRegionStatementContext ctx) { - return new RemoveRegion( - Integer.parseInt(ctx.regionId.getText()), Integer.parseInt(ctx.targetDataNodeId.getText())); + return super.visitMigrateRegionStatement(ctx); } @Override @@ -1236,17 +1199,7 @@ public Node visitClearCacheStatement(final RelationalSqlParser.ClearCacheStateme @Override public Node visitSetSystemStatusStatement( RelationalSqlParser.SetSystemStatusStatementContext ctx) { - SetSystemStatusStatement setSystemStatusStatement = new SetSystemStatusStatement(); - setSystemStatusStatement.setOnCluster( - ctx.localOrClusterMode() == null || ctx.localOrClusterMode().LOCAL() == null); - if (ctx.RUNNING() != null) { - setSystemStatusStatement.setStatus(NodeStatus.Running); - } else if (ctx.READONLY() != null) { - setSystemStatusStatement.setStatus(NodeStatus.ReadOnly); - } else { - throw new SemanticException("Unknown system status in set system command."); - } - return new SetSystemStatus(setSystemStatusStatement, null); + return super.visitSetSystemStatusStatement(ctx); } @Override @@ -1260,13 +1213,6 @@ public Node visitShowCurrentSqlDialectStatement( return new ShowCurrentSqlDialect(getLocation(ctx)); } - @Override - public Node visitSetSqlDialectStatement(RelationalSqlParser.SetSqlDialectStatementContext ctx) { - return new SetSqlDialect( - ctx.TABLE() == null ? IClientSession.SqlDialect.TREE : IClientSession.SqlDialect.TABLE, - getLocation(ctx)); - } - @Override public Node visitShowCurrentDatabaseStatement( RelationalSqlParser.ShowCurrentDatabaseStatementContext ctx) { @@ -1331,11 +1277,7 @@ public Node visitKillQueryStatement(RelationalSqlParser.KillQueryStatementContex @Override public Node visitLoadConfigurationStatement( RelationalSqlParser.LoadConfigurationStatementContext ctx) { - LoadConfigurationStatement loadConfigurationStatement = - new LoadConfigurationStatement(StatementType.LOAD_CONFIGURATION); - loadConfigurationStatement.setOnCluster( - ctx.localOrClusterMode() == null || ctx.localOrClusterMode().LOCAL() == null); - return new LoadConfiguration(loadConfigurationStatement, null); + return super.visitLoadConfigurationStatement(ctx); } @Override @@ -1370,8 +1312,7 @@ public Node visitStartRepairDataStatement( RelationalSqlParser.StartRepairDataStatementContext ctx) { StartRepairDataStatement startRepairDataStatement = new StartRepairDataStatement(StatementType.START_REPAIR_DATA); - startRepairDataStatement.setOnCluster( - ctx.localOrClusterMode() == null || ctx.localOrClusterMode().LOCAL() == null); + startRepairDataStatement.setOnCluster(ctx.localOrClusterMode().LOCAL() == null); return new StartRepairData(startRepairDataStatement, null); } @@ -1379,8 +1320,7 @@ public Node visitStartRepairDataStatement( public Node visitStopRepairDataStatement(RelationalSqlParser.StopRepairDataStatementContext ctx) { StopRepairDataStatement stopRepairDataStatement = new StopRepairDataStatement(StatementType.STOP_REPAIR_DATA); - stopRepairDataStatement.setOnCluster( - ctx.localOrClusterMode() == null || ctx.localOrClusterMode().LOCAL() == null); + stopRepairDataStatement.setOnCluster(ctx.localOrClusterMode().LOCAL() == null); return new StopRepairData(stopRepairDataStatement, null); } @@ -1485,22 +1425,12 @@ public Node visitListRolePrivilegeStatement( @Override public Node visitListUserStatement(RelationalSqlParser.ListUserStatementContext ctx) { - RelationalAuthorStatement stmt = new RelationalAuthorStatement(AuthorRType.LIST_USER); - if (ctx.OF() != null) { - String roleName = ctx.roleName.getText(); - stmt.setRoleName(roleName); - } - return stmt; + return new RelationalAuthorStatement(AuthorRType.LIST_USER); } @Override public Node visitListRoleStatement(RelationalSqlParser.ListRoleStatementContext ctx) { - RelationalAuthorStatement stmt = new RelationalAuthorStatement(AuthorRType.LIST_ROLE); - if (ctx.OF() != null) { - String userName = ctx.userName.getText(); - stmt.setUserName(userName); - } - return stmt; + return new RelationalAuthorStatement(AuthorRType.LIST_ROLE); } private Set parseSystemPrivilege(RelationalSqlParser.SystemPrivilegesContext ctx) { @@ -1514,18 +1444,10 @@ private Set parseSystemPrivilege(RelationalSqlParser.SystemPrivil private Set parseRelationalPrivilege( RelationalSqlParser.ObjectPrivilegesContext ctx) { + List privilegeContexts = ctx.objectPrivilege(); Set privileges = new HashSet<>(); - if (ctx.ALL() != null) { - for (PrivilegeType privilegeType : PrivilegeType.values()) { - if (privilegeType.isRelationalPrivilege()) { - privileges.add(privilegeType); - } - } - } else { - List privilegeContexts = ctx.objectPrivilege(); - for (RelationalSqlParser.ObjectPrivilegeContext privilege : privilegeContexts) { - privileges.add(PrivilegeType.valueOf(privilege.getText().toUpperCase())); - } + for (RelationalSqlParser.ObjectPrivilegeContext privilege : privilegeContexts) { + privileges.add(PrivilegeType.valueOf(privilege.getText().toUpperCase())); } return privileges; } @@ -1611,7 +1533,7 @@ public Node visitRevokeStatement(RelationalSqlParser.RevokeStatementContext ctx) fromUser = ctx.holderType().getText().equalsIgnoreCase("user"); name = ctx.holderName.getText(); boolean grantOption = ctx.revokeGrantOpt() != null; - boolean fromTable = false; + boolean fromTable; Set privileges = new HashSet<>(); // SYSTEM PRIVILEGES OR ALL PRIVILEGES @@ -1633,48 +1555,38 @@ public Node visitRevokeStatement(RelationalSqlParser.RevokeStatementContext ctx) } } else { privileges = parseRelationalPrivilege(ctx.privilegeObjectScope().objectPrivileges()); - boolean revokeAll = ctx.privilegeObjectScope().objectPrivileges().ALL() != null; - String databaseName = ""; - String tableName = ""; - // ON TABLE / DB if (ctx.privilegeObjectScope().objectType() != null) { fromTable = ctx.privilegeObjectScope().objectType().getText().equalsIgnoreCase("table"); + String databaseName = ""; if (fromTable) { databaseName = clientSession.getDatabaseName(); if (databaseName == null) { throw new SemanticException("Database is not set yet."); } - tableName = ctx.privilegeObjectScope().objectName.getText().toLowerCase(); - } else { - databaseName = ctx.privilegeObjectScope().objectName.getText().toLowerCase(); } + String obj = ctx.privilegeObjectScope().objectName.getText(); + return new RelationalAuthorStatement( + fromUser + ? fromTable ? AuthorRType.REVOKE_USER_TB : AuthorRType.REVOKE_USER_DB + : fromTable ? AuthorRType.REVOKE_ROLE_TB : AuthorRType.REVOKE_ROLE_DB, + fromUser ? name : "", + fromUser ? "" : name, + fromTable ? databaseName.toLowerCase() : obj.toLowerCase(), + fromTable ? obj.toLowerCase() : "", + privileges, + grantOption, + null); } else if (ctx.privilegeObjectScope().objectScope() != null) { - fromTable = true; - databaseName = ctx.privilegeObjectScope().objectScope().dbname.getText().toLowerCase(); - tableName = ctx.privilegeObjectScope().objectScope().tbname.getText().toLowerCase(); - } - - // The REVOKE ALL command can revoke privileges for users, databases, and tables. - // When AuthorRType is REVOKE_USER_ALL: - // If both database and table are empty, it clears all privileges globally. - // If a database is specified (non-empty), it revokes all privileges within that database - // scope. - // If a table is specified (non-empty), it revokes privileges specifically for that table. - // For operations involving the ANY scope, REVOKE_USER_ALL cannot be combined with - // database/table - // specifications. However, since ALL privileges are resolved as concrete privileges in this - // context, - // equivalent effects can be achieved by supplementing with REVOKE_USER_ANY operations. - - if (revokeAll && ctx.privilegeObjectScope().ANY() == null) { + String db = ctx.privilegeObjectScope().objectScope().dbname.getText().toLowerCase(); + String tb = ctx.privilegeObjectScope().objectScope().tbname.getText().toLowerCase(); return new RelationalAuthorStatement( - fromUser ? AuthorRType.REVOKE_USER_ALL : AuthorRType.REVOKE_ROLE_ALL, + fromUser ? AuthorRType.REVOKE_USER_TB : AuthorRType.REVOKE_ROLE_TB, fromUser ? name : "", fromUser ? "" : name, - databaseName, - tableName, - Collections.emptySet(), + db, + tb, + privileges, grantOption, null); } else if (ctx.privilegeObjectScope().ANY() != null) { @@ -1684,20 +1596,10 @@ public Node visitRevokeStatement(RelationalSqlParser.RevokeStatementContext ctx) fromUser ? name : "", fromUser ? "" : name, grantOption); - } else { - return new RelationalAuthorStatement( - fromUser - ? fromTable ? AuthorRType.REVOKE_USER_TB : AuthorRType.REVOKE_USER_DB - : fromTable ? AuthorRType.REVOKE_ROLE_TB : AuthorRType.REVOKE_ROLE_DB, - fromUser ? name : "", - fromUser ? "" : name, - databaseName, - tableName, - privileges, - grantOption, - null); } } + // will not get here. + throw new SemanticException("author statement parser error"); } // ********************** query expressions ******************** diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileScheduler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileScheduler.java index 5a76988163fd..d42ee3616d55 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileScheduler.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileScheduler.java @@ -94,7 +94,6 @@ import java.util.HashSet; import java.util.Iterator; import java.util.List; -import java.util.ListIterator; import java.util.Map; import java.util.Objects; import java.util.Set; @@ -216,7 +215,8 @@ public void start() { long startTime = System.nanoTime(); final boolean isFirstPhaseSuccess; try { - isFirstPhaseSuccess = firstPhase(node); + isFirstPhaseSuccess = + firstPhaseWithRetry(node, CONFIG.getLoadTsFileRetryCountOnRegionChange()); } finally { LOAD_TSFILE_COST_METRICS_SET.recordPhaseTimeCost( LoadTsFileCostMetricsSet.FIRST_PHASE, System.nanoTime() - startTime); @@ -270,23 +270,9 @@ public void start() { if (isLoadSuccess) { stateMachine.transitionToFinished(); } else { - final StringBuilder failedTsFiles = - new StringBuilder( - !tsFileNodeList.isEmpty() - ? tsFileNodeList.get(0).getTsFileResource().getTsFilePath() - : ""); - final ListIterator iterator = failedTsFileNodeIndexes.listIterator(1); - while (iterator.hasNext()) { - failedTsFiles - .append(", ") - .append(tsFileNodeList.get(iterator.next()).getTsFileResource().getTsFilePath()); - } final long startTime = System.nanoTime(); try { // if failed to load some TsFiles, then try to convert the TsFiles to Tablets - LOGGER.info( - "Load TsFile(s) failed, will try to convert to tablets and insert. Failed TsFiles: {}", - failedTsFiles); convertFailedTsFilesToTabletsAndRetry(); } finally { LOAD_TSFILE_COST_METRICS_SET.recordPhaseTimeCost( @@ -298,7 +284,30 @@ public void start() { } } - private boolean firstPhase(LoadSingleTsFileNode node) { + private boolean firstPhaseWithRetry(LoadSingleTsFileNode node, int retryCountOnRegionChange) { + retryCountOnRegionChange = Math.max(0, retryCountOnRegionChange); + while (true) { + try { + return firstPhase(node); + } catch (RegionReplicaSetChangedException e) { + if (retryCountOnRegionChange > 0) { + LOGGER.warn( + "Region replica set changed during loading TsFile {}, maybe due to region migration, will retry for {} times.", + node.getTsFileResource(), + retryCountOnRegionChange); + retryCountOnRegionChange--; + } else { + stateMachine.transitionToFailed(e); + LOGGER.warn( + "Region replica set changed during loading TsFile {} after retry.", + node.getTsFileResource()); + return false; + } + } + } + } + + private boolean firstPhase(LoadSingleTsFileNode node) throws RegionReplicaSetChangedException { final TsFileDataManager tsFileDataManager = new TsFileDataManager(this, node, block); try { new TsFileSplitter( @@ -308,6 +317,8 @@ private boolean firstPhase(LoadSingleTsFileNode node) { stateMachine.transitionToFailed(new TSStatus(TSStatusCode.LOAD_FILE_ERROR.getStatusCode())); return false; } + } catch (RegionReplicaSetChangedException e) { + throw e; } catch (IllegalStateException e) { stateMachine.transitionToFailed(e); LOGGER.warn( @@ -687,7 +698,7 @@ private boolean addOrSendChunkData(ChunkData chunkData) throws LoadFileException dataSize -= pieceNode.getDataSize(); block.reduceMemoryUsage(pieceNode.getDataSize()); - regionId2ReplicaSetAndNode.replace( + regionId2ReplicaSetAndNode.put( sortedRegionId, new Pair<>( replicaSet, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java index 49d8f9f97c2f..333d21aeccd4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java @@ -122,10 +122,7 @@ import org.apache.iotdb.db.queryengine.plan.statement.sys.LoadConfigurationStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.MergeStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.SetConfigurationStatement; -import org.apache.iotdb.db.queryengine.plan.statement.sys.SetSqlDialectStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.SetSystemStatusStatement; -import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowCurrentSqlDialectStatement; -import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowCurrentUserStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowQueriesStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowVersionStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.StartRepairDataStatement; @@ -662,17 +659,4 @@ public R visitShowCurrentTimestamp( ShowCurrentTimestampStatement showCurrentTimestampStatement, C context) { return visitStatement(showCurrentTimestampStatement, context); } - - public R visitSetSqlDialect(SetSqlDialectStatement setSqlDialectStatement, C context) { - return visitStatement(setSqlDialectStatement, context); - } - - public R visitShowCurrentSqlDialect( - ShowCurrentSqlDialectStatement showCurrentSqlDialectStatement, C context) { - return visitStatement(showCurrentSqlDialectStatement, context); - } - - public R visitShowCurrentUser(ShowCurrentUserStatement showCurrentUserStatement, C context) { - return visitStatement(showCurrentUserStatement, context); - } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/sys/SetSqlDialectStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/sys/SetSqlDialectStatement.java deleted file mode 100644 index edb63ef4893b..000000000000 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/sys/SetSqlDialectStatement.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.queryengine.plan.statement.sys; - -import org.apache.iotdb.common.rpc.thrift.TSStatus; -import org.apache.iotdb.commons.path.PartialPath; -import org.apache.iotdb.db.protocol.session.IClientSession; -import org.apache.iotdb.db.queryengine.plan.analyze.QueryType; -import org.apache.iotdb.db.queryengine.plan.statement.IConfigStatement; -import org.apache.iotdb.db.queryengine.plan.statement.Statement; -import org.apache.iotdb.db.queryengine.plan.statement.StatementVisitor; - -import java.util.Collections; -import java.util.List; -import java.util.Objects; - -import static org.apache.iotdb.db.auth.AuthorityChecker.SUCCEED; - -public class SetSqlDialectStatement extends Statement implements IConfigStatement { - private final IClientSession.SqlDialect sqlDialect; - - public SetSqlDialectStatement(IClientSession.SqlDialect sqlDialect) { - this.sqlDialect = sqlDialect; - } - - public IClientSession.SqlDialect getSqlDialect() { - return sqlDialect; - } - - @Override - public int hashCode() { - return Objects.hash(sqlDialect); - } - - @Override - public String toString() { - return "SET SQL_DIALECT=" + sqlDialect; - } - - @Override - public List getPaths() { - return Collections.emptyList(); - } - - @Override - public QueryType getQueryType() { - return QueryType.WRITE; - } - - @Override - public R accept(StatementVisitor visitor, C context) { - return visitor.visitSetSqlDialect(this, context); - } - - @Override - public TSStatus checkPermissionBeforeProcess(final String userName) { - return SUCCEED; - } -} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/sys/ShowCurrentSqlDialectStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/sys/ShowCurrentSqlDialectStatement.java deleted file mode 100644 index 73a40db282f8..000000000000 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/sys/ShowCurrentSqlDialectStatement.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.queryengine.plan.statement.sys; - -import org.apache.iotdb.common.rpc.thrift.TSStatus; -import org.apache.iotdb.db.queryengine.plan.analyze.QueryType; -import org.apache.iotdb.db.queryengine.plan.statement.IConfigStatement; -import org.apache.iotdb.db.queryengine.plan.statement.StatementVisitor; -import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowStatement; - -import static org.apache.iotdb.db.auth.AuthorityChecker.SUCCEED; - -public class ShowCurrentSqlDialectStatement extends ShowStatement implements IConfigStatement { - @Override - public R accept(StatementVisitor visitor, C context) { - return visitor.visitShowCurrentSqlDialect(this, context); - } - - @Override - public TSStatus checkPermissionBeforeProcess(String userName) { - return SUCCEED; - } - - @Override - public QueryType getQueryType() { - return QueryType.READ; - } -} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/sys/ShowCurrentUserStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/sys/ShowCurrentUserStatement.java deleted file mode 100644 index 81650ef21aed..000000000000 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/sys/ShowCurrentUserStatement.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.queryengine.plan.statement.sys; - -import org.apache.iotdb.common.rpc.thrift.TSStatus; -import org.apache.iotdb.db.queryengine.plan.analyze.QueryType; -import org.apache.iotdb.db.queryengine.plan.statement.IConfigStatement; -import org.apache.iotdb.db.queryengine.plan.statement.StatementVisitor; -import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowStatement; - -import static org.apache.iotdb.db.auth.AuthorityChecker.SUCCEED; - -public class ShowCurrentUserStatement extends ShowStatement implements IConfigStatement { - @Override - public R accept(StatementVisitor visitor, C context) { - return visitor.visitShowCurrentUser(this, context); - } - - @Override - public TSStatus checkPermissionBeforeProcess(String userName) { - return SUCCEED; - } - - @Override - public QueryType getQueryType() { - return QueryType.READ; - } -} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/AbstractGreatestLeastColumnTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/AbstractGreatestLeastColumnTransformer.java deleted file mode 100644 index f92c5fa2811e..000000000000 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/AbstractGreatestLeastColumnTransformer.java +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.iotdb.db.queryengine.transformation.dag.column.multi; - -import org.apache.iotdb.db.queryengine.transformation.dag.column.ColumnTransformer; - -import org.apache.tsfile.block.column.Column; -import org.apache.tsfile.block.column.ColumnBuilder; -import org.apache.tsfile.read.common.type.Type; -import org.apache.tsfile.read.common.type.TypeEnum; - -import java.util.List; - -public abstract class AbstractGreatestLeastColumnTransformer extends MultiColumnTransformer { - - protected AbstractGreatestLeastColumnTransformer( - Type returnType, List columnTransformerList) { - super(returnType, columnTransformerList); - } - - @Override - protected void doTransform( - List childrenColumns, ColumnBuilder builder, int positionCount) { - for (int i = 0; i < positionCount; i++) { - transform(builder, childrenColumns, i); - } - } - - @Override - protected void doTransform( - List childrenColumns, ColumnBuilder builder, int positionCount, boolean[] selection) { - for (int i = 0; i < positionCount; i++) { - if (selection[i]) { - transform(builder, childrenColumns, i); - } else { - builder.appendNull(); - } - } - } - - protected abstract void transform(ColumnBuilder builder, List childrenColumns, int index); - - public static ColumnTransformer getGreatestColumnTransformer( - Type type, List columnTransformers) { - TypeEnum typeEnum = type.getTypeEnum(); - switch (typeEnum) { - case BOOLEAN: - return new BooleanGreatestColumnTransformer(type, columnTransformers); - case INT32: - case DATE: - return new Int32GreatestColumnTransformer(type, columnTransformers); - case INT64: - case TIMESTAMP: - return new Int64GreatestColumnTransformer(type, columnTransformers); - case FLOAT: - return new FloatGreatestColumnTransformer(type, columnTransformers); - case DOUBLE: - return new DoubleGreatestColumnTransformer(type, columnTransformers); - case STRING: - case TEXT: - return new BinaryGreatestColumnTransformer(type, columnTransformers); - default: - throw new UnsupportedOperationException( - String.format("Unsupported data type: %s", typeEnum)); - } - } - - public static ColumnTransformer getLeastColumnTransformer( - Type type, List columnTransformers) { - TypeEnum typeEnum = type.getTypeEnum(); - switch (typeEnum) { - case BOOLEAN: - return new BooleanLeastColumnTransformer(type, columnTransformers); - case INT32: - case DATE: - return new Int32LeastColumnTransformer(type, columnTransformers); - case INT64: - case TIMESTAMP: - return new Int64LeastColumnTransformer(type, columnTransformers); - case FLOAT: - return new FloatLeastColumnTransformer(type, columnTransformers); - case DOUBLE: - return new DoubleLeastColumnTransformer(type, columnTransformers); - case STRING: - case TEXT: - return new BinaryLeastColumnTransformer(type, columnTransformers); - default: - throw new UnsupportedOperationException( - String.format("Unsupported data type: %s", typeEnum)); - } - } - - @Override - protected void checkType() { - // do nothing - } -} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/BinaryGreatestColumnTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/BinaryGreatestColumnTransformer.java deleted file mode 100644 index e81487bbac3e..000000000000 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/BinaryGreatestColumnTransformer.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.iotdb.db.queryengine.transformation.dag.column.multi; - -import org.apache.iotdb.db.queryengine.transformation.dag.column.ColumnTransformer; - -import org.apache.tsfile.block.column.Column; -import org.apache.tsfile.block.column.ColumnBuilder; -import org.apache.tsfile.read.common.type.Type; -import org.apache.tsfile.utils.Binary; - -import java.util.List; - -public class BinaryGreatestColumnTransformer extends AbstractGreatestLeastColumnTransformer { - protected BinaryGreatestColumnTransformer( - Type returnType, List columnTransformerList) { - super(returnType, columnTransformerList); - } - - @Override - protected void transform(ColumnBuilder builder, List childrenColumns, int index) { - Column firstColumn = childrenColumns.get(0); - Binary maxValue = firstColumn.getBinary(index); - boolean allNull = firstColumn.isNull(index); - for (int i = 1; i < childrenColumns.size(); i++) { - Column column = childrenColumns.get(i); - if (!column.isNull(index)) { - Binary value = column.getBinary(index); - if (allNull || value.compareTo(maxValue) > 0) { - allNull = false; - maxValue = value; - } - } - } - if (allNull) { - builder.appendNull(); - } else { - returnType.writeBinary(builder, maxValue); - } - } -} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/BinaryLeastColumnTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/BinaryLeastColumnTransformer.java deleted file mode 100644 index 6cde29f4c845..000000000000 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/BinaryLeastColumnTransformer.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.iotdb.db.queryengine.transformation.dag.column.multi; - -import org.apache.iotdb.db.queryengine.transformation.dag.column.ColumnTransformer; - -import org.apache.tsfile.block.column.Column; -import org.apache.tsfile.block.column.ColumnBuilder; -import org.apache.tsfile.read.common.type.Type; -import org.apache.tsfile.utils.Binary; - -import java.util.List; - -public class BinaryLeastColumnTransformer extends AbstractGreatestLeastColumnTransformer { - protected BinaryLeastColumnTransformer( - Type returnType, List columnTransformerList) { - super(returnType, columnTransformerList); - } - - @Override - protected void transform(ColumnBuilder builder, List childrenColumns, int index) { - Column firstColumn = childrenColumns.get(0); - Binary minValue = firstColumn.getBinary(index); - boolean allNull = firstColumn.isNull(index); - for (int i = 1; i < childrenColumns.size(); i++) { - Column column = childrenColumns.get(i); - if (!column.isNull(index)) { - Binary value = column.getBinary(index); - if (allNull || value.compareTo(minValue) < 0) { - allNull = false; - minValue = value; - } - } - } - if (allNull) { - builder.appendNull(); - } else { - returnType.writeBinary(builder, minValue); - } - } -} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/BooleanGreatestColumnTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/BooleanGreatestColumnTransformer.java deleted file mode 100644 index 2bc56e6f5bc9..000000000000 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/BooleanGreatestColumnTransformer.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.iotdb.db.queryengine.transformation.dag.column.multi; - -import org.apache.iotdb.db.queryengine.transformation.dag.column.ColumnTransformer; - -import org.apache.tsfile.block.column.Column; -import org.apache.tsfile.block.column.ColumnBuilder; -import org.apache.tsfile.read.common.type.Type; - -import java.util.List; - -public class BooleanGreatestColumnTransformer extends AbstractGreatestLeastColumnTransformer { - protected BooleanGreatestColumnTransformer( - Type returnType, List columnTransformerList) { - super(returnType, columnTransformerList); - } - - @Override - protected void transform(ColumnBuilder builder, List childrenColumns, int index) { - boolean allNull = true; - for (Column column : childrenColumns) { - if (!column.isNull(index)) { - allNull = false; - if (column.getBoolean(index)) { - // find max value true - returnType.writeBoolean(builder, true); - break; - } - } - } - if (allNull) { - // all value are null return null - builder.appendNull(); - } else { - // do not have true, but have false - returnType.writeBoolean(builder, false); - } - } -} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/BooleanLeastColumnTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/BooleanLeastColumnTransformer.java deleted file mode 100644 index 4ea2b717092c..000000000000 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/BooleanLeastColumnTransformer.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.iotdb.db.queryengine.transformation.dag.column.multi; - -import org.apache.iotdb.db.queryengine.transformation.dag.column.ColumnTransformer; - -import org.apache.tsfile.block.column.Column; -import org.apache.tsfile.block.column.ColumnBuilder; -import org.apache.tsfile.read.common.type.Type; - -import java.util.List; - -public class BooleanLeastColumnTransformer extends AbstractGreatestLeastColumnTransformer { - protected BooleanLeastColumnTransformer( - Type returnType, List columnTransformerList) { - super(returnType, columnTransformerList); - } - - @Override - protected void transform(ColumnBuilder builder, List childrenColumns, int index) { - boolean allNull = true; - for (Column column : childrenColumns) { - if (!column.isNull(index)) { - allNull = false; - if (!column.getBoolean(index)) { - returnType.writeBoolean(builder, false); - break; - } - } - } - if (allNull) { - builder.appendNull(); - } else { - returnType.writeBoolean(builder, true); - } - } -} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/DoubleGreatestColumnTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/DoubleGreatestColumnTransformer.java deleted file mode 100644 index e09f0b6df4b2..000000000000 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/DoubleGreatestColumnTransformer.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.iotdb.db.queryengine.transformation.dag.column.multi; - -import org.apache.iotdb.db.queryengine.transformation.dag.column.ColumnTransformer; - -import org.apache.tsfile.block.column.Column; -import org.apache.tsfile.block.column.ColumnBuilder; -import org.apache.tsfile.read.common.type.Type; - -import java.util.List; - -public class DoubleGreatestColumnTransformer extends AbstractGreatestLeastColumnTransformer { - protected DoubleGreatestColumnTransformer( - Type returnType, List columnTransformerList) { - super(returnType, columnTransformerList); - } - - @Override - protected void transform(ColumnBuilder builder, List childrenColumns, int index) { - Column firstColumn = childrenColumns.get(0); - double maxValue = firstColumn.getDouble(index); - boolean allNull = firstColumn.isNull(index); - for (int i = 1; i < childrenColumns.size(); i++) { - Column column = childrenColumns.get(i); - if (!column.isNull(index)) { - double value = column.getDouble(index); - if (allNull || value > maxValue) { - allNull = false; - maxValue = value; - } - } - } - if (allNull) { - builder.appendNull(); - } else { - returnType.writeDouble(builder, maxValue); - } - } -} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/DoubleLeastColumnTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/DoubleLeastColumnTransformer.java deleted file mode 100644 index 2dc19ec75e7a..000000000000 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/DoubleLeastColumnTransformer.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.iotdb.db.queryengine.transformation.dag.column.multi; - -import org.apache.iotdb.db.queryengine.transformation.dag.column.ColumnTransformer; - -import org.apache.tsfile.block.column.Column; -import org.apache.tsfile.block.column.ColumnBuilder; -import org.apache.tsfile.read.common.type.Type; - -import java.util.List; - -public class DoubleLeastColumnTransformer extends AbstractGreatestLeastColumnTransformer { - protected DoubleLeastColumnTransformer( - Type returnType, List columnTransformerList) { - super(returnType, columnTransformerList); - } - - @Override - protected void transform(ColumnBuilder builder, List childrenColumns, int index) { - Column firstColumn = childrenColumns.get(0); - double minValue = firstColumn.getDouble(index); - boolean allNull = firstColumn.isNull(index); - for (int i = 1; i < childrenColumns.size(); i++) { - Column column = childrenColumns.get(i); - if (!column.isNull(index)) { - double value = column.getDouble(index); - if (allNull || value < minValue) { - allNull = false; - minValue = value; - } - } - } - if (allNull) { - builder.appendNull(); - } else { - returnType.writeDouble(builder, minValue); - } - } -} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/FloatGreatestColumnTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/FloatGreatestColumnTransformer.java deleted file mode 100644 index bc5bfc982f8c..000000000000 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/FloatGreatestColumnTransformer.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.iotdb.db.queryengine.transformation.dag.column.multi; - -import org.apache.iotdb.db.queryengine.transformation.dag.column.ColumnTransformer; - -import org.apache.tsfile.block.column.Column; -import org.apache.tsfile.block.column.ColumnBuilder; -import org.apache.tsfile.read.common.type.Type; - -import java.util.List; - -public class FloatGreatestColumnTransformer extends AbstractGreatestLeastColumnTransformer { - protected FloatGreatestColumnTransformer( - Type returnType, List columnTransformerList) { - super(returnType, columnTransformerList); - } - - @Override - protected void transform(ColumnBuilder builder, List childrenColumns, int index) { - Column firstColumn = childrenColumns.get(0); - float maxValue = firstColumn.getFloat(index); - boolean allNull = firstColumn.isNull(index); - for (int i = 1; i < childrenColumns.size(); i++) { - Column column = childrenColumns.get(i); - if (!column.isNull(index)) { - float value = column.getFloat(index); - if (allNull || value > maxValue) { - allNull = false; - maxValue = value; - } - } - } - if (allNull) { - builder.appendNull(); - } else { - returnType.writeFloat(builder, maxValue); - } - } -} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/FloatLeastColumnTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/FloatLeastColumnTransformer.java deleted file mode 100644 index 223f63673390..000000000000 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/FloatLeastColumnTransformer.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.iotdb.db.queryengine.transformation.dag.column.multi; - -import org.apache.iotdb.db.queryengine.transformation.dag.column.ColumnTransformer; - -import org.apache.tsfile.block.column.Column; -import org.apache.tsfile.block.column.ColumnBuilder; -import org.apache.tsfile.read.common.type.Type; - -import java.util.List; - -public class FloatLeastColumnTransformer extends AbstractGreatestLeastColumnTransformer { - protected FloatLeastColumnTransformer( - Type returnType, List columnTransformerList) { - super(returnType, columnTransformerList); - } - - @Override - protected void transform(ColumnBuilder builder, List childrenColumns, int index) { - Column firstColumn = childrenColumns.get(0); - float minValue = firstColumn.getFloat(index); - boolean allNull = firstColumn.isNull(index); - for (int i = 1; i < childrenColumns.size(); i++) { - Column column = childrenColumns.get(i); - if (!column.isNull(index)) { - float value = column.getFloat(index); - if (allNull || value < minValue) { - allNull = false; - minValue = value; - } - } - } - if (allNull) { - builder.appendNull(); - } else { - returnType.writeFloat(builder, minValue); - } - } -} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/Int32GreatestColumnTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/Int32GreatestColumnTransformer.java deleted file mode 100644 index 551970f470a6..000000000000 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/Int32GreatestColumnTransformer.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.iotdb.db.queryengine.transformation.dag.column.multi; - -import org.apache.iotdb.db.queryengine.transformation.dag.column.ColumnTransformer; - -import org.apache.tsfile.block.column.Column; -import org.apache.tsfile.block.column.ColumnBuilder; -import org.apache.tsfile.read.common.type.Type; - -import java.util.List; - -public class Int32GreatestColumnTransformer extends AbstractGreatestLeastColumnTransformer { - protected Int32GreatestColumnTransformer( - Type returnType, List columnTransformerList) { - super(returnType, columnTransformerList); - } - - @Override - protected void transform(ColumnBuilder builder, List childrenColumns, int index) { - Column firstColumn = childrenColumns.get(0); - int maxValue = firstColumn.getInt(index); - boolean allNull = firstColumn.isNull(index); - for (int i = 1; i < childrenColumns.size(); i++) { - Column column = childrenColumns.get(i); - if (!column.isNull(index)) { - int value = column.getInt(index); - if (allNull || value > maxValue) { - allNull = false; - maxValue = value; - } - } - } - if (allNull) { - builder.appendNull(); - } else { - returnType.writeInt(builder, maxValue); - } - } -} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/Int32LeastColumnTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/Int32LeastColumnTransformer.java deleted file mode 100644 index 26b64349fb4f..000000000000 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/Int32LeastColumnTransformer.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.iotdb.db.queryengine.transformation.dag.column.multi; - -import org.apache.iotdb.db.queryengine.transformation.dag.column.ColumnTransformer; - -import org.apache.tsfile.block.column.Column; -import org.apache.tsfile.block.column.ColumnBuilder; -import org.apache.tsfile.read.common.type.Type; - -import java.util.List; - -public class Int32LeastColumnTransformer extends AbstractGreatestLeastColumnTransformer { - protected Int32LeastColumnTransformer( - Type returnType, List columnTransformerList) { - super(returnType, columnTransformerList); - } - - @Override - protected void transform(ColumnBuilder builder, List childrenColumns, int index) { - Column firstColumn = childrenColumns.get(0); - int minValue = firstColumn.getInt(index); - boolean allNull = firstColumn.isNull(index); - for (int i = 1; i < childrenColumns.size(); i++) { - Column column = childrenColumns.get(i); - if (!column.isNull(index)) { - int value = column.getInt(index); - if (allNull || value < minValue) { - allNull = false; - minValue = value; - } - } - } - if (allNull) { - builder.appendNull(); - } else { - returnType.writeInt(builder, minValue); - } - } -} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/Int64GreatestColumnTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/Int64GreatestColumnTransformer.java deleted file mode 100644 index 7818d217d0aa..000000000000 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/Int64GreatestColumnTransformer.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.iotdb.db.queryengine.transformation.dag.column.multi; - -import org.apache.iotdb.db.queryengine.transformation.dag.column.ColumnTransformer; - -import org.apache.tsfile.block.column.Column; -import org.apache.tsfile.block.column.ColumnBuilder; -import org.apache.tsfile.read.common.type.Type; - -import java.util.List; - -public class Int64GreatestColumnTransformer extends AbstractGreatestLeastColumnTransformer { - protected Int64GreatestColumnTransformer( - Type returnType, List columnTransformerList) { - super(returnType, columnTransformerList); - } - - @Override - protected void transform(ColumnBuilder builder, List childrenColumns, int index) { - Column firstColumn = childrenColumns.get(0); - long maxValue = firstColumn.getLong(index); - boolean allNull = firstColumn.isNull(index); - for (int i = 1; i < childrenColumns.size(); i++) { - Column column = childrenColumns.get(i); - if (!column.isNull(index)) { - long value = column.getLong(index); - if (allNull || value > maxValue) { - allNull = false; - maxValue = value; - } - } - } - if (allNull) { - builder.appendNull(); - } else { - returnType.writeLong(builder, maxValue); - } - } -} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/Int64LeastColumnTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/Int64LeastColumnTransformer.java deleted file mode 100644 index 0efaccb37869..000000000000 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/Int64LeastColumnTransformer.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.iotdb.db.queryengine.transformation.dag.column.multi; - -import org.apache.iotdb.db.queryengine.transformation.dag.column.ColumnTransformer; - -import org.apache.tsfile.block.column.Column; -import org.apache.tsfile.block.column.ColumnBuilder; -import org.apache.tsfile.read.common.type.Type; - -import java.util.List; - -public class Int64LeastColumnTransformer extends AbstractGreatestLeastColumnTransformer { - protected Int64LeastColumnTransformer( - Type returnType, List columnTransformerList) { - super(returnType, columnTransformerList); - } - - @Override - protected void transform(ColumnBuilder builder, List childrenColumns, int index) { - Column firstColumn = childrenColumns.get(0); - long minValue = firstColumn.getLong(index); - boolean allNull = firstColumn.isNull(index); - for (int i = 1; i < childrenColumns.size(); i++) { - Column column = childrenColumns.get(i); - if (!column.isNull(index)) { - long value = column.getLong(index); - if (allNull || value < minValue) { - allNull = false; - minValue = value; - } - } - } - if (allNull) { - builder.appendNull(); - } else { - returnType.writeLong(builder, minValue); - } - } -} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java index 12393fb2a209..dbd32b47276f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java @@ -37,16 +37,12 @@ import org.apache.iotdb.db.utils.datastructure.TVList; import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; -import org.apache.tsfile.file.metadata.AbstractAlignedTimeSeriesMetadata; import org.apache.tsfile.file.metadata.AlignedChunkMetadata; import org.apache.tsfile.file.metadata.AlignedTimeSeriesMetadata; import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.ITimeSeriesMetadata; -import org.apache.tsfile.file.metadata.TableDeviceChunkMetadata; -import org.apache.tsfile.file.metadata.TableDeviceTimeSeriesMetadata; import org.apache.tsfile.file.metadata.TimeseriesMetadata; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; @@ -112,7 +108,7 @@ public AlignedResourceByPathUtils(IFullPath fullPath) { * have chunkMetadata, but query will use these, so we need to generate it for them. */ @Override - public AbstractAlignedTimeSeriesMetadata generateTimeSeriesMetadata( + public AlignedTimeSeriesMetadata generateTimeSeriesMetadata( List readOnlyMemChunk, List chunkMetadataList) { TimeseriesMetadata timeTimeSeriesMetadata = new TimeseriesMetadata(); timeTimeSeriesMetadata.setDataSizeOfChunkMetaDataList(-1); @@ -135,11 +131,8 @@ public AbstractAlignedTimeSeriesMetadata generateTimeSeriesMetadata( boolean[] exist = new boolean[alignedFullPath.getSchemaList().size()]; boolean modified = false; - boolean isTable = false; for (IChunkMetadata chunkMetadata : chunkMetadataList) { - AbstractAlignedChunkMetadata alignedChunkMetadata = - (AbstractAlignedChunkMetadata) chunkMetadata; - isTable = isTable || (alignedChunkMetadata instanceof TableDeviceChunkMetadata); + AlignedChunkMetadata alignedChunkMetadata = (AlignedChunkMetadata) chunkMetadata; modified = (modified || alignedChunkMetadata.isModified()); timeStatistics.mergeStatistics(alignedChunkMetadata.getTimeChunkMetadata().getStatistics()); for (int i = 0; i < valueTimeSeriesMetadataList.size(); i++) { @@ -156,9 +149,8 @@ public AbstractAlignedTimeSeriesMetadata generateTimeSeriesMetadata( for (ReadOnlyMemChunk memChunk : readOnlyMemChunk) { if (!memChunk.isEmpty()) { - AbstractAlignedChunkMetadata alignedChunkMetadata = - (AbstractAlignedChunkMetadata) memChunk.getChunkMetaData(); - isTable = isTable || (alignedChunkMetadata instanceof TableDeviceChunkMetadata); + AlignedChunkMetadata alignedChunkMetadata = + (AlignedChunkMetadata) memChunk.getChunkMetaData(); timeStatistics.mergeStatistics(alignedChunkMetadata.getTimeChunkMetadata().getStatistics()); for (int i = 0; i < valueTimeSeriesMetadataList.size(); i++) { if (alignedChunkMetadata.getValueChunkMetadataList().get(i) != null) { @@ -182,9 +174,7 @@ public AbstractAlignedTimeSeriesMetadata generateTimeSeriesMetadata( } } - return isTable - ? new TableDeviceTimeSeriesMetadata(timeTimeSeriesMetadata, valueTimeSeriesMetadataList) - : new AlignedTimeSeriesMetadata(timeTimeSeriesMetadata, valueTimeSeriesMetadataList); + return new AlignedTimeSeriesMetadata(timeTimeSeriesMetadata, valueTimeSeriesMetadataList); } @Override @@ -278,7 +268,7 @@ public List getVisibleMetadataListFromWriter( QueryContext context, long timeLowerBound) { - List chunkMetadataList = new ArrayList<>(); + List chunkMetadataList = new ArrayList<>(); List timeChunkMetadataList = writer.getVisibleMetadataList( alignedFullPath.getDeviceId(), AlignedFullPath.VECTOR_PLACEHOLDER, TSDataType.VECTOR); @@ -295,10 +285,7 @@ public List getVisibleMetadataListFromWriter( // only need time column if (alignedFullPath.getMeasurementList().isEmpty()) { chunkMetadataList.add( - context.isIgnoreAllNullRows() - ? new AlignedChunkMetadata(timeChunkMetadataList.get(i), Collections.emptyList()) - : new TableDeviceChunkMetadata( - timeChunkMetadataList.get(i), Collections.emptyList())); + new AlignedChunkMetadata(timeChunkMetadataList.get(i), Collections.emptyList())); } else { List valueChunkMetadata = new ArrayList<>(); // if all the sub sensors doesn't exist, it will be false @@ -311,9 +298,7 @@ public List getVisibleMetadataListFromWriter( } if (!context.isIgnoreAllNullRows() || exits) { chunkMetadataList.add( - context.isIgnoreAllNullRows() - ? new AlignedChunkMetadata(timeChunkMetadataList.get(i), valueChunkMetadata) - : new TableDeviceChunkMetadata(timeChunkMetadataList.get(i), valueChunkMetadata)); + new AlignedChunkMetadata(timeChunkMetadataList.get(i), valueChunkMetadata)); } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java index f958c4c5def2..6496aaf256c1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java @@ -910,20 +910,10 @@ public TSStatus writeLoadTsFileNode( LoadTsFileRateLimiter.getInstance().acquire(pieceNode.getDataSize()); - final DataRegion dataRegion = getDataRegion(dataRegionId); - if (dataRegion == null) { - LOGGER.warn( - "DataRegion {} not found on this DataNode when writing piece node" - + "of TsFile {} (maybe due to region migration), will skip.", - dataRegionId, - pieceNode.getTsFile()); - return RpcUtils.SUCCESS_STATUS; - } - try { - loadTsFileManager.writeToDataRegion(dataRegion, pieceNode, uuid); + loadTsFileManager.writeToDataRegion(getDataRegion(dataRegionId), pieceNode, uuid); } catch (IOException e) { - LOGGER.warn( + LOGGER.error( "IO error when writing piece node of TsFile {} to DataRegion {}.", pieceNode.getTsFile(), dataRegionId, @@ -931,15 +921,6 @@ public TSStatus writeLoadTsFileNode( status.setCode(TSStatusCode.LOAD_FILE_ERROR.getStatusCode()); status.setMessage(e.getMessage()); return status; - } catch (Exception e) { - LOGGER.warn( - "Exception occurred when writing piece node of TsFile {} to DataRegion {}.", - pieceNode.getTsFile(), - dataRegionId, - e); - status.setCode(TSStatusCode.LOAD_FILE_ERROR.getStatusCode()); - status.setMessage(e.getMessage()); - return status; } return RpcUtils.SUCCESS_STATUS; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/ReadChunkCompactionPerformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/ReadChunkCompactionPerformer.java index c86c02e51b2e..ca5929399767 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/ReadChunkCompactionPerformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/ReadChunkCompactionPerformer.java @@ -36,7 +36,7 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.exception.write.PageException; -import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; +import org.apache.tsfile.file.metadata.AlignedChunkMetadata; import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.read.TsFileSequenceReader; @@ -179,9 +179,8 @@ private void compactAlignedSeries( MultiTsFileDeviceIterator deviceIterator) throws IOException, InterruptedException, IllegalPathException, PageException { checkThreadInterrupted(); - LinkedList>> - readerAndChunkMetadataList = - deviceIterator.getReaderAndChunkMetadataForCurrentAlignedSeries(); + LinkedList>> readerAndChunkMetadataList = + deviceIterator.getReaderAndChunkMetadataForCurrentAlignedSeries(); if (!checkAlignedSeriesExists(readerAndChunkMetadataList)) { return; } @@ -213,9 +212,9 @@ private void checkThreadInterrupted() throws InterruptedException { } private boolean checkAlignedSeriesExists( - LinkedList>> + LinkedList>> readerAndChunkMetadataList) { - for (Pair> readerListPair : + for (Pair> readerListPair : readerAndChunkMetadataList) { if (!readerListPair.right.isEmpty()) { return true; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java index 1c8aed4ca62d..32a1593c4a65 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java @@ -34,7 +34,7 @@ import org.apache.iotdb.db.utils.ModificationUtils; import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; +import org.apache.tsfile.file.metadata.AlignedChunkMetadata; import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; import org.apache.tsfile.file.metadata.IDeviceID; @@ -384,14 +384,14 @@ public Map getAllSchemasOfCurrentDevice() throws IOEx * @throws IOException if io errors occurred */ @SuppressWarnings({"squid:S1319", "squid:S135"}) - public LinkedList>> + public LinkedList>> getReaderAndChunkMetadataForCurrentAlignedSeries() throws IOException, IllegalPathException { if (currentDevice == null || !currentDevice.right) { return new LinkedList<>(); } - LinkedList>> - readerAndChunkMetadataList = new LinkedList<>(); + LinkedList>> readerAndChunkMetadataList = + new LinkedList<>(); for (TsFileResource tsFileResource : tsFileResourcesSortedByAsc) { if (!deviceIteratorMap.containsKey(tsFileResource)) { continue; @@ -403,7 +403,7 @@ public Map getAllSchemasOfCurrentDevice() throws IOEx MetadataIndexNode firstMeasurementNodeOfCurrentDevice = iterator.getFirstMeasurementNodeOfCurrentDevice(); TsFileSequenceReader reader = readerMap.get(tsFileResource); - List alignedChunkMetadataList = + List alignedChunkMetadataList = reader.getAlignedChunkMetadataByMetadataIndexNode( currentDevice.left, firstMeasurementNodeOfCurrentDevice, ignoreAllNullRows); applyModificationForAlignedChunkMetadataList(tsFileResource, alignedChunkMetadataList); @@ -420,7 +420,7 @@ public Map getAllSchemasOfCurrentDevice() throws IOEx * @param alignedChunkMetadataList list of aligned chunk metadata */ private void applyModificationForAlignedChunkMetadataList( - TsFileResource tsFileResource, List alignedChunkMetadataList) + TsFileResource tsFileResource, List alignedChunkMetadataList) throws IllegalPathException { if (alignedChunkMetadataList.isEmpty()) { // all the value chunks is empty chunk @@ -437,7 +437,7 @@ private void applyModificationForAlignedChunkMetadataList( tsFileResource, r -> new ArrayList<>(tsFileResource.getAllModEntries())); // construct the input params List> for QueryUtils.modifyAlignedChunkMetaData - AbstractAlignedChunkMetadata alignedChunkMetadata = alignedChunkMetadataList.get(0); + AlignedChunkMetadata alignedChunkMetadata = alignedChunkMetadataList.get(0); List valueChunkMetadataList = alignedChunkMetadata.getValueChunkMetadataList(); // match time column modifications diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/batch/BatchedFastAlignedSeriesCompactionExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/batch/BatchedFastAlignedSeriesCompactionExecutor.java index 636fba8c2ca8..c2044da7f630 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/batch/BatchedFastAlignedSeriesCompactionExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/batch/BatchedFastAlignedSeriesCompactionExecutor.java @@ -43,7 +43,7 @@ import org.apache.tsfile.common.constant.TsFileConstant; import org.apache.tsfile.exception.write.PageException; -import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; +import org.apache.tsfile.file.metadata.AlignedChunkMetadata; import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; import org.apache.tsfile.file.metadata.IDeviceID; @@ -72,7 +72,7 @@ public class BatchedFastAlignedSeriesCompactionExecutor private final List valueMeasurementSchemas; private final List sortedSourceFiles; - private final Map> alignedChunkMetadataCache; + private final Map> alignedChunkMetadataCache; private final BatchCompactionPlan batchCompactionPlan; private final int batchSize = IoTDBDescriptor.getInstance().getConfig().getCompactionMaxAlignedSeriesNumInOneBatch(); @@ -110,11 +110,11 @@ public BatchedFastAlignedSeriesCompactionExecutor( this.batchCompactionPlan = new BatchCompactionPlan(); } - private List getAlignedChunkMetadataListBySelectedValueColumn( + private List getAlignedChunkMetadataListBySelectedValueColumn( TsFileResource tsFileResource, List selectedValueMeasurementSchemas) throws IOException, IllegalPathException { // 1. get Full AlignedChunkMetadata from cache - List alignedChunkMetadataList = null; + List alignedChunkMetadataList = null; if (alignedChunkMetadataCache.containsKey(tsFileResource)) { alignedChunkMetadataList = alignedChunkMetadataCache.get(tsFileResource); } else { @@ -124,8 +124,8 @@ private List getAlignedChunkMetadataListBySelected } // 2. generate AlignedChunkMetadata list by selected value columns - List filteredAlignedChunkMetadataList = new ArrayList<>(); - for (AbstractAlignedChunkMetadata alignedChunkMetadata : alignedChunkMetadataList) { + List filteredAlignedChunkMetadataList = new ArrayList<>(); + for (AlignedChunkMetadata alignedChunkMetadata : alignedChunkMetadataList) { filteredAlignedChunkMetadataList.add( AlignedSeriesBatchCompactionUtils.filterAlignedChunkMetadataByIndex( alignedChunkMetadata, batchColumnSelection.getSelectedColumnIndexList())); @@ -264,8 +264,8 @@ public void execute() } @Override - protected List getAlignedChunkMetadataList( - TsFileResource resource) throws IOException, IllegalPathException { + protected List getAlignedChunkMetadataList(TsFileResource resource) + throws IOException, IllegalPathException { return getAlignedChunkMetadataListBySelectedValueColumn(resource, measurementSchemas); } @@ -301,10 +301,10 @@ protected ModifiedStatus isPageModified(PageElement pageElement) { IChunkMetadata batchedAlignedChunkMetadata = alignedPageElement.getChunkMetadataElement().chunkMetadata; TsFileResource resource = alignedPageElement.getChunkMetadataElement().fileElement.resource; - List alignedChunkMetadataListOfFile = + List alignedChunkMetadataListOfFile = alignedChunkMetadataCache.get(resource); - AbstractAlignedChunkMetadata originAlignedChunkMetadata = null; - for (AbstractAlignedChunkMetadata alignedChunkMetadata : alignedChunkMetadataListOfFile) { + AlignedChunkMetadata originAlignedChunkMetadata = null; + for (AlignedChunkMetadata alignedChunkMetadata : alignedChunkMetadataListOfFile) { if (alignedChunkMetadata.getOffsetOfChunkHeader() == batchedAlignedChunkMetadata.getOffsetOfChunkHeader()) { originAlignedChunkMetadata = alignedChunkMetadata; @@ -383,8 +383,8 @@ protected ModifiedStatus isPageModified(PageElement pageElement) { } @Override - protected List getAlignedChunkMetadataList( - TsFileResource resource) throws IOException, IllegalPathException { + protected List getAlignedChunkMetadataList(TsFileResource resource) + throws IOException, IllegalPathException { return getAlignedChunkMetadataListBySelectedValueColumn(resource, measurementSchemas); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/batch/BatchedReadChunkAlignedSeriesCompactionExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/batch/BatchedReadChunkAlignedSeriesCompactionExecutor.java index e343ea59b929..c0d16841ab10 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/batch/BatchedReadChunkAlignedSeriesCompactionExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/batch/BatchedReadChunkAlignedSeriesCompactionExecutor.java @@ -38,7 +38,7 @@ import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.tsfile.exception.write.PageException; -import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; +import org.apache.tsfile.file.metadata.AlignedChunkMetadata; import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.statistics.Statistics; @@ -67,14 +67,13 @@ public class BatchedReadChunkAlignedSeriesCompactionExecutor private final int batchSize = IoTDBDescriptor.getInstance().getConfig().getCompactionMaxAlignedSeriesNumInOneBatch(); private final AlignedSeriesBatchCompactionUtils.BatchColumnSelection batchColumnSelection; - private final LinkedList>> + private final LinkedList>> originReaderAndChunkMetadataList; public BatchedReadChunkAlignedSeriesCompactionExecutor( IDeviceID device, TsFileResource targetResource, - LinkedList>> - readerAndChunkMetadataList, + LinkedList>> readerAndChunkMetadataList, CompactionTsFileWriter writer, CompactionTaskSummary summary, boolean ignoreAllNullRows) @@ -114,7 +113,7 @@ private void compactFirstBatch() throws IOException, PageException { selectedColumnSchemaList = batchColumnSelection.getCurrentSelectedColumnSchemaList(); } - LinkedList>> + LinkedList>> batchedReaderAndChunkMetadataList = filterAlignedChunkMetadataList(readerAndChunkMetadataList, selectedColumnIndexList); @@ -138,7 +137,7 @@ private void compactFirstBatch() throws IOException, PageException { private void compactLeftBatches() throws PageException, IOException { while (batchColumnSelection.hasNext()) { batchColumnSelection.next(); - LinkedList>> + LinkedList>> groupReaderAndChunkMetadataList = filterAlignedChunkMetadataList( readerAndChunkMetadataList, batchColumnSelection.getSelectedColumnIndexList()); @@ -156,19 +155,16 @@ private void compactLeftBatches() throws PageException, IOException { } } - private LinkedList>> + private LinkedList>> filterAlignedChunkMetadataList( - List>> - readerAndChunkMetadataList, + List>> readerAndChunkMetadataList, List selectedMeasurementIndexs) { - LinkedList>> + LinkedList>> groupReaderAndChunkMetadataList = new LinkedList<>(); - for (Pair> pair : - readerAndChunkMetadataList) { - List alignedChunkMetadataList = pair.getRight(); - List selectedColumnAlignedChunkMetadataList = - new LinkedList<>(); - for (AbstractAlignedChunkMetadata alignedChunkMetadata : alignedChunkMetadataList) { + for (Pair> pair : readerAndChunkMetadataList) { + List alignedChunkMetadataList = pair.getRight(); + List selectedColumnAlignedChunkMetadataList = new LinkedList<>(); + for (AlignedChunkMetadata alignedChunkMetadata : alignedChunkMetadataList) { selectedColumnAlignedChunkMetadataList.add( AlignedSeriesBatchCompactionUtils.filterAlignedChunkMetadataByIndex( alignedChunkMetadata, selectedMeasurementIndexs)); @@ -185,7 +181,7 @@ public class FirstBatchedReadChunkAlignedSeriesCompactionExecutor public FirstBatchedReadChunkAlignedSeriesCompactionExecutor( IDeviceID device, TsFileResource targetResource, - LinkedList>> + LinkedList>> readerAndChunkMetadataList, CompactionTsFileWriter writer, CompactionTaskSummary summary, @@ -228,8 +224,8 @@ protected boolean isAllValuePageEmpty(PageLoader timePage, List valu String file = timePage.getFile(); ChunkMetadata timeChunkMetadata = timePage.getChunkMetadata(); - List alignedChunkMetadataList = Collections.emptyList(); - for (Pair> pair : + List alignedChunkMetadataList = Collections.emptyList(); + for (Pair> pair : originReaderAndChunkMetadataList) { TsFileSequenceReader reader = pair.getLeft(); if (reader.getFileName().equals(file)) { @@ -238,8 +234,8 @@ protected boolean isAllValuePageEmpty(PageLoader timePage, List valu } } - AbstractAlignedChunkMetadata originAlignedChunkMetadata = null; - for (AbstractAlignedChunkMetadata alignedChunkMetadata : alignedChunkMetadataList) { + AlignedChunkMetadata originAlignedChunkMetadata = null; + for (AlignedChunkMetadata alignedChunkMetadata : alignedChunkMetadataList) { if (alignedChunkMetadata.getOffsetOfChunkHeader() == timeChunkMetadata.getOffsetOfChunkHeader()) { originAlignedChunkMetadata = alignedChunkMetadata; @@ -307,7 +303,7 @@ public class FollowingBatchedReadChunkAlignedSeriesGroupCompactionExecutor public FollowingBatchedReadChunkAlignedSeriesGroupCompactionExecutor( IDeviceID device, TsFileResource targetResource, - LinkedList>> + LinkedList>> readerAndChunkMetadataList, CompactionTsFileWriter writer, CompactionTaskSummary summary, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/batch/utils/AlignedSeriesBatchCompactionUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/batch/utils/AlignedSeriesBatchCompactionUtils.java index c3c58e5bb6a1..25613fc2a1a8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/batch/utils/AlignedSeriesBatchCompactionUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/batch/utils/AlignedSeriesBatchCompactionUtils.java @@ -21,7 +21,7 @@ import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.executor.ModifiedStatus; -import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; +import org.apache.tsfile.file.metadata.AlignedChunkMetadata; import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; import org.apache.tsfile.read.TsFileSequenceReader; @@ -43,18 +43,17 @@ public class AlignedSeriesBatchCompactionUtils { private AlignedSeriesBatchCompactionUtils() {} public static void markAlignedChunkHasDeletion( - LinkedList>> + LinkedList>> readerAndChunkMetadataList) { - for (Pair> pair : - readerAndChunkMetadataList) { - List alignedChunkMetadataList = pair.getRight(); + for (Pair> pair : readerAndChunkMetadataList) { + List alignedChunkMetadataList = pair.getRight(); markAlignedChunkHasDeletion(alignedChunkMetadataList); } } public static void markAlignedChunkHasDeletion( - List alignedChunkMetadataList) { - for (AbstractAlignedChunkMetadata alignedChunkMetadata : alignedChunkMetadataList) { + List alignedChunkMetadataList) { + for (AlignedChunkMetadata alignedChunkMetadata : alignedChunkMetadataList) { IChunkMetadata timeChunkMetadata = alignedChunkMetadata.getTimeChunkMetadata(); for (IChunkMetadata iChunkMetadata : alignedChunkMetadata.getValueChunkMetadataList()) { if (iChunkMetadata != null && iChunkMetadata.isModified()) { @@ -69,8 +68,8 @@ public static boolean isTimeChunk(ChunkMetadata chunkMetadata) { return chunkMetadata.getMeasurementUid().isEmpty(); } - public static AbstractAlignedChunkMetadata filterAlignedChunkMetadataByIndex( - AbstractAlignedChunkMetadata alignedChunkMetadata, List selectedMeasurements) { + public static AlignedChunkMetadata filterAlignedChunkMetadataByIndex( + AlignedChunkMetadata alignedChunkMetadata, List selectedMeasurements) { IChunkMetadata[] valueChunkMetadataArr = new IChunkMetadata[selectedMeasurements.size()]; List originValueChunkMetadataList = alignedChunkMetadata.getValueChunkMetadataList(); @@ -78,13 +77,12 @@ public static AbstractAlignedChunkMetadata filterAlignedChunkMetadataByIndex( int columnIndex = selectedMeasurements.get(i); valueChunkMetadataArr[i] = originValueChunkMetadataList.get(columnIndex); } - return alignedChunkMetadata.createNewChunkMetadata( + return new AlignedChunkMetadata( alignedChunkMetadata.getTimeChunkMetadata(), Arrays.asList(valueChunkMetadataArr)); } - public static AbstractAlignedChunkMetadata fillAlignedChunkMetadataBySchemaList( - AbstractAlignedChunkMetadata originAlignedChunkMetadata, - List schemaList) { + public static AlignedChunkMetadata fillAlignedChunkMetadataBySchemaList( + AlignedChunkMetadata originAlignedChunkMetadata, List schemaList) { List originValueChunkMetadataList = originAlignedChunkMetadata.getValueChunkMetadataList(); IChunkMetadata[] newValueChunkMetadataArr = new IChunkMetadata[schemaList.size()]; @@ -111,14 +109,14 @@ public static AbstractAlignedChunkMetadata fillAlignedChunkMetadataBySchemaList( currentValueChunkMetadataIndex++; } } - return originAlignedChunkMetadata.createNewChunkMetadata( + return new AlignedChunkMetadata( originAlignedChunkMetadata.getTimeChunkMetadata(), Arrays.asList(newValueChunkMetadataArr)); } public static ModifiedStatus calculateAlignedPageModifiedStatus( long startTime, long endTime, - AbstractAlignedChunkMetadata originAlignedChunkMetadata, + AlignedChunkMetadata originAlignedChunkMetadata, boolean ignoreAllNullRows) { ModifiedStatus timePageModifiedStatus = checkIsModified( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastAlignedSeriesCompactionExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastAlignedSeriesCompactionExecutor.java index b489630894bd..64fb585542d7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastAlignedSeriesCompactionExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastAlignedSeriesCompactionExecutor.java @@ -44,12 +44,10 @@ import org.apache.tsfile.exception.write.PageException; import org.apache.tsfile.file.header.ChunkHeader; import org.apache.tsfile.file.header.PageHeader; -import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; import org.apache.tsfile.file.metadata.AlignedChunkMetadata; import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; import org.apache.tsfile.file.metadata.IDeviceID; -import org.apache.tsfile.file.metadata.TableDeviceChunkMetadata; import org.apache.tsfile.read.TsFileSequenceReader; import org.apache.tsfile.read.common.Chunk; import org.apache.tsfile.utils.Pair; @@ -157,8 +155,7 @@ void deserializeFileIntoChunkMetadataQueue(List fileElements) throws IOException, IllegalPathException { for (FileElement fileElement : fileElements) { TsFileResource resource = fileElement.resource; - List alignedChunkMetadataList = - getAlignedChunkMetadataList(resource); + List alignedChunkMetadataList = getAlignedChunkMetadataList(resource); if (alignedChunkMetadataList.isEmpty()) { // all chunks have been deleted in this file or current file does not contain this aligned @@ -177,7 +174,7 @@ void deserializeFileIntoChunkMetadataQueue(List fileElements) } } - protected List getAlignedChunkMetadataList(TsFileResource resource) + protected List getAlignedChunkMetadataList(TsFileResource resource) throws IOException, IllegalPathException { // read time chunk metadatas and value chunk metadatas in the current file List timeChunkMetadatas = null; @@ -219,7 +216,7 @@ protected List getAlignedChunkMetadataList(TsFileR } } - List alignedChunkMetadataList = new ArrayList<>(); + List alignedChunkMetadataList = new ArrayList<>(); // if current file contains this aligned device,then construct aligned chunk metadatas if (timeChunkMetadatas != null) { for (int i = 0; i < timeChunkMetadatas.size(); i++) { @@ -231,10 +228,8 @@ protected List getAlignedChunkMetadataList(TsFileR valueChunkMetadataList.add(chunkMetadata.get(i)); } } - AbstractAlignedChunkMetadata alignedChunkMetadata = - ignoreAllNullRows - ? new AlignedChunkMetadata(timeChunkMetadatas.get(i), valueChunkMetadataList) - : new TableDeviceChunkMetadata(timeChunkMetadatas.get(i), valueChunkMetadataList); + AlignedChunkMetadata alignedChunkMetadata = + new AlignedChunkMetadata(timeChunkMetadatas.get(i), valueChunkMetadataList); alignedChunkMetadataList.add(alignedChunkMetadata); } @@ -350,8 +345,8 @@ protected void deserializeChunkIntoPageQueue(ChunkMetadataElement chunkMetadataE @Override void readChunk(ChunkMetadataElement chunkMetadataElement) throws IOException { updateSummary(chunkMetadataElement, ChunkStatus.READ_IN); - AbstractAlignedChunkMetadata alignedChunkMetadata = - (AbstractAlignedChunkMetadata) chunkMetadataElement.chunkMetadata; + AlignedChunkMetadata alignedChunkMetadata = + (AlignedChunkMetadata) chunkMetadataElement.chunkMetadata; TsFileSequenceReader reader = readerCacheMap.get(chunkMetadataElement.fileElement.resource); chunkMetadataElement.chunk = readChunk(reader, (ChunkMetadata) alignedChunkMetadata.getTimeChunkMetadata()); @@ -423,8 +418,8 @@ protected boolean flushPageToCompactionWriter(PageElement pageElement) protected ModifiedStatus isPageModified(PageElement pageElement) { long startTime = pageElement.getStartTime(); long endTime = pageElement.getEndTime(); - AbstractAlignedChunkMetadata alignedChunkMetadata = - (AbstractAlignedChunkMetadata) pageElement.getChunkMetadataElement().chunkMetadata; + AlignedChunkMetadata alignedChunkMetadata = + (AlignedChunkMetadata) pageElement.getChunkMetadataElement().chunkMetadata; return AlignedSeriesBatchCompactionUtils.calculateAlignedPageModifiedStatus( startTime, endTime, alignedChunkMetadata, ignoreAllNullRows); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/SeriesCompactionExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/SeriesCompactionExecutor.java index c2a5c6c7558f..83dab2e76907 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/SeriesCompactionExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/SeriesCompactionExecutor.java @@ -36,7 +36,7 @@ import org.apache.iotdb.db.utils.datastructure.PatternTreeMapFactory; import org.apache.tsfile.exception.write.PageException; -import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; +import org.apache.tsfile.file.metadata.AlignedChunkMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.read.TimeValuePair; @@ -495,14 +495,14 @@ protected void updateSummary(ChunkMetadataElement chunkMetadataElement, ChunkSta case READ_IN: summary.increaseProcessChunkNum( isAligned - ? ((AbstractAlignedChunkMetadata) chunkMetadataElement.chunkMetadata) + ? ((AlignedChunkMetadata) chunkMetadataElement.chunkMetadata) .getValueChunkMetadataList() .size() + 1 : 1); if (isAligned) { for (IChunkMetadata valueChunkMetadata : - ((AbstractAlignedChunkMetadata) chunkMetadataElement.chunkMetadata) + ((AlignedChunkMetadata) chunkMetadataElement.chunkMetadata) .getValueChunkMetadataList()) { if (valueChunkMetadata == null) { continue; @@ -517,7 +517,7 @@ protected void updateSummary(ChunkMetadataElement chunkMetadataElement, ChunkSta case DIRECTORY_FLUSH: if (isAligned) { summary.increaseDirectlyFlushChunkNum( - ((AbstractAlignedChunkMetadata) (chunkMetadataElement.chunkMetadata)) + ((AlignedChunkMetadata) (chunkMetadataElement.chunkMetadata)) .getValueChunkMetadataList() .size() + 1); @@ -528,7 +528,7 @@ protected void updateSummary(ChunkMetadataElement chunkMetadataElement, ChunkSta case DESERIALIZE_CHUNK: if (isAligned) { summary.increaseDeserializedChunkNum( - ((AbstractAlignedChunkMetadata) (chunkMetadataElement.chunkMetadata)) + ((AlignedChunkMetadata) (chunkMetadataElement.chunkMetadata)) .getValueChunkMetadataList() .size() + 1); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/ReadChunkAlignedSeriesCompactionExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/ReadChunkAlignedSeriesCompactionExecutor.java index 17281c87051e..12c205c472ed 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/ReadChunkAlignedSeriesCompactionExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/ReadChunkAlignedSeriesCompactionExecutor.java @@ -39,7 +39,7 @@ import org.apache.tsfile.exception.write.PageException; import org.apache.tsfile.file.header.ChunkHeader; import org.apache.tsfile.file.header.PageHeader; -import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; +import org.apache.tsfile.file.metadata.AlignedChunkMetadata; import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; import org.apache.tsfile.file.metadata.IDeviceID; @@ -68,7 +68,7 @@ public class ReadChunkAlignedSeriesCompactionExecutor { protected final IDeviceID device; - protected final LinkedList>> + protected final LinkedList>> readerAndChunkMetadataList; protected final TsFileResource targetResource; protected final CompactionTsFileWriter writer; @@ -85,8 +85,7 @@ public class ReadChunkAlignedSeriesCompactionExecutor { public ReadChunkAlignedSeriesCompactionExecutor( IDeviceID device, TsFileResource targetResource, - LinkedList>> - readerAndChunkMetadataList, + LinkedList>> readerAndChunkMetadataList, CompactionTsFileWriter writer, CompactionTaskSummary summary, boolean ignoreAllNullRows) @@ -109,8 +108,7 @@ public ReadChunkAlignedSeriesCompactionExecutor( public ReadChunkAlignedSeriesCompactionExecutor( IDeviceID device, TsFileResource targetResource, - LinkedList>> - readerAndChunkMetadataList, + LinkedList>> readerAndChunkMetadataList, CompactionTsFileWriter writer, CompactionTaskSummary summary, IMeasurementSchema timeSchema, @@ -133,11 +131,11 @@ public ReadChunkAlignedSeriesCompactionExecutor( private void collectValueColumnSchemaList() throws IOException { Map measurementSchemaMap = new HashMap<>(); for (int i = this.readerAndChunkMetadataList.size() - 1; i >= 0; i--) { - Pair> pair = + Pair> pair = this.readerAndChunkMetadataList.get(i); CompactionTsFileReader reader = (CompactionTsFileReader) pair.getLeft(); - List alignedChunkMetadataList = pair.getRight(); - for (AbstractAlignedChunkMetadata alignedChunkMetadata : alignedChunkMetadataList) { + List alignedChunkMetadataList = pair.getRight(); + for (AlignedChunkMetadata alignedChunkMetadata : alignedChunkMetadataList) { if (alignedChunkMetadata == null) { continue; } @@ -178,11 +176,10 @@ private void collectValueColumnSchemaList() throws IOException { } private void fillAlignedChunkMetadataToMatchSchemaList() { - for (Pair> pair : - readerAndChunkMetadataList) { - List alignedChunkMetadataList = pair.getRight(); + for (Pair> pair : readerAndChunkMetadataList) { + List alignedChunkMetadataList = pair.getRight(); for (int i = 0; i < alignedChunkMetadataList.size(); i++) { - AbstractAlignedChunkMetadata alignedChunkMetadata = alignedChunkMetadataList.get(i); + AlignedChunkMetadata alignedChunkMetadata = alignedChunkMetadataList.get(i); alignedChunkMetadataList.set( i, AlignedSeriesBatchCompactionUtils.fillAlignedChunkMetadataBySchemaList( @@ -196,15 +193,15 @@ protected AlignedChunkWriterImpl constructAlignedChunkWriter() { } public void execute() throws IOException, PageException { - for (Pair> readerListPair : + for (Pair> readerListPair : readerAndChunkMetadataList) { TsFileSequenceReader reader = readerListPair.left; - List alignedChunkMetadataList = readerListPair.right; + List alignedChunkMetadataList = readerListPair.right; if (reader instanceof CompactionTsFileReader) { ((CompactionTsFileReader) reader).markStartOfAlignedSeries(); } - for (AbstractAlignedChunkMetadata alignedChunkMetadata : alignedChunkMetadataList) { + for (AlignedChunkMetadata alignedChunkMetadata : alignedChunkMetadataList) { compactWithAlignedChunk(reader, alignedChunkMetadata); } if (reader instanceof CompactionTsFileReader) { @@ -218,7 +215,7 @@ public void execute() throws IOException, PageException { } private void compactWithAlignedChunk( - TsFileSequenceReader reader, AbstractAlignedChunkMetadata alignedChunkMetadata) + TsFileSequenceReader reader, AlignedChunkMetadata alignedChunkMetadata) throws IOException, PageException { ChunkLoader timeChunk = getChunkLoader(reader, (ChunkMetadata) alignedChunkMetadata.getTimeChunkMetadata()); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/FastCrossCompactionWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/FastCrossCompactionWriter.java index d4b20039109e..ae5bc2e75911 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/FastCrossCompactionWriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/FastCrossCompactionWriter.java @@ -26,7 +26,7 @@ import org.apache.tsfile.exception.write.PageException; import org.apache.tsfile.file.header.PageHeader; -import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; +import org.apache.tsfile.file.metadata.AlignedChunkMetadata; import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; import org.apache.tsfile.read.TsFileSequenceReader; @@ -97,8 +97,8 @@ public boolean flushNonAlignedChunk(Chunk chunk, ChunkMetadata chunkMetadata, in @Override public boolean flushAlignedChunk(ChunkMetadataElement chunkMetadataElement, int subTaskId) throws IOException { - AbstractAlignedChunkMetadata alignedChunkMetadata = - (AbstractAlignedChunkMetadata) chunkMetadataElement.chunkMetadata; + AlignedChunkMetadata alignedChunkMetadata = + (AlignedChunkMetadata) chunkMetadataElement.chunkMetadata; IChunkMetadata timeChunkMetadata = alignedChunkMetadata.getTimeChunkMetadata(); List valueChunkMetadatas = alignedChunkMetadata.getValueChunkMetadataList(); Chunk timeChunk = chunkMetadataElement.chunk; @@ -132,8 +132,8 @@ public boolean flushBatchedValueChunk( int subTaskId, AbstractCompactionFlushController flushController) throws IOException { - AbstractAlignedChunkMetadata alignedChunkMetadata = - (AbstractAlignedChunkMetadata) chunkMetadataElement.chunkMetadata; + AlignedChunkMetadata alignedChunkMetadata = + (AlignedChunkMetadata) chunkMetadataElement.chunkMetadata; IChunkMetadata timeChunkMetadata = alignedChunkMetadata.getTimeChunkMetadata(); List valueChunkMetadatas = alignedChunkMetadata.getValueChunkMetadataList(); List valueChunks = chunkMetadataElement.valueChunks; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/FastInnerCompactionWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/FastInnerCompactionWriter.java index 39e621cfaf40..e9709af3dbf6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/FastInnerCompactionWriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/FastInnerCompactionWriter.java @@ -26,7 +26,7 @@ import org.apache.tsfile.exception.write.PageException; import org.apache.tsfile.file.header.PageHeader; -import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; +import org.apache.tsfile.file.metadata.AlignedChunkMetadata; import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; import org.apache.tsfile.read.common.Chunk; @@ -89,8 +89,8 @@ public boolean flushNonAlignedChunk(Chunk chunk, ChunkMetadata chunkMetadata, in @Override public boolean flushAlignedChunk(ChunkMetadataElement chunkMetadataElement, int subTaskId) throws IOException { - AbstractAlignedChunkMetadata alignedChunkMetadata = - (AbstractAlignedChunkMetadata) chunkMetadataElement.chunkMetadata; + AlignedChunkMetadata alignedChunkMetadata = + (AlignedChunkMetadata) chunkMetadataElement.chunkMetadata; IChunkMetadata timeChunkMetadata = alignedChunkMetadata.getTimeChunkMetadata(); List valueChunkMetadatas = alignedChunkMetadata.getValueChunkMetadataList(); Chunk timeChunk = chunkMetadataElement.chunk; @@ -121,8 +121,8 @@ public boolean flushBatchedValueChunk( int subTaskId, AbstractCompactionFlushController flushController) throws IOException { - AbstractAlignedChunkMetadata alignedChunkMetadata = - (AbstractAlignedChunkMetadata) chunkMetadataElement.chunkMetadata; + AlignedChunkMetadata alignedChunkMetadata = + (AlignedChunkMetadata) chunkMetadataElement.chunkMetadata; IChunkMetadata timeChunkMetadata = alignedChunkMetadata.getTimeChunkMetadata(); List valueChunkMetadatas = alignedChunkMetadata.getValueChunkMetadataList(); List valueChunks = chunkMetadataElement.valueChunks; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/io/CompactionDiskTSMIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/io/CompactionDiskTSMIterator.java deleted file mode 100644 index b90769dba37f..000000000000 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/io/CompactionDiskTSMIterator.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.storageengine.dataregion.compaction.io; - -import org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.constant.CompactionType; - -import org.apache.tsfile.file.metadata.ChunkGroupMetadata; -import org.apache.tsfile.write.writer.tsmiterator.DiskTSMIterator; - -import java.io.File; -import java.io.IOException; -import java.util.LinkedList; -import java.util.List; - -public class CompactionDiskTSMIterator extends DiskTSMIterator { - - public CompactionDiskTSMIterator( - CompactionType compactionType, - File cmtFile, - List chunkGroupMetadataList, - LinkedList endPosForEachDevice) - throws IOException { - super(cmtFile, chunkGroupMetadataList, endPosForEachDevice); - this.input = new CompactionTsFileInput(compactionType, this.input); - } -} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/io/CompactionTsFileInput.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/io/CompactionTsFileInput.java index e51f26da14cb..b13475e4fe67 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/io/CompactionTsFileInput.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/io/CompactionTsFileInput.java @@ -19,11 +19,6 @@ package org.apache.iotdb.db.storageengine.dataregion.compaction.io; -import org.apache.iotdb.db.service.metrics.CompactionMetrics; -import org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.CompactionTaskManager; -import org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.constant.CompactionIoDataType; -import org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.constant.CompactionType; - import org.apache.tsfile.exception.StopReadTsFileByInterruptException; import org.apache.tsfile.read.reader.TsFileInput; @@ -34,33 +29,10 @@ public class CompactionTsFileInput implements TsFileInput { private final TsFileInput tsFileInput; - private long metadataOffset = -1; - - /** The type of compaction running. */ - private final CompactionType compactionType; - - /** A flag that indicates if an aligned series is being read. */ - private volatile boolean readingAlignedSeries = false; - - public CompactionTsFileInput(CompactionType compactionType, TsFileInput tsFileInput) { - this.compactionType = compactionType; + public CompactionTsFileInput(TsFileInput tsFileInput) { this.tsFileInput = tsFileInput; } - public void setMetadataOffset(long metadataOffset) { - this.metadataOffset = metadataOffset; - } - - /** Marks the start of reading an aligned series. */ - public void markStartOfAlignedSeries() { - readingAlignedSeries = true; - } - - /** Marks the end of reading an aligned series. */ - public void markEndOfAlignedSeries() { - readingAlignedSeries = false; - } - @Override public long size() throws IOException { try { @@ -99,9 +71,7 @@ public TsFileInput position(long newPosition) throws IOException { @Override public int read(ByteBuffer dst) throws IOException { - acquireReadDataSizeWithCompactionReadRateLimiter(dst.remaining()); int readSize = tsFileInput.read(dst); - updateMetrics(position(), readSize); if (Thread.currentThread().isInterrupted()) { throw new StopReadTsFileByInterruptException(); } @@ -110,9 +80,7 @@ public int read(ByteBuffer dst) throws IOException { @Override public int read(ByteBuffer dst, long position) throws IOException { - acquireReadDataSizeWithCompactionReadRateLimiter(dst.remaining()); int readSize = tsFileInput.read(dst, position); - updateMetrics(position, readSize); if (Thread.currentThread().isInterrupted()) { throw new StopReadTsFileByInterruptException(); } @@ -121,7 +89,7 @@ public int read(ByteBuffer dst, long position) throws IOException { @Override public InputStream wrapAsInputStream() throws IOException { - return new CompactionTsFileInputStreamWrapper(tsFileInput.wrapAsInputStream()); + return tsFileInput.wrapAsInputStream(); } @Override @@ -133,90 +101,4 @@ public void close() throws IOException { public String getFilePath() { return tsFileInput.getFilePath(); } - - private void acquireReadDataSizeWithCompactionReadRateLimiter(int readDataSize) { - CompactionTaskManager.getInstance().getCompactionReadOperationRateLimiter().acquire(1); - CompactionTaskManager.getInstance().getCompactionReadRateLimiter().acquire(readDataSize); - } - - private void updateMetrics(long position, long totalSize) { - if (position >= metadataOffset) { - CompactionMetrics.getInstance() - .recordReadInfo(compactionType, CompactionIoDataType.METADATA, totalSize); - } else { - CompactionMetrics.getInstance() - .recordReadInfo( - compactionType, - readingAlignedSeries - ? CompactionIoDataType.ALIGNED - : CompactionIoDataType.NOT_ALIGNED, - totalSize); - } - } - - private class CompactionTsFileInputStreamWrapper extends InputStream { - - private final InputStream inputStream; - - public CompactionTsFileInputStreamWrapper(InputStream inputStream) { - this.inputStream = inputStream; - } - - @Override - public int read() throws IOException { - acquireReadDataSizeWithCompactionReadRateLimiter(1); - long position = position(); - int readSize = inputStream.read(); - updateMetrics(position, readSize); - return readSize; - } - - @Override - public int read(byte[] b) throws IOException { - acquireReadDataSizeWithCompactionReadRateLimiter(b.length); - long position = position(); - int readSize = inputStream.read(b); - updateMetrics(position, readSize); - return readSize; - } - - @Override - public int read(byte[] b, int off, int len) throws IOException { - acquireReadDataSizeWithCompactionReadRateLimiter(len); - long position = position(); - int readSize = inputStream.read(b, off, len); - updateMetrics(position, readSize); - return readSize; - } - - @Override - public long skip(long n) throws IOException { - return inputStream.skip(n); - } - - @Override - public int available() throws IOException { - return inputStream.available(); - } - - @Override - public void close() throws IOException { - inputStream.close(); - } - - @Override - public synchronized void mark(int readlimit) { - inputStream.mark(readlimit); - } - - @Override - public synchronized void reset() throws IOException { - inputStream.reset(); - } - - @Override - public boolean markSupported() { - return inputStream.markSupported(); - } - } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/io/CompactionTsFileReader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/io/CompactionTsFileReader.java index 7aabb0856433..bf98611db2c2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/io/CompactionTsFileReader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/io/CompactionTsFileReader.java @@ -19,22 +19,29 @@ package org.apache.iotdb.db.storageengine.dataregion.compaction.io; +import org.apache.iotdb.db.service.metrics.CompactionMetrics; +import org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.CompactionTaskManager; +import org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.constant.CompactionIoDataType; import org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.constant.CompactionType; import org.apache.tsfile.file.IMetadataIndexEntry; import org.apache.tsfile.file.header.ChunkHeader; +import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.MetadataIndexNode; import org.apache.tsfile.file.metadata.TimeseriesMetadata; import org.apache.tsfile.file.metadata.enums.MetadataIndexNodeType; import org.apache.tsfile.read.TsFileSequenceReader; +import org.apache.tsfile.read.common.Chunk; import org.apache.tsfile.utils.Pair; import java.io.IOException; +import java.io.InputStream; import java.nio.ByteBuffer; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.function.LongConsumer; /** * This class extends the TsFileSequenceReader class to read and manage TsFile with a focus on @@ -43,6 +50,14 @@ */ public class CompactionTsFileReader extends TsFileSequenceReader { + private long metadataOffset = 0; + + /** The type of compaction running. */ + CompactionType compactionType; + + /** A flag that indicates if an aligned series is being read. */ + private volatile boolean readingAlignedSeries = false; + /** * Constructs a new instance of CompactionTsFileReader. * @@ -52,20 +67,44 @@ public class CompactionTsFileReader extends TsFileSequenceReader { */ public CompactionTsFileReader(String file, CompactionType compactionType) throws IOException { super(file); - CompactionTsFileInput compactionTsFileInput = - new CompactionTsFileInput(compactionType, tsFileInput); - this.tsFileInput = compactionTsFileInput; - compactionTsFileInput.setMetadataOffset(readFileMetadata().getMetaOffset()); + this.tsFileInput = new CompactionTsFileInput(tsFileInput); + this.compactionType = compactionType; + this.metadataOffset = readFileMetadata().getMetaOffset(); + } + + @Override + protected ByteBuffer readData(long position, int totalSize, LongConsumer ioSizeRecorder) + throws IOException { + acquireReadDataSizeWithCompactionReadRateLimiter(totalSize); + ByteBuffer buffer = super.readData(position, totalSize, ioSizeRecorder); + if (position >= metadataOffset) { + CompactionMetrics.getInstance() + .recordReadInfo(compactionType, CompactionIoDataType.METADATA, totalSize); + } else { + CompactionMetrics.getInstance() + .recordReadInfo( + compactionType, + readingAlignedSeries + ? CompactionIoDataType.ALIGNED + : CompactionIoDataType.NOT_ALIGNED, + totalSize); + } + return buffer; } /** Marks the start of reading an aligned series. */ public void markStartOfAlignedSeries() { - ((CompactionTsFileInput) tsFileInput).markStartOfAlignedSeries(); + readingAlignedSeries = true; } /** Marks the end of reading an aligned series. */ public void markEndOfAlignedSeries() { - ((CompactionTsFileInput) tsFileInput).markEndOfAlignedSeries(); + readingAlignedSeries = false; + } + + @Override + public Chunk readMemChunk(ChunkMetadata metaData) throws IOException { + return super.readMemChunk(metaData); } @SuppressWarnings("java:S2177") @@ -73,6 +112,10 @@ public ChunkHeader readChunkHeader(long position) throws IOException { return ChunkHeader.deserializeFrom(tsFileInput, position); } + public InputStream wrapAsInputStream() throws IOException { + return this.tsFileInput.wrapAsInputStream(); + } + public Map>> getTimeseriesMetadataAndOffsetByDevice( MetadataIndexNode measurementNode, @@ -143,6 +186,11 @@ public Map> getTimeseriesMetadataOffsetByDevice( return timeseriesMetadataOffsetMap; } + private void acquireReadDataSizeWithCompactionReadRateLimiter(int readDataSize) { + CompactionTaskManager.getInstance().getCompactionReadOperationRateLimiter().acquire(1); + CompactionTaskManager.getInstance().getCompactionReadRateLimiter().acquire(readDataSize); + } + @Override public boolean equals(Object o) { return super.equals(o); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/io/CompactionTsFileWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/io/CompactionTsFileWriter.java index fcea93d6c2e3..7d89747d2f65 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/io/CompactionTsFileWriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/io/CompactionTsFileWriter.java @@ -36,7 +36,6 @@ import org.apache.tsfile.write.chunk.IChunkWriter; import org.apache.tsfile.write.record.Tablet.ColumnCategory; import org.apache.tsfile.write.writer.TsFileIOWriter; -import org.apache.tsfile.write.writer.tsmiterator.TSMIterator; import java.io.File; import java.io.IOException; @@ -151,14 +150,6 @@ public void endFile() throws IOException { .recordWriteInfo(type, CompactionIoDataType.METADATA, writtenDataSize); } - @Override - protected TSMIterator getTSMIterator() throws IOException { - return hasChunkMetadataInDisk - ? new CompactionDiskTSMIterator( - type, chunkMetadataTempFile, chunkGroupMetadataList, endPosInCMTForDevice) - : TSMIterator.getTSMIteratorInMemory(chunkGroupMetadataList); - } - public boolean isEmptyTargetFile() { return isEmptyTargetFile; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedReadOnlyMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedReadOnlyMemChunk.java index a3eb66b018d3..569db7347ab5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedReadOnlyMemChunk.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedReadOnlyMemChunk.java @@ -30,7 +30,6 @@ import org.apache.tsfile.file.metadata.AlignedChunkMetadata; import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; -import org.apache.tsfile.file.metadata.TableDeviceChunkMetadata; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.file.metadata.statistics.Statistics; import org.apache.tsfile.read.common.TimeRange; @@ -76,11 +75,10 @@ public AlignedReadOnlyMemChunk( timeColumnDeletion, valueColumnsDeletionList, context.isIgnoreAllNullRows()); - initAlignedChunkMetaFromTsBlock(context.isIgnoreAllNullRows()); + initAlignedChunkMetaFromTsBlock(); } - private void initAlignedChunkMetaFromTsBlock(boolean ignoreAllNullRows) - throws QueryProcessException { + private void initAlignedChunkMetaFromTsBlock() throws QueryProcessException { // Time chunk Statistics timeStatistics = Statistics.getStatsByType(TSDataType.VECTOR); IChunkMetadata timeChunkMetadata = @@ -162,9 +160,7 @@ private void initAlignedChunkMetaFromTsBlock(boolean ignoreAllNullRows) } } IChunkMetadata alignedChunkMetadata = - ignoreAllNullRows - ? new AlignedChunkMetadata(timeChunkMetadata, valueChunkMetadataList) - : new TableDeviceChunkMetadata(timeChunkMetadata, valueChunkMetadataList); + new AlignedChunkMetadata(timeChunkMetadata, valueChunkMetadataList); alignedChunkMetadata.setChunkLoader(new MemAlignedChunkLoader(context, this)); alignedChunkMetadata.setVersion(Long.MAX_VALUE); cachedMetaData = alignedChunkMetadata; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java index b6f5629528fa..3b2c5fe29449 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java @@ -80,7 +80,6 @@ import org.apache.iotdb.rpc.TSStatusCode; import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; import org.apache.tsfile.file.metadata.AlignedChunkMetadata; import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; @@ -1789,7 +1788,7 @@ RestorableTsFileIOWriter getWriter() { private void processAlignedChunkMetaDataFromFlushedMemTable( IDeviceID deviceID, - AbstractAlignedChunkMetadata alignedChunkMetadata, + AlignedChunkMetadata alignedChunkMetadata, Map> measurementToChunkMetaMap, Map> measurementToChunkHandleMap, String filePath) { @@ -1842,10 +1841,10 @@ private void buildChunkHandleForFlushedMemTable( Map> measurementToChunkMetaList, Map> measurementToChunkHandleList) { for (IChunkMetadata chunkMetadata : chunkMetadataList) { - if (chunkMetadata instanceof AbstractAlignedChunkMetadata) { + if (chunkMetadata instanceof AlignedChunkMetadata) { processAlignedChunkMetaDataFromFlushedMemTable( deviceID, - (AbstractAlignedChunkMetadata) chunkMetadata, + (AlignedChunkMetadata) chunkMetadata, measurementToChunkMetaList, measurementToChunkHandleList, this.tsFileResource.getTsFilePath()); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/DiskAlignedChunkLoader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/DiskAlignedChunkLoader.java index a7c6eb96d429..adef246ec8d9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/DiskAlignedChunkLoader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/DiskAlignedChunkLoader.java @@ -25,7 +25,7 @@ import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileID; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; -import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; +import org.apache.tsfile.file.metadata.AlignedChunkMetadata; import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; import org.apache.tsfile.read.common.Chunk; @@ -76,8 +76,7 @@ public IChunkReader getChunkReader(IChunkMetadata chunkMetaData, Filter globalTi throws IOException { long t1 = System.nanoTime(); try { - AbstractAlignedChunkMetadata alignedChunkMetadata = - (AbstractAlignedChunkMetadata) chunkMetaData; + AlignedChunkMetadata alignedChunkMetadata = (AlignedChunkMetadata) chunkMetaData; ChunkMetadata timeChunkMetadata = (ChunkMetadata) alignedChunkMetadata.getTimeChunkMetadata(); Chunk timeChunk = ChunkCache.getInstance() diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedChunkReader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedChunkReader.java index 12ac281dffc5..33c85408f0d3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedChunkReader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedChunkReader.java @@ -21,7 +21,7 @@ import org.apache.iotdb.db.storageengine.dataregion.memtable.AlignedReadOnlyMemChunk; -import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; +import org.apache.tsfile.file.metadata.AlignedChunkMetadata; import org.apache.tsfile.read.common.BatchData; import org.apache.tsfile.read.filter.basic.Filter; import org.apache.tsfile.read.reader.IChunkReader; @@ -42,7 +42,7 @@ public MemAlignedChunkReader(AlignedReadOnlyMemChunk readableChunk, Filter globa Collections.singletonList( new MemAlignedPageReader( readableChunk.getTsBlock(), - (AbstractAlignedChunkMetadata) readableChunk.getChunkMetaData(), + (AlignedChunkMetadata) readableChunk.getChunkMetaData(), globalTimeFilter)); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedPageReader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedPageReader.java index 6adfe09c0542..473cd7366c33 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedPageReader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedPageReader.java @@ -22,7 +22,7 @@ import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; +import org.apache.tsfile.file.metadata.AlignedChunkMetadata; import org.apache.tsfile.file.metadata.statistics.Statistics; import org.apache.tsfile.read.common.BatchData; import org.apache.tsfile.read.common.BatchDataFactory; @@ -45,7 +45,7 @@ public class MemAlignedPageReader implements IPageReader { private final TsBlock tsBlock; - private final AbstractAlignedChunkMetadata chunkMetadata; + private final AlignedChunkMetadata chunkMetadata; private Filter recordFilter; private PaginationController paginationController = UNLIMITED_PAGINATION_CONTROLLER; @@ -53,7 +53,7 @@ public class MemAlignedPageReader implements IPageReader { private TsBlockBuilder builder; public MemAlignedPageReader( - TsBlock tsBlock, AbstractAlignedChunkMetadata chunkMetadata, Filter recordFilter) { + TsBlock tsBlock, AlignedChunkMetadata chunkMetadata, Filter recordFilter) { this.tsBlock = tsBlock; this.chunkMetadata = chunkMetadata; this.recordFilter = recordFilter; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/metadata/DiskAlignedChunkMetadataLoader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/metadata/DiskAlignedChunkMetadataLoader.java index f243b51ee1a7..afc57079c88b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/metadata/DiskAlignedChunkMetadataLoader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/metadata/DiskAlignedChunkMetadataLoader.java @@ -26,8 +26,8 @@ import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.iotdb.db.utils.ModificationUtils; -import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; import org.apache.tsfile.file.metadata.AbstractAlignedTimeSeriesMetadata; +import org.apache.tsfile.file.metadata.AlignedChunkMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; import org.apache.tsfile.file.metadata.ITimeSeriesMetadata; import org.apache.tsfile.read.controller.IChunkMetadataLoader; @@ -83,7 +83,7 @@ public DiskAlignedChunkMetadataLoader( public List loadChunkMetadataList(ITimeSeriesMetadata timeSeriesMetadata) { final long t1 = System.nanoTime(); try { - List alignedChunkMetadataList = + List alignedChunkMetadataList = ((AbstractAlignedTimeSeriesMetadata) timeSeriesMetadata).getCopiedChunkMetadataList(); // when alignedChunkMetadataList.size() == 1, it means that the chunk statistics is same as diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java index 74bf838a8d3f..41bbbafe901c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java @@ -77,7 +77,7 @@ import java.util.stream.Stream; import static org.apache.iotdb.db.utils.constant.SqlConstant.ROOT; -import static org.apache.iotdb.db.utils.constant.SqlConstant.TREE_MODEL_DATABASE_PREFIX; +import static org.apache.iotdb.db.utils.constant.SqlConstant.ROOT_DOT; /** * {@link LoadTsFileManager} is used for dealing with {@link LoadTsFilePieceNode} and {@link @@ -445,8 +445,7 @@ private void write(DataPartitionInfo partitionInfo, ChunkData chunkData) throws // Table model needs to register TableSchema final String tableName = chunkData.getDevice() != null ? chunkData.getDevice().getTableName() : null; - if (tableName != null - && !(tableName.startsWith(TREE_MODEL_DATABASE_PREFIX) || tableName.equals(ROOT))) { + if (tableName != null && !(tableName.startsWith(ROOT_DOT) || tableName.equals(ROOT))) { writer .getSchema() .getTableSchemaMap() diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTableStatementDataTypeConvertExecutionVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTableStatementDataTypeConvertExecutionVisitor.java index 646185126a0e..de9dd0908879 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTableStatementDataTypeConvertExecutionVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTableStatementDataTypeConvertExecutionVisitor.java @@ -123,10 +123,6 @@ public Optional visitLoadTsFile( || result.getCode() == TSStatusCode.REDIRECTION_RECOMMEND.getStatusCode() || result.getCode() == TSStatusCode.LOAD_IDEMPOTENT_CONFLICT_EXCEPTION.getStatusCode())) { - LOGGER.warn( - "Failed to convert data type for LoadTsFileStatement: {}, status code is {}.", - loadTsFileStatement, - result.getCode()); return Optional.empty(); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTreeStatementDataTypeConvertExecutionVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTreeStatementDataTypeConvertExecutionVisitor.java index eb7efba36185..6f7d732e6fb4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTreeStatementDataTypeConvertExecutionVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/converter/LoadTreeStatementDataTypeConvertExecutionVisitor.java @@ -109,10 +109,6 @@ file, new IoTDBTreePattern(null), Long.MIN_VALUE, Long.MAX_VALUE, null, null)) { || result.getCode() == TSStatusCode.REDIRECTION_RECOMMEND.getStatusCode() || result.getCode() == TSStatusCode.LOAD_IDEMPOTENT_CONFLICT_EXCEPTION.getStatusCode())) { - LOGGER.warn( - "Failed to convert data type for LoadTsFileStatement: {}, status code is {}.", - loadTsFileStatement, - result.getCode()); return Optional.empty(); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingQueue.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingQueue.java index ca9bf096efa8..cf78aff6d4a2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingQueue.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingQueue.java @@ -97,9 +97,6 @@ public abstract class SubscriptionPrefetchingQueue { private final SubscriptionPrefetchingQueueStates states; - private static final long STATE_REPORT_INTERVAL_IN_MS = 10_000L; - private long lastStateReportTimestamp = System.currentTimeMillis(); - private volatile boolean isCompleted = false; private volatile boolean isClosed = false; @@ -262,7 +259,6 @@ public boolean executePrefetch() { if (isClosed()) { return false; } - reportStateIfNeeded(); // TODO: more refined behavior (prefetch/serialize/...) control if (states.shouldPrefetch()) { tryPrefetch(); @@ -278,13 +274,6 @@ public boolean executePrefetch() { } } - private void reportStateIfNeeded() { - if (System.currentTimeMillis() - lastStateReportTimestamp > STATE_REPORT_INTERVAL_IN_MS) { - LOGGER.info("Subscription: SubscriptionPrefetchingQueue state {}", this); - lastStateReportTimestamp = System.currentTimeMillis(); - } - } - @SafeVarargs private final void remapInFlightEventsSnapshot( final RemappingFunction... functions) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/response/SubscriptionEventTabletResponse.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/response/SubscriptionEventTabletResponse.java index c6150536f21a..cd93cd350ba0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/response/SubscriptionEventTabletResponse.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/response/SubscriptionEventTabletResponse.java @@ -116,6 +116,11 @@ public void fetchNextResponse(final long offset /* unused */) throws Exception { @Override public synchronized void nack() { + if (nextOffset.get() == 1) { + // do nothing if with complete tablets + return; + } + cleanUp(); // should not reset the iterator of batch when init diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/utils/TsFileRewriteOverPrecisedI64Scan.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/utils/TsFileRewriteOverPrecisedI64Scan.java deleted file mode 100644 index a93570905346..000000000000 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/utils/TsFileRewriteOverPrecisedI64Scan.java +++ /dev/null @@ -1,223 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iotdb.db.tools.utils; - -import org.apache.tsfile.encoding.encoder.TSEncodingBuilder; -import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.file.header.ChunkHeader; -import org.apache.tsfile.file.header.PageHeader; -import org.apache.tsfile.file.metadata.statistics.Statistics; -import org.apache.tsfile.read.common.BatchData; -import org.apache.tsfile.read.common.Chunk; -import org.apache.tsfile.read.reader.IPageReader; -import org.apache.tsfile.read.reader.chunk.ChunkReader; -import org.apache.tsfile.read.reader.chunk.TableChunkReader; -import org.apache.tsfile.utils.Pair; -import org.apache.tsfile.utils.TsPrimitiveType; -import org.apache.tsfile.write.chunk.ChunkWriterImpl; -import org.apache.tsfile.write.chunk.ValueChunkWriter; -import org.apache.tsfile.write.schema.MeasurementSchema; -import org.apache.tsfile.write.writer.TsFileIOWriter; - -import java.io.File; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.Collections; -import java.util.List; - -public class TsFileRewriteOverPrecisedI64Scan extends TsFileSequenceScan { - - private final File target; - private TsFileIOWriter writer; - private Chunk currTimeChunk; - - public TsFileRewriteOverPrecisedI64Scan(File target) { - this.target = target; - } - - public static void main(String[] args) { - File sourceFile = new File(args[0]); - File targetFile = new File(args[1]); - TsFileRewriteOverPrecisedI64Scan scan = new TsFileRewriteOverPrecisedI64Scan(targetFile); - scan.scanTsFile(sourceFile); - - long sourceLength = sourceFile.length(); - long targetLength = targetFile.length(); - System.out.printf( - "Before rewrite %d, after rewrite %d, ratio %f%n", - sourceLength, targetLength, sourceLength * 1.0 / targetLength); - } - - @Override - protected boolean onFileOpen(File file) throws IOException { - boolean shouldScan = super.onFileOpen(file); - if (shouldScan) { - writer = new TsFileIOWriter(target); - } - return shouldScan; - } - - @Override - protected void onFileEnd() throws IOException { - writer.endChunkGroup(); - writer.endFile(); - } - - @Override - protected void onChunkGroup() throws IOException { - if (currDeviceID != null) { - writer.endChunkGroup(); - } - super.onChunkGroup(); - writer.startChunkGroup(currDeviceID); - } - - @Override - protected void onChunk(PageVisitor pageVisitor) throws IOException { - reader.position(reader.position() - 1); - Chunk chunk = reader.readMemChunk(reader.position()); - chunk = - new Chunk( - chunk.getHeader(), - chunk.getData(), - Collections.emptyList(), - Statistics.getStatsByType(chunk.getHeader().getDataType())); - currMeasurementID = chunk.getHeader().getMeasurementID(); - currTimeseriesID = new Pair<>(currDeviceID, currMeasurementID); - if (!currDeviceAligned) { - onNonAlignedChunk(chunk); - } else { - onAlignedChunk(chunk); - } - System.out.println("Processed a chunk of " + currDeviceID + "." + currMeasurementID); - reader.position( - reader.position() - + chunk.getHeader().getSerializedSize() - + chunk.getHeader().getDataSize()); - } - - private void onNonAlignedChunk(Chunk chunk) throws IOException { - if (chunk.getHeader().getDataType() != TSDataType.INT64) { - writer.writeChunk(chunk); - } else { - if (!rewriteInt64ChunkNonAligned(chunk)) { - writer.writeChunk(chunk); - } - } - } - - private void onAlignedChunk(Chunk chunk) throws IOException { - if (isTimeChunk || chunk.getHeader().getDataType() != TSDataType.INT64) { - writer.writeChunk(chunk); - if (isTimeChunk) { - currTimeChunk = chunk; - } - } else { - if (!rewriteInt64ChunkAligned(chunk)) { - writer.writeChunk(chunk); - } - } - } - - private boolean rewriteInt64ChunkNonAligned(Chunk chunk) throws IOException { - ChunkReader chunkReader = new ChunkReader(chunk); - ChunkHeader header = chunk.getHeader(); - List pageReaders = chunkReader.loadPageReaderList(); - ChunkWriterImpl chunkWriter = - new ChunkWriterImpl( - new MeasurementSchema( - header.getMeasurementID(), - TSDataType.INT32, - header.getEncodingType(), - header.getCompressionType())); - - for (IPageReader pageReader : pageReaders) { - BatchData batchData = pageReader.getAllSatisfiedPageData(); - while (batchData.hasCurrent()) { - int intVal = (int) batchData.getLong(); - if (intVal != batchData.getLong()) { - // the chunk is not over-precised - return false; - } - chunkWriter.write(batchData.currentTime(), (int) batchData.getLong()); - } - chunkWriter.sealCurrentPage(); - } - chunkWriter.writeToFileWriter(writer); - return true; - } - - private boolean rewriteInt64ChunkAligned(Chunk chunk) throws IOException { - // use TableChunkReader so that nulls will not be skipped - TableChunkReader chunkReader = - new TableChunkReader(currTimeChunk, Collections.singletonList(chunk), null); - ChunkHeader header = chunk.getHeader(); - List pageReaders = chunkReader.loadPageReaderList(); - ValueChunkWriter valueChunkWriter = - new ValueChunkWriter( - header.getMeasurementID(), - header.getCompressionType(), - TSDataType.INT32, - header.getEncodingType(), - TSEncodingBuilder.getEncodingBuilder(header.getEncodingType()) - .getEncoder(TSDataType.INT32)); - - for (IPageReader pageReader : pageReaders) { - BatchData batchData = pageReader.getAllSatisfiedPageData(); - while (batchData.hasCurrent()) { - TsPrimitiveType[] vector = batchData.getVector(); - boolean isNull = vector[0] == null; - if (!isNull) { - int intVal = (int) vector[0].getLong(); - if (intVal != vector[0].getLong()) { - // the chunk is not over-precised - return false; - } - valueChunkWriter.write(batchData.currentTime(), intVal, false); - } else { - valueChunkWriter.write(batchData.currentTime(), 0, true); - } - } - valueChunkWriter.sealCurrentPage(); - } - valueChunkWriter.writeToFileWriter(writer); - return true; - } - - @Override - protected void onTimePage(PageHeader pageHeader, ByteBuffer pageData, ChunkHeader chunkHeader) { - // do nothing - } - - @Override - protected void onValuePage(PageHeader pageHeader, ByteBuffer pageData, ChunkHeader chunkHeader) { - // do nothing - } - - @Override - protected void onNonAlignedPage( - PageHeader pageHeader, ByteBuffer pageData, ChunkHeader chunkHeader) { - // do nothing - } - - @Override - protected void onException(Throwable t) { - throw new RuntimeException(t); - } -} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/utils/TsFileRewriteSmallRangeI64Scan.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/utils/TsFileRewriteSmallRangeI64Scan.java deleted file mode 100644 index 360fc1c05768..000000000000 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/utils/TsFileRewriteSmallRangeI64Scan.java +++ /dev/null @@ -1,230 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iotdb.db.tools.utils; - -import org.apache.tsfile.encoding.encoder.TSEncodingBuilder; -import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.file.header.ChunkHeader; -import org.apache.tsfile.file.header.PageHeader; -import org.apache.tsfile.file.metadata.statistics.Statistics; -import org.apache.tsfile.read.common.BatchData; -import org.apache.tsfile.read.common.Chunk; -import org.apache.tsfile.read.reader.IPageReader; -import org.apache.tsfile.read.reader.chunk.ChunkReader; -import org.apache.tsfile.read.reader.chunk.TableChunkReader; -import org.apache.tsfile.utils.Pair; -import org.apache.tsfile.utils.TsPrimitiveType; -import org.apache.tsfile.write.chunk.ChunkWriterImpl; -import org.apache.tsfile.write.chunk.ValueChunkWriter; -import org.apache.tsfile.write.schema.MeasurementSchema; -import org.apache.tsfile.write.writer.TsFileIOWriter; - -import java.io.File; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.Collections; -import java.util.List; - -public class TsFileRewriteSmallRangeI64Scan extends TsFileSequenceScan { - - private final File target; - private TsFileIOWriter writer; - private Chunk currTimeChunk; - - public TsFileRewriteSmallRangeI64Scan(File target) { - this.target = target; - } - - public static void main(String[] args) throws IOException { - File sourceFile = new File(args[0]); - File targetFile = new File(args[1]); - TsFileRewriteSmallRangeI64Scan scan = new TsFileRewriteSmallRangeI64Scan(targetFile); - scan.scanTsFile(sourceFile); - - long sourceLength = sourceFile.length(); - long targetLength = targetFile.length(); - System.out.printf( - "Before rewrite %d, after rewrite %d, ratio %f%n", - sourceLength, targetLength, sourceLength * 1.0 / targetLength); - } - - @Override - protected boolean onFileOpen(File file) throws IOException { - boolean shouldScan = super.onFileOpen(file); - if (shouldScan) { - writer = new TsFileIOWriter(target); - } - return shouldScan; - } - - @Override - protected void onFileEnd() throws IOException { - writer.endChunkGroup(); - writer.endFile(); - } - - @Override - protected void onChunkGroup() throws IOException { - if (currDeviceID != null) { - writer.endChunkGroup(); - } - super.onChunkGroup(); - writer.startChunkGroup(currDeviceID); - } - - @Override - protected void onChunk(PageVisitor pageVisitor) throws IOException { - reader.position(reader.position() - 1); - Chunk chunk = reader.readMemChunk(reader.position()); - chunk = - new Chunk( - chunk.getHeader(), - chunk.getData(), - Collections.emptyList(), - Statistics.getStatsByType(chunk.getHeader().getDataType())); - currMeasurementID = chunk.getHeader().getMeasurementID(); - currTimeseriesID = new Pair<>(currDeviceID, currMeasurementID); - if (!currDeviceAligned) { - onNonAlignedChunk(chunk); - } else { - onAlignedChunk(chunk); - } - System.out.println("Processed a chunk of " + currDeviceID + "." + currMeasurementID); - reader.position( - reader.position() - + chunk.getHeader().getSerializedSize() - + chunk.getHeader().getDataSize()); - } - - private void onNonAlignedChunk(Chunk chunk) throws IOException { - if (chunk.getHeader().getDataType() != TSDataType.INT64) { - writer.writeChunk(chunk); - } else { - if (!rewriteInt64ChunkNonAligned(chunk)) { - writer.writeChunk(chunk); - } - } - } - - private void onAlignedChunk(Chunk chunk) throws IOException { - if (isTimeChunk || chunk.getHeader().getDataType() != TSDataType.INT64) { - writer.writeChunk(chunk); - if (isTimeChunk) { - currTimeChunk = chunk; - } - } else { - if (!rewriteInt64ChunkAligned(chunk)) { - writer.writeChunk(chunk); - } - } - } - - private boolean rewriteInt64ChunkNonAligned(Chunk chunk) throws IOException { - ChunkReader chunkReader = new ChunkReader(chunk); - ChunkHeader header = chunk.getHeader(); - List pageReaders = chunkReader.loadPageReaderList(); - ChunkWriterImpl chunkWriter = - new ChunkWriterImpl( - new MeasurementSchema( - header.getMeasurementID(), - TSDataType.INT32, - header.getEncodingType(), - header.getCompressionType())); - - Long firstVal = null; - for (IPageReader pageReader : pageReaders) { - BatchData batchData = pageReader.getAllSatisfiedPageData(); - while (batchData.hasCurrent()) { - if (firstVal == null) { - firstVal = batchData.getLong(); - } - long diff = batchData.getLong() - firstVal; - if (diff > Integer.MAX_VALUE) { - return false; - } - chunkWriter.write(batchData.currentTime(), (int) diff); - batchData.next(); - } - chunkWriter.sealCurrentPage(); - } - chunkWriter.writeToFileWriter(writer); - return true; - } - - private boolean rewriteInt64ChunkAligned(Chunk chunk) throws IOException { - // use TableChunkReader so that nulls will not be skipped - TableChunkReader chunkReader = - new TableChunkReader(currTimeChunk, Collections.singletonList(chunk), null); - ChunkHeader header = chunk.getHeader(); - List pageReaders = chunkReader.loadPageReaderList(); - ValueChunkWriter valueChunkWriter = - new ValueChunkWriter( - header.getMeasurementID(), - header.getCompressionType(), - TSDataType.INT32, - header.getEncodingType(), - TSEncodingBuilder.getEncodingBuilder(header.getEncodingType()) - .getEncoder(TSDataType.INT32)); - - Long firstVal = null; - for (IPageReader pageReader : pageReaders) { - BatchData batchData = pageReader.getAllSatisfiedPageData(); - while (batchData.hasCurrent()) { - TsPrimitiveType[] vector = batchData.getVector(); - boolean isNull = vector[0] == null; - if (!isNull) { - if (firstVal == null) { - firstVal = vector[0].getLong(); - } - long diff = vector[0].getLong() - firstVal; - if (diff > Integer.MAX_VALUE) { - return false; - } - valueChunkWriter.write(batchData.currentTime(), (int) diff, false); - } else { - valueChunkWriter.write(batchData.currentTime(), 0, true); - } - } - valueChunkWriter.sealCurrentPage(); - } - valueChunkWriter.writeToFileWriter(writer); - return true; - } - - @Override - protected void onTimePage(PageHeader pageHeader, ByteBuffer pageData, ChunkHeader chunkHeader) { - // do nothing - } - - @Override - protected void onValuePage(PageHeader pageHeader, ByteBuffer pageData, ChunkHeader chunkHeader) { - // do nothing - } - - @Override - protected void onNonAlignedPage( - PageHeader pageHeader, ByteBuffer pageData, ChunkHeader chunkHeader) { - // do nothing - } - - @Override - protected void onException(Throwable t) { - throw new RuntimeException(t); - } -} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/utils/TsFileSequenceScan.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/utils/TsFileSequenceScan.java index eddb7a0e80c4..d9ea636948f8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/utils/TsFileSequenceScan.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/utils/TsFileSequenceScan.java @@ -44,9 +44,6 @@ public abstract class TsFileSequenceScan { protected String currMeasurementID; protected Pair currTimeseriesID; protected boolean currChunkOnePage; - protected ChunkHeader currChunkHeader; - protected boolean currDeviceAligned; - protected boolean isTimeChunk; public TsFileSequenceScan() {} @@ -63,26 +60,23 @@ protected boolean onFileOpen(File file) throws IOException { protected void onFileEnd() throws IOException {} protected void onChunk(PageVisitor pageVisitor) throws IOException { - currChunkHeader = reader.readChunkHeader(marker); - if (currChunkHeader.getDataSize() == 0) { + ChunkHeader chunkHeader = reader.readChunkHeader(marker); + if (chunkHeader.getDataSize() == 0) { // empty value chunk return; } - currMeasurementID = currChunkHeader.getMeasurementID(); + currMeasurementID = chunkHeader.getMeasurementID(); currTimeseriesID = new Pair<>(currDeviceID, currMeasurementID); - int dataSize = currChunkHeader.getDataSize(); - onChunkData(pageVisitor, dataSize); - } + int dataSize = chunkHeader.getDataSize(); - protected void onChunkData(PageVisitor pageVisitor, int dataSize) throws IOException { while (dataSize > 0) { PageHeader pageHeader = reader.readPageHeader( - currChunkHeader.getDataType(), - (currChunkHeader.getChunkType() & 0x3F) == MetaMarker.CHUNK_HEADER); - ByteBuffer pageData = reader.readPage(pageHeader, currChunkHeader.getCompressionType()); - pageVisitor.onPage(pageHeader, pageData, currChunkHeader); + chunkHeader.getDataType(), + (chunkHeader.getChunkType() & 0x3F) == MetaMarker.CHUNK_HEADER); + ByteBuffer pageData = reader.readPage(pageHeader, chunkHeader.getCompressionType()); + pageVisitor.onPage(pageHeader, pageData, chunkHeader); dataSize -= pageHeader.getSerializedPageSize(); } } @@ -129,38 +123,26 @@ public void scanTsFile(File tsFile) { switch (marker) { case MetaMarker.ONLY_ONE_PAGE_CHUNK_HEADER: currChunkOnePage = true; - currDeviceAligned = false; - isTimeChunk = false; onChunk(this::onNonAlignedPage); break; case MetaMarker.CHUNK_HEADER: currChunkOnePage = false; - currDeviceAligned = false; - isTimeChunk = false; onChunk(this::onNonAlignedPage); break; case MetaMarker.ONLY_ONE_PAGE_TIME_CHUNK_HEADER: currChunkOnePage = true; - currDeviceAligned = true; - isTimeChunk = true; onChunk(this::onTimePage); break; case MetaMarker.TIME_CHUNK_HEADER: currChunkOnePage = false; - currDeviceAligned = true; - isTimeChunk = true; onChunk(this::onTimePage); break; case MetaMarker.ONLY_ONE_PAGE_VALUE_CHUNK_HEADER: currChunkOnePage = true; - currDeviceAligned = true; - isTimeChunk = false; onChunk(this::onValuePage); break; case MetaMarker.VALUE_CHUNK_HEADER: currChunkOnePage = false; - currDeviceAligned = true; - isTimeChunk = false; onChunk(this::onValuePage); break; case MetaMarker.CHUNK_GROUP_HEADER: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/utils/TsFileStatisticScan.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/utils/TsFileStatisticScan.java deleted file mode 100644 index 094f12dcc846..000000000000 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/utils/TsFileStatisticScan.java +++ /dev/null @@ -1,266 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iotdb.db.tools.utils; - -import org.apache.tsfile.common.conf.TSFileDescriptor; -import org.apache.tsfile.encoding.decoder.Decoder; -import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.file.header.ChunkHeader; -import org.apache.tsfile.file.header.PageHeader; -import org.apache.tsfile.file.metadata.IDeviceID; -import org.apache.tsfile.file.metadata.enums.TSEncoding; -import org.apache.tsfile.file.metadata.statistics.Statistics; -import org.apache.tsfile.read.common.BatchData; -import org.apache.tsfile.read.reader.page.AlignedPageReader; -import org.apache.tsfile.read.reader.page.PageReader; -import org.apache.tsfile.utils.Binary; -import org.apache.tsfile.utils.Pair; -import org.apache.tsfile.utils.TsPrimitiveType; - -import java.io.File; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.Collections; -import java.util.EnumMap; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Set; - -public class TsFileStatisticScan extends TsFileSequenceScan { - - // (deviceId, measurementId) -> data type - private final Map, TSDataType> seriesDataTypeMap = new HashMap<>(); - private final Map dataTypeSizeMap = new EnumMap<>(TSDataType.class); - private final Map dataTypePointMap = new EnumMap<>(TSDataType.class); - private final Map dataTypeChunkMap = new EnumMap<>(TSDataType.class); - private final List distinctBinaryValueNumInChunks = new LinkedList<>(); - // the number of Int64 chunks that can be represented by Int32 - private int overPrecisedInt64ChunkNum; - // the number of Int64 chunks that cannot be represented by Int32 - private int justPrecisedInt64ChunkNum; - // the number of Int64 chunks whose range can be represented by Int32 - private int smallRangeInt64ChunkNum; - // the number of Int64 chunks whose range cannot be represented by Int32 - private int largeRangeInt64ChunkNum; - private boolean currChunkJustPrecised; - private boolean currChunkLargeRange; - private Set distinctBinarySet = new HashSet<>(); - private PageHeader currTimePageHeader; - private ByteBuffer currTimePageBuffer; - - public static void main(String[] args) { - TsFileStatisticScan t = new TsFileStatisticScan(); - t.scanTsFile(new File(args[0])); - } - - @Override - protected void onFileEnd() throws IOException { - super.onFileEnd(); - Map seriesTypeCountMap = new EnumMap<>(TSDataType.class); - for (TSDataType type : seriesDataTypeMap.values()) { - seriesTypeCountMap.compute(type, (t, v) -> v == null ? 1 : v + 1); - } - System.out.println("Series data type count: " + seriesTypeCountMap); - System.out.println( - "Int64 series statistics: overPrecised " - + overPrecisedInt64ChunkNum - + ", justPrecised " - + justPrecisedInt64ChunkNum - + ", smallRange " - + smallRangeInt64ChunkNum - + ", largeRange " - + largeRangeInt64ChunkNum); - System.out.println("data type -> size: " + dataTypeSizeMap); - System.out.println("data type -> point count: " + dataTypePointMap); - System.out.println("data type -> chunk count: " + dataTypeChunkMap); - System.out.println( - "average distinct binary value num: " - + distinctBinaryValueNumInChunks.stream().mapToInt(i -> i).average().orElse(0.0)); - } - - @Override - protected void onChunk(PageVisitor pageVisitor) throws IOException { - currChunkJustPrecised = false; - currChunkLargeRange = false; - - super.onChunk(pageVisitor); - if (!isTimeChunk) { - seriesDataTypeMap.computeIfAbsent(currTimeseriesID, cid -> currChunkHeader.getDataType()); - dataTypeSizeMap.compute( - currChunkHeader.getDataType(), - (type, size) -> - size == null - ? currChunkHeader.getSerializedSize() + currChunkHeader.getDataSize() - : size + currChunkHeader.getSerializedSize() + currChunkHeader.getDataSize()); - dataTypeChunkMap.compute( - currChunkHeader.getDataType(), (type, size) -> size == null ? 1 : size + 1); - } - if (currChunkHeader.getDataType() == TSDataType.INT64) { - if (currChunkJustPrecised) { - justPrecisedInt64ChunkNum++; - } else { - overPrecisedInt64ChunkNum++; - } - - if (currChunkLargeRange) { - largeRangeInt64ChunkNum++; - } else { - smallRangeInt64ChunkNum++; - } - } else if (currChunkHeader.getDataType() == TSDataType.TEXT - || currChunkHeader.getDataType() == TSDataType.STRING) { - distinctBinaryValueNumInChunks.add(distinctBinarySet.size()); - distinctBinarySet.clear(); - } - } - - @Override - protected void onTimePage(PageHeader pageHeader, ByteBuffer pageData, ChunkHeader chunkHeader) - throws IOException { - currTimePageHeader = pageHeader; - currTimePageBuffer = pageData; - } - - @Override - protected void onValuePage(PageHeader pageHeader, ByteBuffer pageData, ChunkHeader chunkHeader) - throws IOException { - TSDataType dataType = chunkHeader.getDataType(); - if (dataType == TSDataType.INT64) { - Statistics statistics = (Statistics) pageHeader.getStatistics(); - Long minValue = statistics.getMinValue(); - Long maxValue = statistics.getMaxValue(); - if (minValue < Integer.MIN_VALUE || maxValue > Integer.MAX_VALUE) { - currChunkJustPrecised = true; - } - if (maxValue - minValue > Integer.MAX_VALUE) { - currChunkLargeRange = true; - } - } else if (dataType == TSDataType.TEXT || dataType == TSDataType.STRING) { - AlignedPageReader pageReader = - new AlignedPageReader( - currTimePageHeader, - currTimePageBuffer, - Decoder.getDecoderByType( - TSEncoding.valueOf(TSFileDescriptor.getInstance().getConfig().getTimeEncoder()), - TSDataType.INT64), - Collections.singletonList(pageHeader), - Collections.singletonList(pageData), - Collections.singletonList(dataType), - Collections.singletonList( - Decoder.getDecoderByType(chunkHeader.getEncodingType(), dataType)), - null); - BatchData batchData = pageReader.getAllSatisfiedPageData(); - while (batchData.hasCurrent()) { - TsPrimitiveType[] vector = batchData.getVector(); - if (vector[0] != null) { - distinctBinarySet.add(vector[0].getBinary()); - } - batchData.next(); - } - } - dataTypePointMap.compute( - currChunkHeader.getDataType(), - (type, size) -> - size == null - ? pageHeader.getStatistics().getCount() - : size + pageHeader.getStatistics().getCount()); - } - - @Override - protected void onNonAlignedPage( - PageHeader pageHeader, ByteBuffer pageData, ChunkHeader chunkHeader) throws IOException { - TSDataType dataType = chunkHeader.getDataType(); - if (pageHeader.getStatistics() != null) { - if (dataType == TSDataType.INT64) { - Statistics statistics = (Statistics) pageHeader.getStatistics(); - Long minValue = statistics.getMinValue(); - Long maxValue = statistics.getMaxValue(); - if (minValue < Integer.MIN_VALUE || maxValue > Integer.MAX_VALUE) { - currChunkJustPrecised = true; - } - if (maxValue - minValue > Integer.MAX_VALUE) { - currChunkLargeRange = true; - } - } else if (dataType == TSDataType.TEXT || dataType == TSDataType.STRING) { - PageReader pageReader = - new PageReader( - pageHeader, - pageData, - chunkHeader.getDataType(), - Decoder.getDecoderByType(chunkHeader.getEncodingType(), chunkHeader.getDataType()), - Decoder.getDecoderByType( - TSEncoding.valueOf(TSFileDescriptor.getInstance().getConfig().getTimeEncoder()), - TSDataType.INT64)); - BatchData allSatisfiedPageData = pageReader.getAllSatisfiedPageData(true); - while (allSatisfiedPageData.hasCurrent()) { - distinctBinarySet.add(allSatisfiedPageData.getBinary()); - allSatisfiedPageData.next(); - } - } - dataTypePointMap.compute( - currChunkHeader.getDataType(), - (type, size) -> - size == null - ? pageHeader.getStatistics().getCount() - : size + pageHeader.getStatistics().getCount()); - } else { - PageReader pageReader = - new PageReader( - pageHeader, - pageData, - chunkHeader.getDataType(), - Decoder.getDecoderByType(chunkHeader.getEncodingType(), chunkHeader.getDataType()), - Decoder.getDecoderByType( - TSEncoding.valueOf(TSFileDescriptor.getInstance().getConfig().getTimeEncoder()), - TSDataType.INT64)); - BatchData allSatisfiedPageData = pageReader.getAllSatisfiedPageData(true); - long minLongValue = Long.MAX_VALUE; - long maxLongValue = Long.MIN_VALUE; - int cnt = 0; - while (allSatisfiedPageData.hasCurrent()) { - if (dataType == TSDataType.INT64) { - long val = (long) allSatisfiedPageData.currentValue(); - if (val < Integer.MIN_VALUE || val > Integer.MAX_VALUE) { - currChunkJustPrecised = true; - } - minLongValue = Math.min(minLongValue, val); - maxLongValue = Math.max(maxLongValue, val); - } else if (dataType == TSDataType.TEXT || dataType == TSDataType.STRING) { - distinctBinarySet.add(allSatisfiedPageData.getBinary()); - } - cnt++; - allSatisfiedPageData.next(); - } - int finalCnt = cnt; - dataTypePointMap.compute( - currChunkHeader.getDataType(), (type, size) -> size == null ? finalCnt : size + finalCnt); - if (maxLongValue - minLongValue > Integer.MAX_VALUE) { - currChunkLargeRange = true; - } - } - } - - @Override - protected void onException(Throwable t) { - t.printStackTrace(); - } -} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ErrorHandlingUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ErrorHandlingUtils.java index 57527c85af86..247a0a3fe828 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ErrorHandlingUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ErrorHandlingUtils.java @@ -106,11 +106,7 @@ public static TSStatus onQueryException(Exception e, String operation, TSStatusC || status.getCode() == TSStatusCode.ILLEGAL_PATH.getStatusCode() || status.getCode() == TSStatusCode.NUMERIC_VALUE_OUT_OF_RANGE.getStatusCode() || status.getCode() == TSStatusCode.DIVISION_BY_ZERO.getStatusCode() - || status.getCode() == TSStatusCode.DATE_OUT_OF_RANGE.getStatusCode() - || status.getCode() == TSStatusCode.TABLE_NOT_EXISTS.getStatusCode() - || status.getCode() == TSStatusCode.TABLE_ALREADY_EXISTS.getStatusCode() - || status.getCode() == TSStatusCode.COLUMN_NOT_EXISTS.getStatusCode() - || status.getCode() == TSStatusCode.COLUMN_ALREADY_EXISTS.getStatusCode()) { + || status.getCode() == TSStatusCode.DATE_OUT_OF_RANGE.getStatusCode()) { LOGGER.info(message); } else { LOGGER.warn(message, e); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ModificationUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ModificationUtils.java index f9a6d28e2b98..599662076480 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ModificationUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ModificationUtils.java @@ -25,7 +25,6 @@ import org.apache.iotdb.db.storageengine.dataregion.modification.TableDeletionEntry; import org.apache.iotdb.db.storageengine.dataregion.modification.TreeDeletionEntry; -import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; import org.apache.tsfile.file.metadata.AlignedChunkMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; import org.apache.tsfile.file.metadata.IDeviceID; @@ -102,7 +101,7 @@ public static void modifyAlignedChunkMetaData( } private static void modifyValueColumns( - AbstractAlignedChunkMetadata metaData, List> valueColumnsModifications) { + AlignedChunkMetadata metaData, List> valueColumnsModifications) { List valueChunkMetadataList = metaData.getValueChunkMetadataList(); // deal with each sub sensor for (int j = 0; j < valueChunkMetadataList.size(); j++) { @@ -117,7 +116,7 @@ private static void modifyValueColumns( } private static boolean areAllValueColumnsDeleted( - AbstractAlignedChunkMetadata alignedChunkMetadata, boolean modified) { + AlignedChunkMetadata alignedChunkMetadata, boolean modified) { // the whole aligned path need to be removed, only set to be true if all the sub sensors // are deleted and ignoreAllNullRows is true @@ -157,11 +156,11 @@ private static boolean areAllValueColumnsDeleted( } public static void modifyAlignedChunkMetaData( - List chunkMetaData, + List chunkMetaData, List timeColumnModifications, List> valueColumnsModifications, boolean ignoreAllNullRows) { - for (AbstractAlignedChunkMetadata metaData : chunkMetaData) { + for (AlignedChunkMetadata metaData : chunkMetaData) { IChunkMetadata timeColumnChunkMetadata = metaData.getTimeChunkMetadata(); for (ModEntry modification : timeColumnModifications) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TimestampPrecisionUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TimestampPrecisionUtils.java index 676a50237824..768d0030e96d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TimestampPrecisionUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TimestampPrecisionUtils.java @@ -70,7 +70,7 @@ public static void checkTimestampPrecision(long time) { } switch (TIMESTAMP_PRECISION) { case "ms": - if (time > 10_000_000_000_000L || time < -10_000_000_000_000L) { + if (time > 10_000_000_000_000L) { throw new SemanticException( String.format( "Current system timestamp precision is %s, " @@ -79,7 +79,7 @@ public static void checkTimestampPrecision(long time) { } break; case "us": - if (time > 10_000_000_000_000_000L || time < -10_000_000_000_000L) { + if (time > 10_000_000_000_000_000L) { throw new SemanticException( String.format( "Current system timestamp precision is %s, " diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/constant/SqlConstant.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/constant/SqlConstant.java index 625af8935614..cad8f229c4eb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/constant/SqlConstant.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/constant/SqlConstant.java @@ -36,7 +36,7 @@ protected SqlConstant() { public static final String ROOT = "root"; - public static final String TREE_MODEL_DATABASE_PREFIX = "root."; + public static final String ROOT_DOT = "root."; public static final String QUOTE = "'"; public static final String DQUOTE = "\""; diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/Utils.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/Utils.java index 327d4a34c39e..0c6eac0dbed1 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/Utils.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/Utils.java @@ -41,7 +41,6 @@ public static List createMockTsBlocks(int numOfTsBlocks, long mockTsBlo for (int i = 0; i < numOfTsBlocks; i++) { TsBlock mockTsBlock = Mockito.mock(TsBlock.class); Mockito.when(mockTsBlock.getRetainedSizeInBytes()).thenReturn(mockTsBlockSize); - Mockito.when(mockTsBlock.getSizeInBytes()).thenReturn(mockTsBlockSize); mockTsBlocks.add(mockTsBlock); } @@ -51,7 +50,6 @@ public static List createMockTsBlocks(int numOfTsBlocks, long mockTsBlo public static TsBlock createMockTsBlock(long mockTsBlockSize) { TsBlock mockTsBlock = Mockito.mock(TsBlock.class); Mockito.when(mockTsBlock.getRetainedSizeInBytes()).thenReturn(mockTsBlockSize); - Mockito.when(mockTsBlock.getSizeInBytes()).thenReturn(mockTsBlockSize); return mockTsBlock; } @@ -146,7 +144,6 @@ public static TsBlockSerde createMockTsBlockSerde(long mockTsBlockSize) { TsBlockSerde mockTsBlockSerde = Mockito.mock(TsBlockSerde.class); TsBlock mockTsBlock = Mockito.mock(TsBlock.class); Mockito.when(mockTsBlock.getRetainedSizeInBytes()).thenReturn(mockTsBlockSize); - Mockito.when(mockTsBlock.getSizeInBytes()).thenReturn(mockTsBlockSize); Mockito.when(mockTsBlockSerde.deserialize(Mockito.any(ByteBuffer.class))) .thenReturn(mockTsBlock); return mockTsBlockSerde; diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/MergeTreeSortOperatorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/MergeTreeSortOperatorTest.java index 6041f37bcec4..b479a5787907 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/MergeTreeSortOperatorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/MergeTreeSortOperatorTest.java @@ -55,7 +55,6 @@ import org.apache.iotdb.db.storageengine.dataregion.read.reader.series.SeriesReaderTestUtil; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.iotdb.db.utils.datastructure.SortKey; -import org.apache.iotdb.isession.SessionConfig; import com.google.common.collect.ImmutableList; import com.google.common.util.concurrent.ListenableFuture; @@ -1845,11 +1844,6 @@ public IClientSession.SqlDialect getSQLDialect() { return IClientSession.SqlDialect.TREE; } - @Override - public String getUser() { - return SessionConfig.DEFAULT_USER; - } - @Override public void start() {} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/OperatorMemoryTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/OperatorMemoryTest.java index e7f662ab573d..1bba28545bf3 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/OperatorMemoryTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/OperatorMemoryTest.java @@ -253,7 +253,6 @@ public void pipelineExchangeOperatorTest() { public void lastCacheScanOperatorTest() { TsBlock tsBlock = Mockito.mock(TsBlock.class); Mockito.when(tsBlock.getRetainedSizeInBytes()).thenReturn(1024L); - Mockito.when(tsBlock.getSizeInBytes()).thenReturn(1024L); LastCacheScanOperator lastCacheScanOperator = new LastCacheScanOperator(null, null, tsBlock); assertEquals(1024, lastCacheScanOperator.calculateMaxPeekMemory()); @@ -384,7 +383,6 @@ public void lastQueryOperatorTest() { public void lastQuerySortOperatorTest() { TsBlock tsBlock = Mockito.mock(TsBlock.class); Mockito.when(tsBlock.getRetainedSizeInBytes()).thenReturn(16 * 1024L); - Mockito.when(tsBlock.getSizeInBytes()).thenReturn(16 * 1024L); Mockito.when(tsBlock.getPositionCount()).thenReturn(16); List children = new ArrayList<>(4); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGeneratorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGeneratorTest.java index 30024d5e59c2..06b51954d533 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGeneratorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGeneratorTest.java @@ -783,9 +783,6 @@ public void testComplexGrantRevoke() { // 1. test complex privilege on single path :"root.**" Set allPriv = new HashSet<>(); for (PrivilegeType type : PrivilegeType.values()) { - if (type.isRelationalPrivilege()) { - continue; - } allPriv.add(type.toString()); } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ShowQueriesTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ShowQueriesTest.java index 63b7b783d747..bdf44401aade 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ShowQueriesTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ShowQueriesTest.java @@ -32,7 +32,6 @@ import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.QUERY_ID_TABLE_MODEL; import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.START_TIME_TABLE_MODEL; import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.STATEMENT; -import static org.apache.iotdb.commons.schema.column.ColumnHeaderConstant.USER; import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanAssert.assertPlan; import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.collect; import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.exchange; @@ -61,8 +60,7 @@ public void testNormal() { START_TIME_TABLE_MODEL, DATA_NODE_ID_TABLE_MODEL, ELAPSED_TIME_TABLE_MODEL, - STATEMENT.toLowerCase(Locale.ENGLISH), - USER.toLowerCase(Locale.ENGLISH))))); + STATEMENT.toLowerCase(Locale.ENGLISH))))); // - Exchange // Output - Collect - Exchange @@ -139,7 +137,6 @@ public void testNonSelectAll() { START_TIME_TABLE_MODEL, DATA_NODE_ID_TABLE_MODEL, ELAPSED_TIME_TABLE_MODEL, - STATEMENT.toLowerCase(Locale.ENGLISH), - USER.toLowerCase(Locale.ENGLISH)))))); + STATEMENT.toLowerCase(Locale.ENGLISH)))))); } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/FastCompactionPerformerWithInconsistentCompressionTypeAndEncodingTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/FastCompactionPerformerWithInconsistentCompressionTypeAndEncodingTest.java index f62480c55291..55cb8625da37 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/FastCompactionPerformerWithInconsistentCompressionTypeAndEncodingTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/FastCompactionPerformerWithInconsistentCompressionTypeAndEncodingTest.java @@ -32,7 +32,7 @@ import org.apache.tsfile.file.MetaMarker; import org.apache.tsfile.file.header.ChunkHeader; import org.apache.tsfile.file.header.PageHeader; -import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; +import org.apache.tsfile.file.metadata.AlignedChunkMetadata; import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; import org.apache.tsfile.file.metadata.IDeviceID; @@ -622,9 +622,9 @@ private void validateSingleTsFileWithAlignedSeries(TsFileSequenceReader reader) throws IOException { Map compressionTypeMap = new HashMap<>(); for (IDeviceID device : reader.getAllDevices()) { - List alignedChunkMetadataList = + List alignedChunkMetadataList = reader.getAlignedChunkMetadata(device, true); - for (AbstractAlignedChunkMetadata alignedChunkMetadata : alignedChunkMetadataList) { + for (AlignedChunkMetadata alignedChunkMetadata : alignedChunkMetadataList) { IChunkMetadata timeChunkMetadata = alignedChunkMetadata.getTimeChunkMetadata(); List valueChunkMetadataList = alignedChunkMetadata.getValueChunkMetadataList(); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/FastInnerCompactionPerformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/FastInnerCompactionPerformerTest.java index f2537e9bfddf..8c9088f5cebc 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/FastInnerCompactionPerformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/FastInnerCompactionPerformerTest.java @@ -39,7 +39,7 @@ import org.apache.tsfile.common.conf.TSFileDescriptor; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.exception.write.WriteProcessException; -import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; +import org.apache.tsfile.file.metadata.AlignedChunkMetadata; import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; import org.apache.tsfile.file.metadata.IDeviceID; @@ -2047,10 +2047,10 @@ public void testMergeAlignedSeriesTimeValuePairFromDifferentFiles() TsFileResource targetResource = tsFileManager.getTsFileList(false).get(0); try (TsFileSequenceReader reader = new TsFileSequenceReader(targetResource.getTsFilePath())) { - List chunkMetadataList = + List chunkMetadataList = reader.getAlignedChunkMetadata( IDeviceID.Factory.DEFAULT_FACTORY.create("root.testsg.d1"), true); - for (AbstractAlignedChunkMetadata alignedChunkMetadata : chunkMetadataList) { + for (AlignedChunkMetadata alignedChunkMetadata : chunkMetadataList) { ChunkMetadata timeChunkMetadata = (ChunkMetadata) alignedChunkMetadata.getTimeChunkMetadata(); Chunk timeChunk = reader.readMemChunk(timeChunkMetadata); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairUnsortedFileCompactionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairUnsortedFileCompactionTest.java index f2e39d8f8668..a9c559a00a52 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairUnsortedFileCompactionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairUnsortedFileCompactionTest.java @@ -46,7 +46,7 @@ import org.apache.iotdb.db.storageengine.dataregion.utils.TsFileResourceUtils; import org.apache.tsfile.exception.write.WriteProcessException; -import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; +import org.apache.tsfile.file.metadata.AlignedChunkMetadata; import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; import org.apache.tsfile.file.metadata.IDeviceID; @@ -682,10 +682,10 @@ public void testMergeAlignedSeriesPointWithSameTimestamp() throws IOException { Assert.assertTrue(task.start()); TsFileResource target = tsFileManager.getTsFileList(false).get(0); try (TsFileSequenceReader reader = new TsFileSequenceReader(target.getTsFilePath())) { - List chunkMetadataList = + List chunkMetadataList = reader.getAlignedChunkMetadata( IDeviceID.Factory.DEFAULT_FACTORY.create("root.testsg.d1"), true); - for (AbstractAlignedChunkMetadata alignedChunkMetadata : chunkMetadataList) { + for (AlignedChunkMetadata alignedChunkMetadata : chunkMetadataList) { ChunkMetadata timeChunkMetadata = (ChunkMetadata) alignedChunkMetadata.getTimeChunkMetadata(); Chunk timeChunk = reader.readMemChunk(timeChunkMetadata); @@ -796,7 +796,7 @@ public void testSplitChunk() throws IOException { Assert.assertTrue(task.start()); TsFileResource target = tsFileManager.getTsFileList(false).get(0); try (TsFileSequenceReader reader = new TsFileSequenceReader(target.getTsFilePath())) { - List chunkMetadataList = + List chunkMetadataList = reader.getAlignedChunkMetadata( IDeviceID.Factory.DEFAULT_FACTORY.create("root.testsg.d1"), true); Assert.assertEquals(3, chunkMetadataList.size()); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/CompactionTableModelTestFileWriter.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/CompactionTableModelTestFileWriter.java index 9bc597083739..0e5fa29a41af 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/CompactionTableModelTestFileWriter.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/CompactionTableModelTestFileWriter.java @@ -54,7 +54,7 @@ public void registerTableSchema(String tableName, List idColumnNames) { for (String idColumnName : idColumnNames) { measurementSchemas.add( new MeasurementSchema( - idColumnName, TSDataType.STRING, TSEncoding.PLAIN, CompressionType.UNCOMPRESSED)); + idColumnName, TSDataType.TEXT, TSEncoding.PLAIN, CompressionType.UNCOMPRESSED)); columnTypes.add(ColumnCategory.TAG); } tableSchema.merge(new TableSchema(tableName, measurementSchemas, columnTypes)); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/BatchCompactionUtilsTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/BatchCompactionUtilsTest.java index c257ee4ec203..ef804437361b 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/BatchCompactionUtilsTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/BatchCompactionUtilsTest.java @@ -34,7 +34,6 @@ import org.apache.tsfile.exception.write.PageException; import org.apache.tsfile.exception.write.WriteProcessException; import org.apache.tsfile.file.header.PageHeader; -import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; import org.apache.tsfile.file.metadata.AlignedChunkMetadata; import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; @@ -108,7 +107,7 @@ public void testBatchCompactionPointReader() throws IOException { true); try (TsFileSequenceReader reader = new TsFileSequenceReader(seqResource1.getTsFile().getAbsolutePath())) { - AbstractAlignedChunkMetadata alignedChunkMetadata = + AlignedChunkMetadata alignedChunkMetadata = reader .getAlignedChunkMetadata( IDeviceID.Factory.DEFAULT_FACTORY.create("root.testsg.d0"), true) @@ -312,14 +311,14 @@ public void testMapAlignedChunkMetadata1() { new MeasurementSchema("s1", TSDataType.INT32), new MeasurementSchema("s2", TSDataType.INT32), new MeasurementSchema("s4", TSDataType.INT32)); - AbstractAlignedChunkMetadata newAlignedChunkMetadata = + AlignedChunkMetadata newAlignedChunkMetadata = AlignedSeriesBatchCompactionUtils.fillAlignedChunkMetadataBySchemaList( alignedChunkMetadata, measurementSchemas); Assert.assertEquals( - Arrays.asList("s0", "s1", "s2", "s4"), newAlignedChunkMetadata.getValueChunkMetadataList().stream() .map(IChunkMetadata::getMeasurementUid) - .collect(Collectors.toList())); + .collect(Collectors.toList()), + Arrays.asList("s0", "s1", "s2", "s4")); } @Test @@ -334,14 +333,14 @@ public void testMapAlignedChunkMetadata2() { Arrays.asList( new MeasurementSchema("s0", TSDataType.INT32), new MeasurementSchema("s4", TSDataType.INT32)); - AbstractAlignedChunkMetadata newAlignedChunkMetadata = + AlignedChunkMetadata newAlignedChunkMetadata = AlignedSeriesBatchCompactionUtils.fillAlignedChunkMetadataBySchemaList( alignedChunkMetadata, measurementSchemas); Assert.assertEquals( - Arrays.asList(null, "s4"), newAlignedChunkMetadata.getValueChunkMetadataList().stream() .map(chunkMetadata -> chunkMetadata == null ? null : chunkMetadata.getMeasurementUid()) - .collect(Collectors.toList())); + .collect(Collectors.toList()), + Arrays.asList(null, "s4")); } @Test @@ -369,23 +368,23 @@ public void testMapAlignedChunkMetadata3() { new MeasurementSchema("s2", TSDataType.INT32), new MeasurementSchema("s3", TSDataType.INT32), new MeasurementSchema("s4", TSDataType.INT32)); - AbstractAlignedChunkMetadata newAlignedChunkMetadata1 = + AlignedChunkMetadata newAlignedChunkMetadata1 = AlignedSeriesBatchCompactionUtils.fillAlignedChunkMetadataBySchemaList( alignedChunkMetadata1, measurementSchemas); Assert.assertEquals( - Arrays.asList("s0", "s1", "s2", null, null), newAlignedChunkMetadata1.getValueChunkMetadataList().stream() .map(chunkMetadata -> chunkMetadata == null ? null : chunkMetadata.getMeasurementUid()) - .collect(Collectors.toList())); + .collect(Collectors.toList()), + Arrays.asList("s0", "s1", "s2", null, null)); - AbstractAlignedChunkMetadata newAlignedChunkMetadata2 = + AlignedChunkMetadata newAlignedChunkMetadata2 = AlignedSeriesBatchCompactionUtils.fillAlignedChunkMetadataBySchemaList( alignedChunkMetadata2, measurementSchemas); Assert.assertEquals( - Arrays.asList(null, null, null, "s3", "s4"), newAlignedChunkMetadata2.getValueChunkMetadataList().stream() .map(chunkMetadata -> chunkMetadata == null ? null : chunkMetadata.getMeasurementUid()) - .collect(Collectors.toList())); + .collect(Collectors.toList()), + Arrays.asList(null, null, null, "s3", "s4")); } @Test @@ -412,22 +411,22 @@ public void testMapAlignedChunkMetadata4() { Arrays.asList( new MeasurementSchema("s2", TSDataType.INT32), new MeasurementSchema("s4", TSDataType.INT32)); - AbstractAlignedChunkMetadata newAlignedChunkMetadata1 = + AlignedChunkMetadata newAlignedChunkMetadata1 = AlignedSeriesBatchCompactionUtils.fillAlignedChunkMetadataBySchemaList( alignedChunkMetadata1, measurementSchemas); Assert.assertEquals( - Arrays.asList("s2", null), newAlignedChunkMetadata1.getValueChunkMetadataList().stream() .map(chunkMetadata -> chunkMetadata == null ? null : chunkMetadata.getMeasurementUid()) - .collect(Collectors.toList())); + .collect(Collectors.toList()), + Arrays.asList("s2", null)); - AbstractAlignedChunkMetadata newAlignedChunkMetadata2 = + AlignedChunkMetadata newAlignedChunkMetadata2 = AlignedSeriesBatchCompactionUtils.fillAlignedChunkMetadataBySchemaList( alignedChunkMetadata2, measurementSchemas); Assert.assertEquals( - Arrays.asList(null, "s4"), newAlignedChunkMetadata2.getValueChunkMetadataList().stream() .map(chunkMetadata -> chunkMetadata == null ? null : chunkMetadata.getMeasurementUid()) - .collect(Collectors.toList())); + .collect(Collectors.toList()), + Arrays.asList(null, "s4")); } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/TimestampPrecisionUtilsTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/TimestampPrecisionUtilsTest.java index 7c7893e49af8..87585cc663f9 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/TimestampPrecisionUtilsTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/TimestampPrecisionUtilsTest.java @@ -45,11 +45,6 @@ public void testCheckIllegalMsTimestampPrecision() { fail(); } catch (Exception ignored) { } - try { - TimestampPrecisionUtils.checkTimestampPrecision(-1694689856546000L); - fail(); - } catch (Exception ignored) { - } } @Test @@ -73,11 +68,6 @@ public void testCheckIllegalUsTimestampPrecision() { fail(); } catch (Exception ignored) { } - try { - TimestampPrecisionUtils.checkTimestampPrecision(-1694689856546000000L); - fail(); - } catch (Exception ignored) { - } } @Test diff --git a/iotdb-core/metrics/core/pom.xml b/iotdb-core/metrics/core/pom.xml index 9c763dd8aa3a..f68d953adb56 100644 --- a/iotdb-core/metrics/core/pom.xml +++ b/iotdb-core/metrics/core/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-metrics - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT metrics-core IoTDB: Core: Metrics: API Impl @@ -32,7 +32,7 @@ org.apache.iotdb metrics-interface - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT io.micrometer diff --git a/iotdb-core/metrics/interface/pom.xml b/iotdb-core/metrics/interface/pom.xml index 8708e74d4b44..695fbc879d15 100644 --- a/iotdb-core/metrics/interface/pom.xml +++ b/iotdb-core/metrics/interface/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-metrics - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT metrics-interface IoTDB: Core: Metrics: Metrics API @@ -33,17 +33,17 @@ org.apache.iotdb iotdb-session - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.apache.iotdb isession - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.apache.iotdb service-rpc - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.apache.tsfile diff --git a/iotdb-core/metrics/pom.xml b/iotdb-core/metrics/pom.xml index 7ea896c0f2d4..2b62015bb9fd 100644 --- a/iotdb-core/metrics/pom.xml +++ b/iotdb-core/metrics/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-core - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT iotdb-metrics pom diff --git a/iotdb-core/node-commons/pom.xml b/iotdb-core/node-commons/pom.xml index ccc104f539a1..8700c4de34b6 100644 --- a/iotdb-core/node-commons/pom.xml +++ b/iotdb-core/node-commons/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-core - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT node-commons IoTDB: Core: Node Commons @@ -38,7 +38,7 @@ org.apache.iotdb service-rpc - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.apache.tsfile @@ -48,37 +48,37 @@ org.apache.iotdb udf-api - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.apache.iotdb trigger-api - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.apache.iotdb pipe-api - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.apache.iotdb iotdb-thrift-confignode - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.apache.iotdb iotdb-thrift - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.apache.iotdb iotdb-thrift-commons - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.apache.iotdb iotdb-thrift-consensus - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.apache.tsfile @@ -88,12 +88,12 @@ org.apache.iotdb metrics-interface - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.apache.iotdb metrics-core - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT io.dropwizard.metrics @@ -193,7 +193,7 @@ org.apache.iotdb iotdb-thrift-ainode - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT compile diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template index 4f13c3b8e20b..20a9677ce443 100644 --- a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template +++ b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template @@ -1730,10 +1730,10 @@ continuous_query_min_every_interval_in_ms=1000 pipe_lib_dir=ext/pipe # The maximum number of threads that can be used to execute the pipe subtasks in PipeSubtaskExecutor. -# When <= 0, use max(5, CPU core number). +# The actual value will be min(pipe_subtask_executor_max_thread_num, max(1, CPU core number / 2)). # effectiveMode: restart # Datatype: int -pipe_subtask_executor_max_thread_num=0 +pipe_subtask_executor_max_thread_num=5 # The connection timeout (in milliseconds) for the thrift client. # effectiveMode: restart @@ -1742,16 +1742,14 @@ pipe_sink_timeout_ms=900000 # The maximum number of selectors that can be used in the sink. # Recommend to set this value to less than or equal to pipe_sink_max_client_number. -# When <= 0, use max(4, CPU core number). # effectiveMode: restart # Datatype: int -pipe_sink_selector_number=0 +pipe_sink_selector_number=4 # The maximum number of clients that can be used in the sink. -# When <= 0, use max(16, CPU core number). # effectiveMode: restart # Datatype: int -pipe_sink_max_client_number=0 +pipe_sink_max_client_number=16 # Whether to enable receiving pipe data through air gap. # The receiver can only return 0 or 1 in tcp mode to indicate whether the data is received successfully. diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/authorizer/BasicAuthorizer.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/authorizer/BasicAuthorizer.java index d4fe42d8055d..1867a207d088 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/authorizer/BasicAuthorizer.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/authorizer/BasicAuthorizer.java @@ -158,17 +158,6 @@ public void revokePrivilegeFromUser(String username, PrivilegeUnion union) throw userManager.revokePrivilegeFromEntity(username, union); } - @Override - public void revokeAllPrivilegeFromUser(String userName) throws AuthException { - checkAdmin(userName, "Invalid operation, administrator cannot revoke privileges"); - User user = userManager.getEntity(userName); - if (user == null) { - throw new AuthException( - TSStatusCode.USER_NOT_EXIST, String.format("User %s does not exist", userName)); - } - user.revokeAllRelationalPrivileges(); - } - @Override public void createRole(String roleName) throws AuthException { AuthUtils.validateRolename(roleName); @@ -212,16 +201,6 @@ public void revokePrivilegeFromRole(String roleName, PrivilegeUnion union) throw roleManager.revokePrivilegeFromEntity(roleName, union); } - @Override - public void revokeAllPrivilegeFromRole(String roleName) throws AuthException { - Role role = roleManager.getEntity(roleName); - if (role == null) { - throw new AuthException( - TSStatusCode.ROLE_NOT_EXIST, String.format("Role %s does not exist", roleName)); - } - role.revokeAllRelationalPrivileges(); - } - @Override public void grantRoleToUser(String roleName, String userName) throws AuthException { checkAdmin(userName, "Invalid operation, cannot grant role to administrator"); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/authorizer/IAuthorizer.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/authorizer/IAuthorizer.java index dbe9bf8550b5..d7f8cd26770b 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/authorizer/IAuthorizer.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/authorizer/IAuthorizer.java @@ -81,8 +81,6 @@ public interface IAuthorizer extends SnapshotProcessor { */ void revokePrivilegeFromUser(String userName, PrivilegeUnion union) throws AuthException; - void revokeAllPrivilegeFromUser(String userName) throws AuthException; - /** * Create a role. * @@ -117,8 +115,6 @@ public interface IAuthorizer extends SnapshotProcessor { */ void revokePrivilegeFromRole(String roleName, PrivilegeUnion union) throws AuthException; - void revokeAllPrivilegeFromRole(String roleName) throws AuthException; - /** * Grant a role to a user. * diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/entity/Role.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/entity/Role.java index 4e5a026f4ada..32ecdd9437af 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/entity/Role.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/auth/entity/Role.java @@ -365,14 +365,6 @@ public void revokeAnyScopePrivilegeGrantOption(PrivilegeType priv) { anyScopePrivilegeGrantOptSet.remove(priv); } - public void revokeAllRelationalPrivileges() { - objectPrivilegeMap = new HashMap<>(); - sysPrivilegeSet.removeIf(PrivilegeType::forRelationalSys); - sysPriGrantOpt.removeIf(PrivilegeType::forRelationalSys); - anyScopePrivilegeSet = new HashSet<>(); - anyScopePrivilegeGrantOptSet = new HashSet<>(); - } - public void grantDBPrivilege(String dbName, PrivilegeType priv, boolean grantOption) { DatabasePrivilege databasePrivilege = getObjectPrivilege(dbName); databasePrivilege.grantDBPrivilege(priv); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java index 566a37e82ebc..b2eeaf25f2cb 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java @@ -207,7 +207,7 @@ public class CommonConfig { /** The maximum number of threads that can be used to execute subtasks in PipeSubtaskExecutor. */ private int pipeSubtaskExecutorMaxThreadNum = - Math.max(5, Runtime.getRuntime().availableProcessors()); + Math.min(5, Math.max(1, Runtime.getRuntime().availableProcessors() / 2)); private int pipeNonForwardingEventsProgressReportInterval = 100; @@ -232,10 +232,8 @@ public class CommonConfig { private long pipeConnectorRetryIntervalMs = 1000L; private boolean pipeConnectorRPCThriftCompressionEnabled = false; - private int pipeAsyncConnectorSelectorNumber = - Math.max(4, Runtime.getRuntime().availableProcessors()); - private int pipeAsyncConnectorMaxClientNumber = - Math.max(16, Runtime.getRuntime().availableProcessors()); + private int pipeAsyncConnectorSelectorNumber = 4; + private int pipeAsyncConnectorMaxClientNumber = 16; private double pipeAllSinksRateLimitBytesPerSecond = -1; private int rateLimiterHotReloadCheckIntervalMs = 1000; @@ -261,7 +259,7 @@ public class CommonConfig { private long pipeMaxAllowedLinkedTsFileCount = 100; private float pipeMaxAllowedLinkedDeletedTsFileDiskUsagePercentage = 0.1F; private long pipeStuckRestartIntervalSeconds = 120; - private long pipeStuckRestartMinIntervalMs = 5 * 60 * 1000L; // 5 minutes + private long pipeStuckRestartMinIntervalMs = 30 * 60 * 1000L; // 30 minutes private int pipeMetaReportMaxLogNumPerRound = 10; private int pipeMetaReportMaxLogIntervalRounds = 36; @@ -942,7 +940,10 @@ public int getPipeSubtaskExecutorMaxThreadNum() { } public void setPipeSubtaskExecutorMaxThreadNum(int pipeSubtaskExecutorMaxThreadNum) { - this.pipeSubtaskExecutorMaxThreadNum = pipeSubtaskExecutorMaxThreadNum; + this.pipeSubtaskExecutorMaxThreadNum = + Math.min( + pipeSubtaskExecutorMaxThreadNum, + Math.max(1, Runtime.getRuntime().availableProcessors() / 2)); } public long getPipeSubtaskExecutorPendingQueueMaxBlockingTimeMs() { diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonDescriptor.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonDescriptor.java index 3661feefa6ee..529d1053c422 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonDescriptor.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonDescriptor.java @@ -306,13 +306,13 @@ private void loadPipeProps(TrimProperties properties) { "pipe_realtime_queue_poll_history_threshold", Integer.toString(config.getPipeRealTimeQueuePollHistoryThreshold())))); - int pipeSubtaskExecutorMaxThreadNum = + config.setPipeSubtaskExecutorMaxThreadNum( Integer.parseInt( properties.getProperty( "pipe_subtask_executor_max_thread_num", - Integer.toString(config.getPipeSubtaskExecutorMaxThreadNum()))); - if (pipeSubtaskExecutorMaxThreadNum > 0) { - config.setPipeSubtaskExecutorMaxThreadNum(pipeSubtaskExecutorMaxThreadNum); + Integer.toString(config.getPipeSubtaskExecutorMaxThreadNum())))); + if (config.getPipeSubtaskExecutorMaxThreadNum() <= 0) { + config.setPipeSubtaskExecutorMaxThreadNum(5); } config.setPipeSubtaskExecutorBasicCheckPointIntervalByConsumedEventCount( Integer.parseInt( @@ -405,26 +405,22 @@ private void loadPipeProps(TrimProperties properties) { properties.getProperty( "pipe_connector_rpc_thrift_compression_enabled", String.valueOf(config.isPipeConnectorRPCThriftCompressionEnabled()))))); - int pipeAsyncConnectorSelectorNumber = + + config.setPipeAsyncConnectorSelectorNumber( Integer.parseInt( Optional.ofNullable(properties.getProperty("pipe_sink_selector_number")) .orElse( properties.getProperty( "pipe_async_connector_selector_number", - String.valueOf(config.getPipeAsyncConnectorSelectorNumber())))); - if (pipeAsyncConnectorSelectorNumber > 0) { - config.setPipeAsyncConnectorSelectorNumber(pipeAsyncConnectorSelectorNumber); - } - int pipeAsyncConnectorMaxClientNumber = + String.valueOf(config.getPipeAsyncConnectorSelectorNumber()))))); + config.setPipeAsyncConnectorMaxClientNumber( Integer.parseInt( Optional.ofNullable(properties.getProperty("pipe_sink_max_client_number")) .orElse( properties.getProperty( "pipe_async_connector_max_client_number", - String.valueOf(config.getPipeAsyncConnectorMaxClientNumber())))); - if (pipeAsyncConnectorMaxClientNumber > 0) { - config.setPipeAsyncConnectorMaxClientNumber(pipeAsyncConnectorMaxClientNumber); - } + String.valueOf(config.getPipeAsyncConnectorMaxClientNumber()))))); + config.setPipeAllSinksRateLimitBytesPerSecond( Double.parseDouble( properties.getProperty( diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/connection/BlockingPendingQueue.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/connection/BlockingPendingQueue.java index 82518bda14d9..83ad7f04727f 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/connection/BlockingPendingQueue.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/connection/BlockingPendingQueue.java @@ -29,7 +29,6 @@ import java.util.concurrent.BlockingQueue; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Consumer; public abstract class BlockingPendingQueue { @@ -43,8 +42,6 @@ public abstract class BlockingPendingQueue { protected final PipeEventCounter eventCounter; - protected final AtomicBoolean isClosed = new AtomicBoolean(false); - protected BlockingPendingQueue( final BlockingQueue pendingQueue, final PipeEventCounter eventCounter) { this.pendingQueue = pendingQueue; @@ -52,7 +49,6 @@ protected BlockingPendingQueue( } public boolean waitedOffer(final E event) { - checkBeforeOffer(event); try { final boolean offered = pendingQueue.offer(event, MAX_BLOCKING_TIME_MS, TimeUnit.MILLISECONDS); @@ -68,7 +64,6 @@ public boolean waitedOffer(final E event) { } public boolean directOffer(final E event) { - checkBeforeOffer(event); final boolean offered = pendingQueue.offer(event); if (offered) { eventCounter.increaseEventCount(event); @@ -77,7 +72,6 @@ public boolean directOffer(final E event) { } public boolean put(final E event) { - checkBeforeOffer(event); try { pendingQueue.put(event); eventCounter.increaseEventCount(event); @@ -108,7 +102,6 @@ public E waitedPoll() { } public void clear() { - isClosed.set(true); pendingQueue.clear(); eventCounter.reset(); } @@ -119,7 +112,6 @@ public void forEach(final Consumer action) { } public void discardAllEvents() { - isClosed.set(true); pendingQueue.removeIf( event -> { if (event instanceof EnrichedEvent) { @@ -166,10 +158,4 @@ public int getTsFileInsertionEventCount() { public int getPipeHeartbeatEventCount() { return eventCounter.getPipeHeartbeatEventCount(); } - - protected void checkBeforeOffer(final E event) { - if (isClosed.get() && event instanceof EnrichedEvent) { - ((EnrichedEvent) event).clearReferenceCount(BlockingPendingQueue.class.getName()); - } - } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/options/PipeInclusionOptions.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/options/PipeInclusionOptions.java index 61478ed88b16..a4a91f504a0a 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/options/PipeInclusionOptions.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/options/PipeInclusionOptions.java @@ -30,7 +30,6 @@ import java.util.Collections; import java.util.HashMap; import java.util.HashSet; -import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.atomic.AtomicReference; @@ -46,16 +45,10 @@ public class PipeInclusionOptions { private static final Logger LOGGER = LoggerFactory.getLogger(PipeInclusionOptions.class); - public static final List treeOnlySyncPrefixes = - Collections.singletonList(new PartialPath(new String[] {"schema", "timeseries"})); - public static final List tableOnlySyncPrefixes = - Collections.unmodifiableList( - Arrays.asList( - new PartialPath(new String[] {"schema", "table"}), - new PartialPath(new String[] {"auth", "role", "grant", "table"}), - new PartialPath(new String[] {"auth", "role", "revoke", "table"}), - new PartialPath(new String[] {"auth", "user", "grant", "table"}), - new PartialPath(new String[] {"auth", "user", "revoke", "table"}))); + public static final PartialPath treeOnlySyncPrefix = + new PartialPath(new String[] {"schema", "timeseries"}); + public static final PartialPath tableOnlySyncPrefix = + new PartialPath(new String[] {"schema", "table"}); public static final String ALL = "all"; private static final Set OPTIONS = new HashSet<>(); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/column/ColumnHeaderConstant.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/column/ColumnHeaderConstant.java index 7b7d01ca55f4..2bafd0e5678e 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/column/ColumnHeaderConstant.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/column/ColumnHeaderConstant.java @@ -213,36 +213,6 @@ private ColumnHeaderConstant() { public static final String SCHEMA_REGION_GROUP_NUM_TABLE_MODEL = "schema_region_group_num"; public static final String DATA_REGION_GROUP_NUM_TABLE_MODEL = "data_region_group_num"; - public static final String REGION_ID_TABLE_MODEL = "region_id"; - public static final String DATANODE_ID_TABLE_MODEL = "datanode_id"; - public static final String SERIES_SLOT_NUM_TABLE_MODEL = "series_slot_num"; - public static final String TIME_SLOT_NUM_TABLE_MODEL = "time_slot_num"; - public static final String RPC_ADDRESS_TABLE_MODEL = "rpc_address"; - public static final String RPC_PORT_TABLE_MODEL = "rpc_port"; - public static final String INTERNAL_ADDRESS_TABLE_MODEL = "internal_address"; - public static final String CREATE_TIME_TABLE_MODEL = "create_time"; - public static final String TS_FILE_SIZE_BYTES_TABLE_MODEL = "tsfile_size_bytes"; - - public static final String CREATION_TIME_TABLE_MODEL = "creation_time"; - public static final String PIPE_SOURCE_TABLE_MODEL = "pipe_source"; - public static final String PIPE_PROCESSOR_TABLE_MODEL = "pipe_processor"; - public static final String PIPE_SINK_TABLE_MODEL = "pipe_sink"; - public static final String EXCEPTION_MESSAGE_TABLE_MODEL = "exception_message"; - public static final String REMAINING_EVENT_COUNT_TABLE_MODEL = "remaining_event_count"; - public static final String ESTIMATED_REMAINING_SECONDS_TABLE_MODEL = - "estimated_remaining_seconds"; - - public static final String PLUGIN_NAME_TABLE_MODEL = "plugin_name"; - public static final String PLUGIN_TYPE_TABLE_MODEL = "plugin_type"; - public static final String CLASS_NAME_TABLE_MODEL = "class_name"; - public static final String PLUGIN_JAR_TABLE_MODEL = "plugin_jar"; - - public static final String TOPIC_NAME_TABLE_MODEL = "topic_name"; - public static final String TOPIC_CONFIGS_TABLE_MODEL = "topic_configs"; - - public static final String CONSUMER_GROUP_NAME_TABLE_MODEL = "consumer_group_name"; - public static final String SUBSCRIBED_CONSUMERS_TABLE_MODEL = "subscribed_consumers"; - // column names for show space quota public static final String QUOTA_TYPE = "QuotaType"; public static final String LIMIT = "Limit"; diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/InformationSchema.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/InformationSchema.java index 6fdf456ab76a..f2f4e548d4ce 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/InformationSchema.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/InformationSchema.java @@ -37,11 +37,6 @@ public class InformationSchema { public static final String DATABASES = "databases"; public static final String TABLES = "tables"; public static final String COLUMNS = "columns"; - public static final String REGIONS = "regions"; - public static final String PIPES = "pipes"; - public static final String PIPE_PLUGINS = "pipe_plugins"; - public static final String TOPICS = "topics"; - public static final String SUBSCRIPTIONS = "subscriptions"; static { final TsTable queriesTable = new TsTable(QUERIES); @@ -57,9 +52,6 @@ public class InformationSchema { queriesTable.addColumnSchema( new AttributeColumnSchema( ColumnHeaderConstant.STATEMENT.toLowerCase(Locale.ENGLISH), TSDataType.STRING)); - queriesTable.addColumnSchema( - new AttributeColumnSchema( - ColumnHeaderConstant.USER.toLowerCase(Locale.ENGLISH), TSDataType.STRING)); queriesTable.removeColumnSchema(TsTable.TIME_COLUMN_NAME); schemaTables.put(QUERIES, queriesTable); @@ -122,107 +114,6 @@ public class InformationSchema { ColumnHeaderConstant.STATUS.toLowerCase(Locale.ENGLISH), TSDataType.STRING)); columnTable.removeColumnSchema(TsTable.TIME_COLUMN_NAME); schemaTables.put(COLUMNS, columnTable); - - final TsTable regionTable = new TsTable(REGIONS); - regionTable.addColumnSchema( - new TagColumnSchema(ColumnHeaderConstant.REGION_ID_TABLE_MODEL, TSDataType.INT32)); - regionTable.addColumnSchema( - new TagColumnSchema(ColumnHeaderConstant.DATANODE_ID_TABLE_MODEL, TSDataType.INT32)); - regionTable.addColumnSchema( - new AttributeColumnSchema( - ColumnHeaderConstant.TYPE.toLowerCase(Locale.ENGLISH), TSDataType.STRING)); - regionTable.addColumnSchema( - new AttributeColumnSchema( - ColumnHeaderConstant.STATUS.toLowerCase(Locale.ENGLISH), TSDataType.STRING)); - regionTable.addColumnSchema( - new AttributeColumnSchema( - ColumnHeaderConstant.DATABASE.toLowerCase(Locale.ENGLISH), TSDataType.STRING)); - regionTable.addColumnSchema( - new AttributeColumnSchema( - ColumnHeaderConstant.SERIES_SLOT_NUM_TABLE_MODEL, TSDataType.INT32)); - regionTable.addColumnSchema( - new AttributeColumnSchema( - ColumnHeaderConstant.TIME_SLOT_NUM_TABLE_MODEL, TSDataType.INT64)); - regionTable.addColumnSchema( - new AttributeColumnSchema(ColumnHeaderConstant.RPC_ADDRESS_TABLE_MODEL, TSDataType.STRING)); - regionTable.addColumnSchema( - new AttributeColumnSchema(ColumnHeaderConstant.RPC_PORT_TABLE_MODEL, TSDataType.INT32)); - regionTable.addColumnSchema( - new AttributeColumnSchema( - ColumnHeaderConstant.INTERNAL_ADDRESS_TABLE_MODEL, TSDataType.STRING)); - regionTable.addColumnSchema( - new AttributeColumnSchema( - ColumnHeaderConstant.ROLE.toLowerCase(Locale.ENGLISH), TSDataType.STRING)); - regionTable.addColumnSchema( - new AttributeColumnSchema( - ColumnHeaderConstant.CREATE_TIME_TABLE_MODEL, TSDataType.TIMESTAMP)); - regionTable.addColumnSchema( - new AttributeColumnSchema( - ColumnHeaderConstant.TS_FILE_SIZE_BYTES_TABLE_MODEL, TSDataType.INT64)); - regionTable.removeColumnSchema(TsTable.TIME_COLUMN_NAME); - schemaTables.put(REGIONS, regionTable); - - final TsTable pipeTable = new TsTable(PIPES); - pipeTable.addColumnSchema( - new TagColumnSchema( - ColumnHeaderConstant.ID.toLowerCase(Locale.ENGLISH), TSDataType.STRING)); - pipeTable.addColumnSchema( - new AttributeColumnSchema( - ColumnHeaderConstant.CREATION_TIME_TABLE_MODEL, TSDataType.TIMESTAMP)); - pipeTable.addColumnSchema( - new AttributeColumnSchema( - ColumnHeaderConstant.STATE.toLowerCase(Locale.ENGLISH), TSDataType.STRING)); - pipeTable.addColumnSchema( - new AttributeColumnSchema(ColumnHeaderConstant.PIPE_SOURCE_TABLE_MODEL, TSDataType.STRING)); - pipeTable.addColumnSchema( - new AttributeColumnSchema( - ColumnHeaderConstant.PIPE_PROCESSOR_TABLE_MODEL, TSDataType.STRING)); - pipeTable.addColumnSchema( - new AttributeColumnSchema(ColumnHeaderConstant.PIPE_SINK_TABLE_MODEL, TSDataType.STRING)); - pipeTable.addColumnSchema( - new AttributeColumnSchema( - ColumnHeaderConstant.EXCEPTION_MESSAGE_TABLE_MODEL, TSDataType.STRING)); - pipeTable.addColumnSchema( - new AttributeColumnSchema( - ColumnHeaderConstant.REMAINING_EVENT_COUNT_TABLE_MODEL, TSDataType.INT64)); - pipeTable.addColumnSchema( - new AttributeColumnSchema( - ColumnHeaderConstant.ESTIMATED_REMAINING_SECONDS_TABLE_MODEL, TSDataType.DOUBLE)); - pipeTable.removeColumnSchema(TsTable.TIME_COLUMN_NAME); - schemaTables.put(PIPES, pipeTable); - - final TsTable pipePluginTable = new TsTable(PIPE_PLUGINS); - pipePluginTable.addColumnSchema( - new TagColumnSchema(ColumnHeaderConstant.PLUGIN_NAME_TABLE_MODEL, TSDataType.STRING)); - pipePluginTable.addColumnSchema( - new AttributeColumnSchema(ColumnHeaderConstant.PLUGIN_TYPE_TABLE_MODEL, TSDataType.STRING)); - pipePluginTable.addColumnSchema( - new AttributeColumnSchema(ColumnHeaderConstant.CLASS_NAME_TABLE_MODEL, TSDataType.STRING)); - pipePluginTable.addColumnSchema( - new AttributeColumnSchema(ColumnHeaderConstant.PLUGIN_JAR_TABLE_MODEL, TSDataType.STRING)); - pipePluginTable.removeColumnSchema(TsTable.TIME_COLUMN_NAME); - schemaTables.put(PIPE_PLUGINS, pipePluginTable); - - final TsTable topicTable = new TsTable(TOPICS); - topicTable.addColumnSchema( - new TagColumnSchema(ColumnHeaderConstant.TOPIC_NAME_TABLE_MODEL, TSDataType.STRING)); - topicTable.addColumnSchema( - new AttributeColumnSchema( - ColumnHeaderConstant.TOPIC_CONFIGS_TABLE_MODEL, TSDataType.STRING)); - topicTable.removeColumnSchema(TsTable.TIME_COLUMN_NAME); - schemaTables.put(TOPICS, topicTable); - - final TsTable subscriptionTable = new TsTable(SUBSCRIPTIONS); - subscriptionTable.addColumnSchema( - new TagColumnSchema(ColumnHeaderConstant.TOPIC_NAME_TABLE_MODEL, TSDataType.STRING)); - subscriptionTable.addColumnSchema( - new TagColumnSchema( - ColumnHeaderConstant.CONSUMER_GROUP_NAME_TABLE_MODEL, TSDataType.STRING)); - subscriptionTable.addColumnSchema( - new AttributeColumnSchema( - ColumnHeaderConstant.SUBSCRIBED_CONSUMERS_TABLE_MODEL, TSDataType.STRING)); - subscriptionTable.removeColumnSchema(TsTable.TIME_COLUMN_NAME); - schemaTables.put(SUBSCRIPTIONS, subscriptionTable); } public static Map getSchemaTables() { diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java index 65ecf6280b55..906c9f05968e 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java @@ -98,7 +98,6 @@ public enum Metric { MEM("mem"), CACHE("cache"), CACHE_HIT_RATE("cache_hit"), - CACHE_ENTRIES_NUM("cache_entries_num"), QUANTITY("quantity"), LEADER_QUANTITY("leader_quantity"), SCHEMA_REGION("schema_region"), diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/TableBuiltinScalarFunction.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/TableBuiltinScalarFunction.java index 6d5b1ef4721a..ee9b3d043b42 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/TableBuiltinScalarFunction.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/TableBuiltinScalarFunction.java @@ -65,8 +65,6 @@ public enum TableBuiltinScalarFunction { E("e"), DATE_BIN("date_bin"), FORMAT("format"), - GREATEST("greatest"), - LEAST("least"), ; private final String functionName; diff --git a/iotdb-core/pom.xml b/iotdb-core/pom.xml index 71e5457cde61..77bec8db8895 100644 --- a/iotdb-core/pom.xml +++ b/iotdb-core/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-parent - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT iotdb-core pom diff --git a/iotdb-core/relational-grammar/pom.xml b/iotdb-core/relational-grammar/pom.xml index df149c0441e9..a27124cb36d9 100644 --- a/iotdb-core/relational-grammar/pom.xml +++ b/iotdb-core/relational-grammar/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-core - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT iotdb-relational-grammar IoTDB: Core: Relational-Antlr-Parser diff --git a/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 b/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 index 473e64713131..20180597005e 100644 --- a/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 +++ b/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 @@ -108,9 +108,6 @@ statement | countTimeSlotListStatement | showSeriesSlotListStatement | migrateRegionStatement - | reconstructRegionStatement - | extendRegionStatement - | removeRegionStatement | removeDataNodeStatement // Admin Statement @@ -126,7 +123,6 @@ statement | loadConfigurationStatement | setConfigurationStatement | showCurrentSqlDialectStatement - | setSqlDialectStatement | showCurrentUserStatement | showCurrentDatabaseStatement | showCurrentTimestampStatement @@ -483,18 +479,6 @@ migrateRegionStatement : MIGRATE REGION regionId=INTEGER_VALUE FROM fromId=INTEGER_VALUE TO toId=INTEGER_VALUE ; -reconstructRegionStatement - : RECONSTRUCT REGION regionIds+=INTEGER_VALUE (COMMA regionIds+=INTEGER_VALUE)* ON targetDataNodeId=INTEGER_VALUE - ; - -extendRegionStatement - : EXTEND REGION regionId=INTEGER_VALUE TO targetDataNodeId=INTEGER_VALUE - ; - -removeRegionStatement - : REMOVE REGION regionId=INTEGER_VALUE FROM targetDataNodeId=INTEGER_VALUE - ; - removeDataNodeStatement : REMOVE DATANODE dataNodeId=INTEGER_VALUE (',' dataNodeId=INTEGER_VALUE)* ; @@ -563,10 +547,6 @@ showCurrentSqlDialectStatement : SHOW CURRENT_SQL_DIALECT ; -setSqlDialectStatement - : SET SQL_DIALECT EQ (TABLE | TREE) - ; - showCurrentUserStatement : SHOW CURRENT_USER ; @@ -624,11 +604,11 @@ listRolePrivilegeStatement ; listUserStatement - : LIST USER (OF ROLE roleName=identifier)? + : LIST USER ; listRoleStatement - : LIST ROLE (OF USER userName=identifier)? + : LIST ROLE ; @@ -639,7 +619,7 @@ revokeStatement privilegeObjectScope : systemPrivileges | objectPrivileges ON objectType objectName=identifier - | objectPrivileges ON (TABLE)? objectScope + | objectPrivileges ON objectScope | objectPrivileges ON ANY | ALL ; @@ -650,7 +630,6 @@ systemPrivileges objectPrivileges : objectPrivilege (',' objectPrivilege)* - | ALL ; objectScope @@ -1209,7 +1188,6 @@ EXCLUDING: 'EXCLUDING'; EXECUTE: 'EXECUTE'; EXISTS: 'EXISTS'; EXPLAIN: 'EXPLAIN'; -EXTEND: 'EXTEND'; EXTRACT: 'EXTRACT'; EXTRACTOR: 'EXTRACTOR'; FALSE: 'FALSE'; @@ -1411,7 +1389,6 @@ SINK: 'SINK'; SKIP_TOKEN: 'SKIP'; SOME: 'SOME'; SOURCE: 'SOURCE'; -SQL_DIALECT: 'SQL_DIALECT'; START: 'START'; STATS: 'STATS'; STOP: 'STOP'; @@ -1439,7 +1416,6 @@ TOPIC: 'TOPIC'; TOPICS: 'TOPICS'; TRAILING: 'TRAILING'; TRANSACTION: 'TRANSACTION'; -TREE: 'TREE'; TRIM: 'TRIM'; TRUE: 'TRUE'; TRUNCATE: 'TRUNCATE'; diff --git a/iotdb-protocol/openapi/pom.xml b/iotdb-protocol/openapi/pom.xml index 27c35db316ec..c82eebcc0cf1 100644 --- a/iotdb-protocol/openapi/pom.xml +++ b/iotdb-protocol/openapi/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-protocol - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT openapi IoTDB: Protocol: OpenAPI diff --git a/iotdb-protocol/pom.xml b/iotdb-protocol/pom.xml index 84bdca026a6a..523ea7f37dae 100644 --- a/iotdb-protocol/pom.xml +++ b/iotdb-protocol/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-parent - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT iotdb-protocol pom diff --git a/iotdb-protocol/thrift-ainode/pom.xml b/iotdb-protocol/thrift-ainode/pom.xml index 4ac2efd8ce51..3ed313f9ab39 100644 --- a/iotdb-protocol/thrift-ainode/pom.xml +++ b/iotdb-protocol/thrift-ainode/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-protocol - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT iotdb-thrift-ainode IoTDB: Protocol: Thrift AI Node @@ -41,7 +41,7 @@ org.apache.iotdb iotdb-thrift-commons - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT diff --git a/iotdb-protocol/thrift-commons/pom.xml b/iotdb-protocol/thrift-commons/pom.xml index b2c14fc62b42..a9ebdba2fe22 100644 --- a/iotdb-protocol/thrift-commons/pom.xml +++ b/iotdb-protocol/thrift-commons/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-protocol - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT iotdb-thrift-commons IoTDB: Protocol: Thrift Commons diff --git a/iotdb-protocol/thrift-confignode/pom.xml b/iotdb-protocol/thrift-confignode/pom.xml index 2816a1a21a9f..7d7f00db430a 100644 --- a/iotdb-protocol/thrift-confignode/pom.xml +++ b/iotdb-protocol/thrift-confignode/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-protocol - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT iotdb-thrift-confignode IoTDB: Protocol: Thrift Config Node @@ -41,7 +41,7 @@ org.apache.iotdb iotdb-thrift-commons - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT diff --git a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift index c649566fe071..7385fafd2e08 100644 --- a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift +++ b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift @@ -310,25 +310,21 @@ struct TMigrateRegionReq { 1: required i32 regionId 2: required i32 fromId 3: required i32 toId - 4: required common.Model model } struct TReconstructRegionReq { 1: required list regionIds 2: required i32 dataNodeId - 3: required common.Model model } struct TExtendRegionReq { 1: required i32 regionId 2: required i32 dataNodeId - 3: required common.Model model } struct TRemoveRegionReq { 1: required i32 regionId 2: required i32 dataNodeId - 3: required common.Model model } // Authorize diff --git a/iotdb-protocol/thrift-consensus/pom.xml b/iotdb-protocol/thrift-consensus/pom.xml index 1297b6ba431d..6c4fcb3fc153 100644 --- a/iotdb-protocol/thrift-consensus/pom.xml +++ b/iotdb-protocol/thrift-consensus/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-protocol - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT iotdb-thrift-consensus IoTDB: Protocol: Thrift Consensus @@ -33,7 +33,7 @@ org.apache.iotdb iotdb-thrift-commons - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.slf4j diff --git a/iotdb-protocol/thrift-datanode/pom.xml b/iotdb-protocol/thrift-datanode/pom.xml index 8fae19237f8f..3204b36c956c 100644 --- a/iotdb-protocol/thrift-datanode/pom.xml +++ b/iotdb-protocol/thrift-datanode/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-protocol - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT iotdb-thrift IoTDB: Protocol: Thrift Data Node @@ -33,7 +33,7 @@ org.apache.iotdb iotdb-thrift-commons - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.apache.thrift diff --git a/library-udf/pom.xml b/library-udf/pom.xml index ac695ec9a319..48af935e19ae 100644 --- a/library-udf/pom.xml +++ b/library-udf/pom.xml @@ -24,7 +24,7 @@ org.apache.iotdb iotdb-parent - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT library-udf IoTDB: UDF @@ -41,7 +41,7 @@ org.apache.iotdb udf-api - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT org.slf4j diff --git a/pom.xml b/pom.xml index 62d393eb2044..c11760ef8788 100644 --- a/pom.xml +++ b/pom.xml @@ -28,7 +28,7 @@ org.apache.iotdb iotdb-parent - 2.0.2-SNAPSHOT + 2.0.0-SNAPSHOT pom Apache IoTDB Project Parent POM This is the top level project that builds, packages the iotdb engine, client, and integration libs. @@ -106,7 +106,7 @@ 0.11.5 3.26.2 5.14.0 - 2.5.2 + 2.5.0 3.1 4.13.2 @@ -168,7 +168,7 @@ 0.14.1 1.9 1.5.6-3 - 2.1.0-250207-SNAPSHOT + 2.0.0-250118-SNAPSHOT - ${project.parent.build.directory}/generated-sources-csharp - -