Skip to content

Commit 0642fa9

Browse files
cgivrecgivre
committed
Added Exec Config Options
1 parent d5b4004 commit 0642fa9

File tree

6 files changed

+122
-63
lines changed

6 files changed

+122
-63
lines changed

‎distribution/src/main/resources/drill-override-example.conf‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -322,6 +322,21 @@ drill.exec: {
322322
root: "/app/drill"
323323
}
324324
},
325+
326+
# Settings for Apache Daffodil schemata.
327+
daffodil: {
328+
# number of retry attempts to update remote function registry
329+
# if registry version was changed during update
330+
retry-attempts: 10,
331+
directory: {
332+
# Override this property if custom file system should be used to create remote directories
333+
# instead of default taken from Hadoop configuration
334+
fs: "hdfs:///",
335+
# Set this property if custom absolute root should be used for remote directories
336+
root: "/app/drill"
337+
}
338+
},
339+
325340
# Settings for Temporary Tables (see https://issues.apache.org/jira/browse/DRILL-4956 for details).
326341
# Temporary table can be created ONLY in default temporary workspace.
327342
# Full workspace name should be indicated (including schema and workspace separated by dot).

‎exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@
1717
*/
1818
package org.apache.drill.exec;
1919

20-
import java.util.Arrays;
2120
import org.apache.drill.common.config.DrillConfig;
2221
import org.apache.drill.exec.physical.impl.common.HashTable;
2322
import org.apache.drill.exec.rpc.user.InboundImpersonationManager;
@@ -42,6 +41,8 @@
4241
import org.apache.drill.exec.testing.ExecutionControls;
4342
import org.apache.drill.exec.vector.ValueVector;
4443

44+
import java.util.Arrays;
45+
4546
public final class ExecConstants {
4647
private ExecConstants() {
4748
// Don't allow instantiation
@@ -343,6 +344,17 @@ private ExecConstants() {
343344
public static final String UDF_DIRECTORY_TMP = "drill.exec.udf.directory.tmp";
344345
public static final String UDF_DISABLE_DYNAMIC = "drill.exec.udf.disable_dynamic";
345346

347+
/**
348+
* Configuration properties for Apache Daffodil
349+
*/
350+
public static final String DFDL_RETRY_ATTEMPTS = "drill.exec.daffodil.retry-attempts";
351+
public static final String DFDL_DIRECTORY_LOCAL = "drill.exec.daffodil.directory.local";
352+
public static final String DFDL_DIRECTORY_FS = "drill.exec.daffodil.directory.fs";
353+
public static final String DFDL_DIRECTORY_ROOT = "drill.exec.daffodil.directory.root";
354+
public static final String DFDL_DIRECTORY_STAGING = "drill.exec.daffodil.directory.staging";
355+
public static final String DFDL_DIRECTORY_REGISTRY = "drill.exec.daffodil.directory.registry";
356+
public static final String DFDL_DIRECTORY_TMP = "drill.exec.daffodil.directory.tmp";
357+
346358
/**
347359
* Local temporary directory is used as base for temporary storage of Dynamic UDF jars.
348360
*/

‎exec/java-exec/src/main/resources/drill-module.conf‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -410,6 +410,31 @@ drill.exec: {
410410
threadpool_size: 8,
411411
decode_threadpool_size: 1
412412
},
413+
daffodil: {
414+
retry-attempts: 5,
415+
directory: {
416+
# Base directory for remote and local daffodil directories, unique among clusters.
417+
base: ${drill.exec.zk.root}"/daffodil",
418+
419+
# Path to local daffodil, always created on local file system.
420+
# Root for these directory is generated at runtime unless Drill temporary directory is set.
421+
local: ${drill.exec.daffodil.directory.base}"/local",
422+
423+
# Set this property if custom file system should be used to create remote directories, ex: fs: "file:///".
424+
# fs: "",
425+
# Set this property if custom absolute root should be used for remote directories, ex: root: "/app/drill".
426+
# root: "",
427+
428+
# Relative path to all remote daffodil directories.
429+
# Directories are created under default file system taken from Hadoop configuration
430+
# unless ${drill.exec.daffodil.directory.fs} is set.
431+
# User home directory is used as root unless ${drill.exec.daffodil.directory.root} is set.
432+
staging: ${drill.exec.daffodil.directory.base}"/staging",
433+
registry: ${drill.exec.daffodil.directory.base}"/registry",
434+
tmp: ${drill.exec.daffodil.directory.base}"/tmp"
435+
}
436+
},
437+
413438
udf: {
414439
retry-attempts: 5,
415440
# Disables (parts of) the dynamic UDF functionality.

‎exec/java-exec/src/test/java/org/apache/drill/test/BaseTestQuery.java‎

Lines changed: 22 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -17,24 +17,8 @@
1717
*/
1818
package org.apache.drill.test;
1919

20-
import static org.apache.drill.exec.util.StoragePluginTestUtils.ROOT_SCHEMA;
21-
import static org.apache.drill.exec.util.StoragePluginTestUtils.TMP_SCHEMA;
22-
import static org.hamcrest.core.StringContains.containsString;
23-
import static org.junit.Assert.assertEquals;
24-
import static org.hamcrest.MatcherAssert.assertThat;
25-
import static org.junit.Assert.fail;
26-
27-
import java.io.File;
28-
import java.io.IOException;
29-
import java.io.PrintWriter;
30-
import java.net.URL;
31-
import java.nio.charset.StandardCharsets;
32-
import java.util.ArrayList;
33-
import java.util.Arrays;
34-
import java.util.List;
35-
import java.util.Properties;
36-
import java.util.concurrent.atomic.AtomicInteger;
37-
20+
import com.google.common.base.Preconditions;
21+
import com.google.common.io.Resources;
3822
import org.apache.drill.common.config.DrillConfig;
3923
import org.apache.drill.common.config.DrillProperties;
4024
import org.apache.drill.common.exceptions.UserException;
@@ -65,15 +49,31 @@
6549
import org.apache.drill.exec.util.StoragePluginTestUtils;
6650
import org.apache.drill.exec.util.VectorUtil;
6751
import org.apache.drill.exec.vector.ValueVector;
68-
import com.google.common.base.Preconditions;
69-
import com.google.common.io.Resources;
7052
import org.apache.drill.test.DrillTestWrapper.TestServices;
7153
import org.apache.hadoop.fs.FileSystem;
7254
import org.junit.AfterClass;
7355
import org.junit.BeforeClass;
7456
import org.slf4j.Logger;
7557
import org.slf4j.LoggerFactory;
7658

59+
import java.io.File;
60+
import java.io.IOException;
61+
import java.io.PrintWriter;
62+
import java.net.URL;
63+
import java.nio.charset.StandardCharsets;
64+
import java.util.ArrayList;
65+
import java.util.Arrays;
66+
import java.util.List;
67+
import java.util.Properties;
68+
import java.util.concurrent.atomic.AtomicInteger;
69+
70+
import static org.apache.drill.exec.util.StoragePluginTestUtils.ROOT_SCHEMA;
71+
import static org.apache.drill.exec.util.StoragePluginTestUtils.TMP_SCHEMA;
72+
import static org.hamcrest.MatcherAssert.assertThat;
73+
import static org.hamcrest.core.StringContains.containsString;
74+
import static org.junit.Assert.assertEquals;
75+
import static org.junit.Assert.fail;
76+
7777
/**
7878
* deprecated Use {@link ClusterTest} instead.
7979
*
@@ -168,6 +168,8 @@ protected static Properties cloneDefaultTestConfigProperties() {
168168
props.setProperty(ExecConstants.SYS_STORE_PROVIDER_LOCAL_PATH, dirTestWatcher.getStoreDir().getAbsolutePath());
169169
props.setProperty(ExecConstants.UDF_DIRECTORY_ROOT, dirTestWatcher.getHomeDir().getAbsolutePath());
170170
props.setProperty(ExecConstants.UDF_DIRECTORY_FS, FileSystem.DEFAULT_FS);
171+
props.setProperty(ExecConstants.DFDL_DIRECTORY_ROOT, dirTestWatcher.getHomeDir().getAbsolutePath());
172+
props.setProperty(ExecConstants.DFDL_DIRECTORY_FS, FileSystem.DEFAULT_FS);
171173

172174
return props;
173175
}

‎exec/java-exec/src/test/java/org/apache/drill/test/ClusterFixture.java‎

Lines changed: 27 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -17,30 +17,13 @@
1717
*/
1818
package org.apache.drill.test;
1919

20-
import static org.apache.drill.exec.util.StoragePluginTestUtils.DFS_TMP_SCHEMA;
21-
import static org.apache.drill.exec.util.StoragePluginTestUtils.ROOT_SCHEMA;
22-
import static org.apache.drill.exec.util.StoragePluginTestUtils.TMP_SCHEMA;
23-
24-
import java.io.File;
25-
import java.io.IOException;
26-
import java.net.URI;
27-
import java.net.URL;
28-
import java.nio.charset.StandardCharsets;
29-
import java.nio.file.Paths;
30-
import java.sql.Connection;
31-
import java.sql.DriverManager;
32-
import java.sql.SQLException;
33-
import java.util.ArrayList;
34-
import java.util.Collection;
35-
import java.util.HashMap;
36-
import java.util.List;
37-
import java.util.Map;
38-
import java.util.Optional;
39-
import java.util.Properties;
40-
20+
import com.google.common.base.Preconditions;
21+
import com.google.common.collect.ImmutableMap;
22+
import com.google.common.io.Resources;
4123
import org.apache.drill.common.config.DrillProperties;
4224
import org.apache.drill.common.logical.FormatPluginConfig;
4325
import org.apache.drill.common.logical.StoragePluginConfig;
26+
import org.apache.drill.common.logical.security.PlainCredentialsProvider;
4427
import org.apache.drill.exec.ExecConstants;
4528
import org.apache.drill.exec.ZookeeperHelper;
4629
import org.apache.drill.exec.client.DrillClient;
@@ -58,15 +41,32 @@
5841
import org.apache.drill.exec.store.dfs.FileSystemConfig;
5942
import org.apache.drill.exec.store.dfs.WorkspaceConfig;
6043
import org.apache.drill.exec.store.mock.MockStorageEngineConfig;
61-
import org.apache.drill.common.logical.security.PlainCredentialsProvider;
6244
import org.apache.drill.exec.store.sys.store.provider.ZookeeperPersistentStoreProvider;
6345
import org.apache.drill.exec.util.StoragePluginTestUtils;
64-
import com.google.common.base.Preconditions;
65-
import com.google.common.collect.ImmutableMap;
66-
import com.google.common.io.Resources;
6746
import org.apache.drill.test.DrillTestWrapper.TestServices;
6847
import org.apache.hadoop.fs.FileSystem;
6948

49+
import java.io.File;
50+
import java.io.IOException;
51+
import java.net.URI;
52+
import java.net.URL;
53+
import java.nio.charset.StandardCharsets;
54+
import java.nio.file.Paths;
55+
import java.sql.Connection;
56+
import java.sql.DriverManager;
57+
import java.sql.SQLException;
58+
import java.util.ArrayList;
59+
import java.util.Collection;
60+
import java.util.HashMap;
61+
import java.util.List;
62+
import java.util.Map;
63+
import java.util.Optional;
64+
import java.util.Properties;
65+
66+
import static org.apache.drill.exec.util.StoragePluginTestUtils.DFS_TMP_SCHEMA;
67+
import static org.apache.drill.exec.util.StoragePluginTestUtils.ROOT_SCHEMA;
68+
import static org.apache.drill.exec.util.StoragePluginTestUtils.TMP_SCHEMA;
69+
7070
/**
7171
* Test fixture to start a Drillbit with provide options, create a client, and
7272
* execute queries. Can be used in JUnit tests, or in ad-hoc programs. Provides
@@ -615,8 +615,10 @@ public static ClusterFixtureBuilder builder(BaseDirTestWatcher dirTestWatcher) {
615615
props.putAll(ClusterFixture.TEST_CONFIGURATIONS);
616616
props.setProperty(ExecConstants.DRILL_TMP_DIR, dirTestWatcher.getTmpDir().getAbsolutePath());
617617
props.setProperty(ExecConstants.UDF_DIRECTORY_ROOT, dirTestWatcher.getHomeDir().getAbsolutePath());
618+
props.setProperty(ExecConstants.DFDL_DIRECTORY_ROOT, dirTestWatcher.getHomeDir().getAbsolutePath());
618619
props.setProperty(ExecConstants.SYS_STORE_PROVIDER_LOCAL_PATH, dirTestWatcher.getStoreDir().getAbsolutePath());
619620
props.setProperty(ExecConstants.UDF_DIRECTORY_FS, FileSystem.DEFAULT_FS);
621+
props.setProperty(ExecConstants.DFDL_DIRECTORY_FS, FileSystem.DEFAULT_FS);
620622
// ALTER SESSION profiles are seldom interesting
621623
props.setProperty(ExecConstants.SKIP_ALTER_SESSION_QUERY_PROFILE, Boolean.TRUE.toString());
622624

‎exec/java-exec/src/test/java/org/apache/drill/test/OperatorFixture.java‎

Lines changed: 20 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -17,28 +17,18 @@
1717
*/
1818
package org.apache.drill.test;
1919

20-
import org.apache.drill.exec.alias.AliasRegistryProvider;
21-
import org.apache.drill.exec.physical.resultSet.ResultSetLoader;
22-
import org.apache.drill.exec.physical.resultSet.impl.ResultSetLoaderImpl;
23-
import org.apache.drill.exec.physical.resultSet.impl.ResultSetLoaderImpl.ResultSetOptions;
24-
import org.apache.drill.exec.server.DrillbitContext;
25-
import org.apache.drill.metastore.MetastoreRegistry;
2620
import com.google.common.base.Function;
2721
import com.google.common.base.Preconditions;
2822
import com.google.common.collect.Lists;
2923
import com.google.common.util.concurrent.ListenableFuture;
30-
import java.util.ArrayList;
31-
import java.util.Arrays;
32-
import java.util.List;
33-
import java.util.concurrent.Callable;
34-
import java.util.concurrent.ExecutorService;
3524
import io.netty.buffer.DrillBuf;
3625
import org.apache.calcite.schema.SchemaPlus;
3726
import org.apache.drill.common.config.DrillConfig;
3827
import org.apache.drill.common.scanner.ClassPathScanner;
3928
import org.apache.drill.common.scanner.persistence.ScanResult;
4029
import org.apache.drill.common.types.TypeProtos;
4130
import org.apache.drill.exec.ExecConstants;
31+
import org.apache.drill.exec.alias.AliasRegistryProvider;
4232
import org.apache.drill.exec.compile.ClassBuilder;
4333
import org.apache.drill.exec.compile.CodeCompiler;
4434
import org.apache.drill.exec.exception.OutOfMemoryException;
@@ -56,6 +46,15 @@
5646
import org.apache.drill.exec.ops.OperatorContext;
5747
import org.apache.drill.exec.ops.OperatorStats;
5848
import org.apache.drill.exec.physical.base.PhysicalOperator;
49+
import org.apache.drill.exec.physical.resultSet.ResultSetLoader;
50+
import org.apache.drill.exec.physical.resultSet.impl.ResultSetLoaderImpl;
51+
import org.apache.drill.exec.physical.resultSet.impl.ResultSetLoaderImpl.ResultSetOptions;
52+
import org.apache.drill.exec.physical.rowSet.DirectRowSet;
53+
import org.apache.drill.exec.physical.rowSet.HyperRowSetImpl;
54+
import org.apache.drill.exec.physical.rowSet.IndirectRowSet;
55+
import org.apache.drill.exec.physical.rowSet.RowSet;
56+
import org.apache.drill.exec.physical.rowSet.RowSet.ExtendableRowSet;
57+
import org.apache.drill.exec.physical.rowSet.RowSetBuilder;
5958
import org.apache.drill.exec.planner.PhysicalPlanReaderTestFactory;
6059
import org.apache.drill.exec.proto.ExecProtos;
6160
import org.apache.drill.exec.record.BatchSchema;
@@ -65,22 +64,24 @@
6564
import org.apache.drill.exec.record.metadata.MetadataUtils;
6665
import org.apache.drill.exec.record.metadata.TupleMetadata;
6766
import org.apache.drill.exec.record.selection.SelectionVector2;
67+
import org.apache.drill.exec.server.DrillbitContext;
6868
import org.apache.drill.exec.server.options.OptionManager;
6969
import org.apache.drill.exec.server.options.SystemOptionManager;
7070
import org.apache.drill.exec.store.PartitionExplorer;
7171
import org.apache.drill.exec.store.sys.PersistentStoreProvider;
7272
import org.apache.drill.exec.store.sys.store.provider.LocalPersistentStoreProvider;
7373
import org.apache.drill.exec.testing.ExecutionControls;
7474
import org.apache.drill.exec.work.filter.RuntimeFilterWritable;
75+
import org.apache.drill.metastore.MetastoreRegistry;
7576
import org.apache.drill.test.ClusterFixtureBuilder.RuntimeOption;
76-
import org.apache.drill.exec.physical.rowSet.DirectRowSet;
77-
import org.apache.drill.exec.physical.rowSet.HyperRowSetImpl;
78-
import org.apache.drill.exec.physical.rowSet.IndirectRowSet;
79-
import org.apache.drill.exec.physical.rowSet.RowSet;
80-
import org.apache.drill.exec.physical.rowSet.RowSet.ExtendableRowSet;
81-
import org.apache.drill.exec.physical.rowSet.RowSetBuilder;
8277
import org.apache.hadoop.fs.FileSystem;
8378
import org.apache.hadoop.security.UserGroupInformation;
79+
80+
import java.util.ArrayList;
81+
import java.util.Arrays;
82+
import java.util.List;
83+
import java.util.concurrent.Callable;
84+
import java.util.concurrent.ExecutorService;
8485
import java.util.concurrent.TimeUnit;
8586

8687
/**
@@ -137,6 +138,8 @@ public Builder(BaseDirTestWatcher dirTestWatcher)
137138
configBuilder.put(ExecConstants.HASHJOIN_SPILL_DIRS, Arrays.asList(dirTestWatcher.getSpillDir().getAbsolutePath()));
138139
configBuilder.put(ExecConstants.UDF_DIRECTORY_ROOT, dirTestWatcher.getHomeDir().getAbsolutePath());
139140
configBuilder.put(ExecConstants.UDF_DIRECTORY_FS, FileSystem.DEFAULT_FS);
141+
configBuilder.put(ExecConstants.DFDL_DIRECTORY_ROOT, dirTestWatcher.getHomeDir().getAbsolutePath());
142+
configBuilder.put(ExecConstants.DFDL_DIRECTORY_FS, FileSystem.DEFAULT_FS);
140143
}
141144
}
142145

0 commit comments

Comments
 (0)