Skip to content

Commit fc5ab8e

Browse files
clunclun
committed
Dsbulk
1 parent c8b3b27 commit fc5ab8e

File tree

11 files changed

+244
-21
lines changed

11 files changed

+244
-21
lines changed

‎astra-shell/NOTES.MD

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3,15 +3,10 @@
33

44
```
55
mvn package -Dmaven.test.skip=true -Dmaven.javadoc.skip=true
6-
cp ./target/astra-shell-0.3.2-SNAPSHOT-shaded.jar ~/.astra/cli/astra-shell.jar
6+
cp ./target/astra-shell-0.1-alpha3-shaded.jar ~/.astra/cli/astra-shell.jar
77
```
88

9-
# Executing the command
109

11-
```
12-
java -jar ./target/astra-shell-0.3.1-SNAPSHOT-shaded.jar show-dbs
13-
```
14-
1510

1611

1712
sdk, brew, chocolaty

‎astra-shell/astra

Lines changed: 0 additions & 4 deletions
This file was deleted.

‎astra-shell/dependency-reduced-pom.xml

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,12 @@
33
<parent>
44
<artifactId>astra-sdk-parent</artifactId>
55
<groupId>com.datastax.astra</groupId>
6-
<version>0.3.2-SNAPSHOT</version>
6+
<version>0.3.4-SNAPSHOT</version>
77
</parent>
88
<modelVersion>4.0.0</modelVersion>
99
<artifactId>astra-shell</artifactId>
1010
<name>+ astra-shell</name>
11+
<version>0.1-alpha3</version>
1112
<build>
1213
<plugins>
1314
<plugin>
@@ -182,6 +183,7 @@
182183
<maven-plugin-exec.version>3.0.0</maven-plugin-exec.version>
183184
<jansi.version>2.4.0</jansi.version>
184185
<commons-compress.version>1.21</commons-compress.version>
186+
<astra-sdk.version>0.3.3</astra-sdk.version>
185187
<commons-lang.version>3.12.0</commons-lang.version>
186188
<native.maven.plugin.version>0.9.11</native.maven.plugin.version>
187189
</properties>

‎astra-shell/pom.xml

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,14 +6,16 @@
66
<artifactId>astra-shell</artifactId>
77
<name>+ astra-shell</name>
88
<packaging>jar</packaging>
9+
<version>0.1-alpha3</version>
910

1011
<parent>
1112
<groupId>com.datastax.astra</groupId>
1213
<artifactId>astra-sdk-parent</artifactId>
13-
<version>0.3.2-SNAPSHOT</version>
14+
<version>0.3.4-SNAPSHOT</version>
1415
</parent>
1516

1617
<properties>
18+
<astra-sdk.version>0.3.3</astra-sdk.version>
1719
<jansi.version>2.4.0</jansi.version>
1820
<airline.version>2.8.5</airline.version>
1921
<commons-cli.version>1.5.0</commons-cli.version>
@@ -45,7 +47,7 @@
4547
<dependency>
4648
<groupId>com.datastax.astra</groupId>
4749
<artifactId>astra-sdk</artifactId>
48-
<version>${project.version}</version>
50+
<version>${astra-sdk.version}</version>
4951
<exclusions>
5052
<exclusion>
5153
<groupId>org.apache.pulsar</groupId>

‎astra-shell/src/main/java/com/datastax/astra/shell/AstraCli.java

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -9,23 +9,24 @@
99
import com.datastax.astra.shell.cmd.config.ConfigCreate;
1010
import com.datastax.astra.shell.cmd.config.ConfigDefault;
1111
import com.datastax.astra.shell.cmd.config.ConfigDelete;
12-
import com.datastax.astra.shell.cmd.config.ConfigList;
1312
import com.datastax.astra.shell.cmd.config.ConfigGet;
13+
import com.datastax.astra.shell.cmd.config.ConfigList;
1414
import com.datastax.astra.shell.cmd.config.ConfigSetup;
1515
import com.datastax.astra.shell.cmd.db.DbCqlShellCli;
1616
import com.datastax.astra.shell.cmd.db.DbCreateCli;
1717
import com.datastax.astra.shell.cmd.db.DbCreateKeyspaceCli;
1818
import com.datastax.astra.shell.cmd.db.DbDeleteCli;
19-
import com.datastax.astra.shell.cmd.db.DbListCli;
19+
import com.datastax.astra.shell.cmd.db.DbDSBulkCli;
2020
import com.datastax.astra.shell.cmd.db.DbGetCli;
21+
import com.datastax.astra.shell.cmd.db.DbListCli;
2122
import com.datastax.astra.shell.cmd.db.OperationsDb;
22-
import com.datastax.astra.shell.cmd.iam.RoleListCli;
2323
import com.datastax.astra.shell.cmd.iam.OperationIam;
2424
import com.datastax.astra.shell.cmd.iam.RoleGetCli;
25+
import com.datastax.astra.shell.cmd.iam.RoleListCli;
2526
import com.datastax.astra.shell.cmd.iam.UserDeleteCli;
27+
import com.datastax.astra.shell.cmd.iam.UserGetCli;
2628
import com.datastax.astra.shell.cmd.iam.UserInviteCli;
2729
import com.datastax.astra.shell.cmd.iam.UserListCli;
28-
import com.datastax.astra.shell.cmd.iam.UserGetCli;
2930
import com.datastax.astra.shell.cmd.shell.ShellCommand;
3031
import com.datastax.astra.shell.out.LoggerShell;
3132
import com.github.rvesse.airline.annotations.Cli;
@@ -53,7 +54,8 @@
5354
DbDeleteCli.class,
5455
DbGetCli.class,
5556
DbListCli.class,
56-
DbCreateKeyspaceCli.class
57+
DbCreateKeyspaceCli.class,
58+
DbDSBulkCli.class
5759
}),
5860
@Group(name = BaseConfigCommand.COMMAND_CONFIG, description = "Manage configuration file", commands = {
5961
ConfigCreate.class,
@@ -118,6 +120,7 @@ public static void main(String[] args) {
118120

119121
} catch(Exception e) {
120122
LoggerShell.error("Execution error:" + e.getMessage());
123+
e.printStackTrace();
121124
//e.printStackTrace();
122125
}
123126
}

‎astra-shell/src/main/java/com/datastax/astra/shell/cmd/BaseCliCommand.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ public abstract class BaseCliCommand extends BaseShellCommand {
2121
// --- Authentication ---
2222

2323
/** Authentication token used if not provided in config. */
24-
@Option(name = { "-t", "--token" },
24+
@Option(name = { "--token" },
2525
title = "AUTH_TOKEN",
2626
description = "Key to use authenticate each call.")
2727
protected String token;

‎astra-shell/src/main/java/com/datastax/astra/shell/cmd/config/OperationsConfig.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,7 @@ public class OperationsConfig {
2929
/**
3030
* Hide default constructor
3131
*/
32-
private OperationsConfig() {
33-
}
32+
private OperationsConfig() {}
3433

3534
/**
3635
* Show configuration in the output.
@@ -82,6 +81,7 @@ public static List<String> listOrganizations(Map<String, Map<String, String>> se
8281
* @param sections
8382
* list of sections
8483
* @return
84+
* organization name if exists
8585
*/
8686
public static Optional<String> findDefaultOrganizationName(Map<String, Map<String, String>> sections) {
8787
String defaultOrgName = null;

‎astra-shell/src/main/java/com/datastax/astra/shell/cmd/db/DbDSBulkCli.java

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
package com.datastax.astra.shell.cmd.db;
2+
3+
import java.util.List;
4+
5+
import com.datastax.astra.shell.ExitCode;
6+
import com.datastax.astra.shell.cmd.BaseCliCommand;
7+
import com.github.rvesse.airline.annotations.Arguments;
8+
import com.github.rvesse.airline.annotations.Command;
9+
10+
/**
11+
* This command allows to load data with DsBulk.
12+
*
13+
* @author Cedrick LUNVEN (@clunven)
14+
*/
15+
@Command(name = "dsbulk", description = "Load data leveraging DSBulk")
16+
public class DbDSBulkCli extends BaseCliCommand {
17+
18+
@Arguments(description = "Use DSBulk arguments")
19+
private List<String> dsbulkArguments;
20+
21+
/** {@inheritDoc} */
22+
@Override
23+
public ExitCode execute() {
24+
if (dsbulkArguments.size() < 3) {
25+
throw new IllegalArgumentException("Please use format astra db dsbulk <DB_NAME> [dsbulk options]");
26+
}
27+
return OperationsDb.runDsBulk(dsbulkArguments);
28+
}
29+
30+
}

‎astra-shell/src/main/java/com/datastax/astra/shell/cmd/db/OperationsDb.java

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
import com.datastax.astra.shell.out.ShellTable;
3333
import com.datastax.astra.shell.utils.CqlShellOptions;
3434
import com.datastax.astra.shell.utils.CqlShellUtils;
35+
import com.datastax.astra.shell.utils.DsBulkUtils;
3536

3637
/**
3738
* Utility class for command `db`
@@ -394,4 +395,34 @@ public static ExitCode startCqlShell(CqlShellOptions options, String database) {
394395
} catch (InterruptedException e) {}
395396
return ExitCode.SUCCESS;
396397
}
398+
399+
/**
400+
* Start DsBulk when needed.
401+
*
402+
* @param options
403+
* dsbulks options, database name is the first argument
404+
* @return
405+
* exit code
406+
*/
407+
public static ExitCode runDsBulk(List<String> options) {
408+
// Install dsbulk for Astra and set permissions
409+
DsBulkUtils.installDsBulk();
410+
411+
try {
412+
Optional<DatabaseClient> dbClient = OperationsDb.getDatabaseClient(options.get(0));
413+
if (dbClient.isPresent()) {
414+
Database db = dbClient.get().find().get();
415+
System.out.println("\nDSBulk is starting please wait ...");
416+
Process dsbulkProc = DsBulkUtils.runDsBulk(db, options.subList(1, options.size()));
417+
if (dsbulkProc == null) ExitCode.INTERNAL_ERROR.exit();
418+
dsbulkProc.waitFor();
419+
} else {
420+
return ExitCode.NOT_FOUND;
421+
}
422+
} catch (IOException e) {
423+
LoggerShell.error("Cannot start DSBULK");
424+
ExitCode.INTERNAL_ERROR.exit();
425+
} catch (InterruptedException e) {}
426+
return ExitCode.SUCCESS;
427+
}
397428
}

‎astra-shell/src/main/java/com/datastax/astra/shell/utils/DsBulkUtils.java

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
package com.datastax.astra.shell.utils;
2+
3+
import java.io.File;
4+
import java.io.IOException;
5+
import java.util.ArrayList;
6+
import java.util.List;
7+
8+
import org.apache.commons.lang3.StringUtils;
9+
10+
import com.datastax.astra.sdk.config.AstraClientConfig;
11+
import com.datastax.astra.sdk.databases.domain.Database;
12+
import com.datastax.astra.shell.AstraCli;
13+
import com.datastax.astra.shell.ExitCode;
14+
import com.datastax.astra.shell.ShellContext;
15+
import com.datastax.astra.shell.out.LoggerShell;
16+
import com.datastax.stargate.sdk.utils.Utils;
17+
18+
/**
19+
* Utilities to work with DSBulk.
20+
*
21+
* @author Cedrick LUNVEN (@clunven)
22+
*/
23+
public class DsBulkUtils {
24+
25+
/** Version Number. */
26+
public static final String DSBULK_VERSION = "1.9.1";
27+
28+
/** Archive name. */
29+
public static final String DSBULK_TARBALL = "dsbulk-" + DSBULK_VERSION + ".tar.gz";
30+
31+
/** URL. */
32+
public static final String DSBULK_DOWNLOAD = "https://downloads.datastax.com/dsbulk/" + DSBULK_TARBALL;
33+
34+
/** Folder name of dslbulk once untar. */
35+
public static final String DSBULK_FOLDER = "dsbulk-" + DSBULK_VERSION;
36+
37+
/**
38+
* Hide default construtor
39+
*/
40+
private DsBulkUtils() {}
41+
42+
/**
43+
* Check if cqlshel has been installed.
44+
*
45+
* @return
46+
* if the folder exist
47+
*/
48+
public static boolean isDsBulkInstalled() {
49+
File dsbulkFolder = new File(AstraCli.ASTRA_HOME + File.separator + DSBULK_FOLDER);
50+
return dsbulkFolder.exists() && dsbulkFolder.isDirectory();
51+
}
52+
53+
/**
54+
* Download targz and unzip.
55+
*/
56+
public static void installDsBulk() {
57+
if (!isDsBulkInstalled()) {
58+
LoggerShell.info("DSBulk has not been found, downloading, please wait...");
59+
String destination = AstraCli.ASTRA_HOME + File.separator + DSBULK_TARBALL;
60+
Utils.downloadFile(DSBULK_DOWNLOAD, destination);
61+
File dsbulkTarball = new File (destination);
62+
if (dsbulkTarball.exists()) {
63+
LoggerShell.info("File Downloaded. Extracting archive, please wait...");
64+
try {
65+
FileUtils.extactTargz(dsbulkTarball, new File (AstraCli.ASTRA_HOME));
66+
if (isDsBulkInstalled()) {
67+
// Change file permission
68+
File dsBulkFile = new File(AstraCli.ASTRA_HOME + File.separator
69+
+ DSBULK_FOLDER + File.separator
70+
+ "bin" + File.separator
71+
+ "dsbulk");
72+
dsBulkFile.setExecutable(true, false);
73+
dsBulkFile.setReadable(true, false);
74+
dsBulkFile.setWritable(true, false);
75+
76+
LoggerShell.success("DSBulk is installed");
77+
dsbulkTarball.delete();
78+
}
79+
} catch (IOException e) {
80+
LoggerShell.error("Cannot extract tar archive:" + e.getMessage());
81+
ExitCode.PARSE_ERROR.exit();
82+
}
83+
}
84+
} else {
85+
LoggerShell.info("DSBulk is already installed");
86+
}
87+
}
88+
89+
/**
90+
* Install CqlShell if needed and start the program.
91+
*
92+
* @param dsbulkParams
93+
* parameters for dsbulk
94+
* @param db
95+
* database retrieved
96+
* @return
97+
* unix process for cqlsh
98+
* @throws IOException
99+
* errors occured
100+
*/
101+
public static Process runDsBulk(Database db, List<String> dsbulkParams)
102+
throws IOException {
103+
List<String> commandDsbulk = new ArrayList<>();
104+
commandDsbulk.add(new StringBuilder()
105+
.append(AstraCli.ASTRA_HOME + File.separator + DSBULK_FOLDER)
106+
.append(File.separator + "bin")
107+
.append(File.separator + "dsbulk")
108+
.toString());
109+
commandDsbulk.addAll(dsbulkParams);
110+
commandDsbulk.add("-u");
111+
commandDsbulk.add("token");
112+
commandDsbulk.add("-p");
113+
commandDsbulk.add(ShellContext.getInstance().getToken());
114+
commandDsbulk.add("-b");
115+
File scb = new File(new StringBuilder()
116+
.append(AstraCli.ASTRA_HOME + File.separator + AstraCli.SCB_FOLDER + File.separator)
117+
.append(AstraClientConfig.buildScbFileName(db.getId(), db.getInfo().getRegion()))
118+
.toString());
119+
if (!scb.exists()) {
120+
LoggerShell.error("Cloud Secure Bundle '" + scb.getAbsolutePath() + "' has not been found.");
121+
ExitCode.NOT_FOUND.exit();
122+
}
123+
commandDsbulk.add(scb.getAbsolutePath());
124+
LoggerShell.info("RUNNING: " + StringUtils.join(commandDsbulk, " "));
125+
ProcessBuilder pb = new ProcessBuilder(commandDsbulk.toArray(new String[0]));
126+
pb.inheritIO();
127+
return pb.start();
128+
}
129+
130+
131+
}

0 commit comments

Comments
 (0)