Skip to content

Commit 14f524a

Browse files
witold-swierzywitold-swierzy
authored
Add files via upload
1 parent f7b1bf9 commit 14f524a

File tree

5 files changed

+697
-0
lines changed

5 files changed

+697
-0
lines changed

‎data-platform/autonomous-database/autonomous-json/MongoDBReplay/src/Applier.java

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
package utils.dbutils;
2+
3+
import com.google.gson.Gson;
4+
import com.google.gson.JsonObject;
5+
import com.mongodb.client.MongoClient;
6+
import com.mongodb.client.MongoClients;
7+
import com.mongodb.client.MongoDatabase;
8+
import org.bson.BsonDocument;
9+
import org.bson.Document;
10+
11+
import java.io.*;
12+
import java.time.LocalDateTime;
13+
14+
public class Applier {
15+
private static int numOfSucceses = 0;
16+
private static int numOfErrors = 0;
17+
private static int numOfAllCommands = 0;
18+
private static String dbName, oldDbname;
19+
private static MongoClient client;
20+
private static MongoDatabase db;
21+
private static BufferedReader inputFile;
22+
23+
public static void printSettings(boolean footer) {
24+
if (!footer)
25+
Config.logMessage(LocalDateTime.now() + " : Starting commands application. ", Config.LOG_LEVEL_SUMMARY);
26+
else {
27+
Config.logMessage(LocalDateTime.now() + " : Commands application completed.", Config.LOG_LEVEL_SUMMARY);
28+
Config.logMessage("Summary", Config.LOG_LEVEL_SUMMARY);
29+
}
30+
31+
if (Config.inputFileName != null)
32+
Config.logMessage("Input file : " + Config.inputFileName, Config.LOG_LEVEL_SUMMARY);
33+
else
34+
Config.logMessage("Input redirected to StdIn", Config.LOG_LEVEL_SUMMARY);
35+
36+
if (Config.logFileName != null)
37+
Config.logMessage("Log file : " + Config.logFileName, Config.LOG_LEVEL_SUMMARY);
38+
else
39+
Config.logMessage("Logging set to StdOut.", Config.LOG_LEVEL_SUMMARY);
40+
Config.logMessage("Log Level : "+Config.logLevel, Config.LOG_LEVEL_SUMMARY);
41+
Config.logMessage("Database connect string : "+Config.connectString, Config.LOG_LEVEL_SUMMARY);
42+
if (footer)
43+
{
44+
Config.logMessage("Number of all commands : " + numOfAllCommands, Config.LOG_LEVEL_SUMMARY);
45+
Config.logMessage("Number of successful executions : " + numOfSucceses, Config.LOG_LEVEL_SUMMARY);
46+
Config.logMessage("Number of errors : " + numOfErrors, Config.LOG_LEVEL_SUMMARY);
47+
}
48+
}
49+
50+
public static void initialize() throws Exception {
51+
Config.readConfiguration(Config.APPLY);
52+
printSettings(false);
53+
client = MongoClients.create(Config.connectString);
54+
db = client.getDatabase(Config.dbName);
55+
if (Config.inputFileName != null)
56+
inputFile = new BufferedReader((new FileReader(Config.inputFileName)));
57+
else
58+
inputFile = new BufferedReader(new InputStreamReader(System.in));
59+
dbName = Config.dbName;
60+
oldDbname = Config.dbName;
61+
}
62+
63+
public static synchronized void shutdown() {
64+
try {
65+
printSettings(true);
66+
Config.logFile.close();
67+
} catch (Exception e) {
68+
e.printStackTrace();
69+
System.exit(0);
70+
}
71+
}
72+
73+
public static void main(String[] args) {
74+
String line = "";
75+
JsonObject commandJSON;
76+
BsonDocument commandBSON;
77+
Document commandResult;
78+
79+
try {
80+
initialize();
81+
while ((line = inputFile.readLine()) != null) {
82+
numOfAllCommands++;
83+
try {
84+
Config.logMessage("Command #"+numOfAllCommands+" : "+line, Config.LOG_LEVEL_ALL);
85+
Gson gson = new Gson();
86+
commandJSON = gson.fromJson(line, JsonObject.class);
87+
dbName = commandJSON.getAsJsonPrimitive("$db").getAsString();
88+
if (!dbName.equals(oldDbname)) {
89+
db = client.getDatabase(dbName);
90+
oldDbname = dbName;
91+
}
92+
commandJSON.remove("$db");
93+
commandBSON = BsonDocument.parse(commandJSON.toString());
94+
commandResult = db.runCommand(commandBSON);
95+
Config.logMessage("Result : "+commandResult.toString().substring(8),Config.LOG_LEVEL_ALL);
96+
numOfSucceses++;
97+
} catch(Exception e) {
98+
numOfErrors++;
99+
Config.logMessage("Error #"+numOfErrors+" : "+line, Config.LOG_LEVEL_ERRORS);
100+
Config.logMessage(e.toString(), Config.LOG_LEVEL_ERRORS);
101+
}
102+
}
103+
} catch (Exception e)
104+
{e.printStackTrace();}
105+
shutdown();
106+
}
107+
}

‎data-platform/autonomous-database/autonomous-json/MongoDBReplay/src/Config.java

Lines changed: 246 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,246 @@
1+
package utils.dbutils;
2+
3+
import com.google.gson.*;
4+
5+
import java.io.File;
6+
import java.io.FileOutputStream;
7+
import java.io.PrintStream;
8+
import java.io.Reader;
9+
import java.nio.file.Files;
10+
import java.nio.file.Paths;
11+
12+
public class Config {
13+
public static final int EXTRACT=1;
14+
public static final int APPLY=2;
15+
16+
public static final int LOG_LEVEL_SUMMARY = 0;
17+
public static final int LOG_LEVEL_ERRORS = 1;
18+
public static final int LOG_LEVEL_ALL = 2;
19+
20+
public static int runningMode;
21+
public static String configDir;
22+
public static String inputFileName;
23+
public static String configFileName;
24+
public static PrintStream logFile;
25+
public static String logFileName;
26+
public static int logLevel = 0;
27+
public static String shutdownFileName;
28+
public static String outputDir;
29+
public static JsonArray includeDbs;
30+
public static JsonArray excludeDbs;
31+
public static JsonArray includeCmds;
32+
public static JsonArray excludeCmds;
33+
public static boolean commandsLogging;
34+
public static String connectString;
35+
public static String dbName;
36+
public static int outputMode = 0; // 0 - mongosh script, default
37+
// 1 - json
38+
public static int inputFileFormat = 0; // 0 - Momngod log
39+
// 1 - db.system.profile collection dump
40+
// 2 - db.system.profile direct reads
41+
42+
public static int executionTracing = 0; // 0 - not tracing
43+
// 1 - query planner
44+
// 2 - execution stats
45+
// 3 - allPlansExecution
46+
public static ManagingThread t = new ManagingThread();
47+
private static Reader reader;
48+
private static JsonObject configObject;
49+
50+
public static boolean traceAllDbs() {
51+
return (includeDbs == null && excludeDbs == null);
52+
}
53+
54+
public static boolean traceInclDbs() {
55+
return (includeDbs != null);
56+
}
57+
58+
public static boolean traceExclDbs() {
59+
return (excludeDbs != null);
60+
}
61+
62+
public static boolean traceInclCmds() {
63+
return (includeCmds != null);
64+
}
65+
66+
public static boolean traceExclCmds() {
67+
return (excludeCmds != null);
68+
}
69+
70+
public static boolean traceAllCmds() {
71+
return (includeCmds == null && excludeCmds == null);
72+
}
73+
74+
public static boolean traceDatabase(String dbName) {
75+
boolean contains = false;
76+
if ( includeDbs == null && excludeDbs == null )
77+
contains = true;
78+
else if ( excludeDbs == null )
79+
contains = includeDbs.contains(new JsonParser().parse(dbName));
80+
else contains = !excludeDbs.contains(new JsonParser().parse(dbName));
81+
return contains;
82+
}
83+
84+
public static boolean traceCommand(JsonObject command) {
85+
boolean contains = false;
86+
int i = 0;
87+
if (includeCmds == null && excludeCmds == null)
88+
contains = true;
89+
else if (excludeCmds == null)
90+
while (!contains && i < includeCmds.asList().toArray().length) {
91+
if (command.has(includeCmds.asList().get(i).getAsString()))
92+
contains = true;
93+
i++;
94+
}
95+
else if (includeCmds == null) {
96+
contains = true;
97+
while (contains && i < excludeCmds.asList().toArray().length) {
98+
if (command.has(excludeCmds.asList().get(i).getAsString()))
99+
contains = false;
100+
i++;
101+
}
102+
}
103+
return contains;
104+
}
105+
106+
public static void readExtractConfiguration() {
107+
try {
108+
109+
if (configObject.has("INPUT_FILE"))
110+
inputFileName = configObject.getAsJsonPrimitive("INPUT_FILE").getAsString();
111+
else
112+
t.start();
113+
114+
if (configObject.has("OUTPUT_DIR"))
115+
outputDir = configObject.getAsJsonPrimitive("OUTPUT_DIR").getAsString();
116+
117+
if (configObject.has("INCLUDE_COMMANDS") &&
118+
configObject.has("EXCLUDE_COMMANDS"))
119+
throw new Exception("You cannot set INCLUDE_COMMANDS and EXCLUDE_COMMANDS at the same time. Please, review the available documentation.");
120+
121+
if (configObject.has("INCLUDE_DATABASES"))
122+
includeDbs = configObject.getAsJsonArray("INCLUDE_DATABASES");
123+
124+
if (configObject.has("EXCLUDE_DATABASES"))
125+
excludeDbs = configObject.getAsJsonArray("EXCLUDE_DATABASES");
126+
127+
if (configObject.has("INCLUDE_COMMANDS"))
128+
includeCmds = configObject.getAsJsonArray("INCLUDE_COMMANDS");
129+
130+
if (configObject.has("EXCLUDE_COMMANDS"))
131+
excludeCmds = configObject.getAsJsonArray("EXCLUDE_COMMANDS");
132+
133+
if (configObject.has("COMMANDS_LOGGING"))
134+
commandsLogging = configObject.getAsJsonPrimitive("COMMANDS_LOGGING").getAsBoolean();
135+
else
136+
commandsLogging = true;
137+
138+
if (configObject.has("EXECUTION_PLAN_TRACING"))
139+
executionTracing = configObject.getAsJsonPrimitive("EXECUTION_PLAN_TRACING").getAsInt();
140+
141+
if (configObject.has("INPUT_FILE_FORMAT")) {
142+
if ( configObject.getAsJsonPrimitive("INPUT_FILE_FORMAT").getAsString().equals("MONGO_LOG"))
143+
inputFileFormat = 0;
144+
else if ( configObject.getAsJsonPrimitive("INPUT_FILE_FORMAT").getAsString().equals("MONGO_PROFILE"))
145+
inputFileFormat = 1;
146+
else throw new Exception("INPUT_FILE_FORMAT can be set to MONGO_LOG or MONGO_PROFILE");
147+
}
148+
else inputFileFormat = 0;
149+
150+
if (configObject.has("INPUT_CONNECT_STRING")) {
151+
if (configObject.has("INPUT_FILE_FORMAT") || configObject.has("INPUT_FILE"))
152+
throw new Exception("If INPUT_CONNECT_STRING is set then INPUT_FILE and INPUT_FILE_FORMAT cannot be set");
153+
154+
if (configObject.has("DB_NAME"))
155+
dbName = configObject.getAsJsonPrimitive("DB_NAME").getAsString();
156+
else throw new Exception("DB_NAME parameter is mandatory when INPUT_CONNECT_STRING is set.");
157+
158+
connectString = configObject.get("INPUT_CONNECT_STRING").getAsString();
159+
inputFileFormat = 2;
160+
}
161+
162+
if (configObject.has("OUTPUT_MODE")) {
163+
if ( configObject.getAsJsonPrimitive("OUTPUT_MODE").getAsString().equals("SCRIPT"))
164+
outputMode = 0;
165+
else if (configObject.getAsJsonPrimitive("OUTPUT_MODE").getAsString().equals("JSON"))
166+
outputMode = 1;
167+
else throw new Exception("OUTPUT_MODE can be set to SCRIPT or JSON values only");
168+
}
169+
else
170+
outputMode = 0;
171+
172+
173+
174+
} catch (Exception e) {
175+
e.printStackTrace();
176+
System.exit(0);
177+
}
178+
}
179+
180+
public static void readApplyConfiguration() {
181+
try {
182+
if (configObject.has("CONNECT_STRING"))
183+
connectString = configObject.getAsJsonPrimitive("CONNECT_STRING").getAsString();
184+
else throw new Exception("CONNECT_STRING parameter is mandatory.");
185+
186+
187+
if (configObject.has("DB_NAME"))
188+
dbName = configObject.getAsJsonPrimitive("DB_NAME").getAsString();
189+
else throw new Exception("DB_NAME parameter is mandatory.");
190+
191+
if (configObject.has("INPUT_FILE"))
192+
inputFileName = configObject.getAsJsonPrimitive("INPUT_FILE").getAsString();
193+
else
194+
t.start();
195+
} catch (Exception e) {
196+
e.printStackTrace();
197+
System.exit(0);
198+
}
199+
}
200+
201+
public static void readConfiguration(int mode) {
202+
try {
203+
runningMode = mode;
204+
configDir = System.getenv("MR_CONFIG_DIR");
205+
if (configDir == null || configDir.length() == 0)
206+
throw new Exception("MR_CONFIG_DIR environment variable is mandatory. Please review the available documentation.");
207+
208+
if (mode == EXTRACT) {
209+
configFileName = configDir + File.separator + "mdbecfg.json";
210+
shutdownFileName = configDir + File.separator + "mdbes.label";
211+
}
212+
else {
213+
configFileName = configDir + File.separator + "mdbacfg.json";
214+
shutdownFileName = configDir + File.separator + "mdbas.label";
215+
}
216+
217+
reader = Files.newBufferedReader(Paths.get(configFileName));
218+
configObject = (new Gson()).fromJson(reader, JsonObject.class);
219+
220+
if (configObject.has("LOG_FILE")) {
221+
logFileName = configObject.getAsJsonPrimitive("LOG_FILE").getAsString();
222+
logFile = new PrintStream(new FileOutputStream(Config.logFileName), true);
223+
}
224+
else
225+
logFile = System.err;
226+
227+
if (configObject.has("LOG_LEVEL"))
228+
logLevel = configObject.getAsJsonPrimitive("LOG_LEVEL").getAsInt();
229+
230+
if (mode == EXTRACT)
231+
readExtractConfiguration();
232+
else
233+
readApplyConfiguration();
234+
235+
reader.close();
236+
} catch (Exception e) {
237+
e.printStackTrace();
238+
System.exit(0);
239+
}
240+
}
241+
242+
public static void logMessage(String message, int level) {
243+
if (level <= logLevel)
244+
logFile.println(message);
245+
}
246+
}

0 commit comments

Comments
 (0)