[SYSTEMDS-3810] New codegen explain types for debugging

Implements new "-explain" command, "codegen" and "codegen_recompile" to
show generated code independent of LOG level

Closes #2165.

Author: Frxms

src/main/java/org/apache/sysds/api/DMLOptions.java

@@ -201,7 +201,9 @@ else if (lineageType.equalsIgnoreCase("debugger"))
 				else if (explainType.equalsIgnoreCase("runtime")) dmlOptions.explainType = ExplainType.RUNTIME;
 				else if (explainType.equalsIgnoreCase("recompile_hops")) dmlOptions.explainType = ExplainType.RECOMPILE_HOPS;
 				else if (explainType.equalsIgnoreCase("recompile_runtime")) dmlOptions.explainType = ExplainType.RECOMPILE_RUNTIME;
-				else throw new org.apache.commons.cli.ParseException("Invalid argument specified for -hops option, must be one of [hops, runtime, recompile_hops, recompile_runtime]");
+				else if (explainType.equalsIgnoreCase("codegen")) dmlOptions.explainType = ExplainType.CODEGEN;
+				else if (explainType.equalsIgnoreCase("codegen_recompile")) dmlOptions.explainType = ExplainType.CODEGEN_RECOMPILE;
+				else throw new org.apache.commons.cli.ParseException("Invalid argument specified for -hops option, must be one of [hops, runtime, recompile_hops, recompile_runtime, codegen, codegen_recompile]");
 			}
 		}
 
@@ -376,7 +378,7 @@ private static Options createCLIOptions() {
 		Option memOpt = OptionBuilder.withDescription("monitors and reports max memory consumption in CP; default off")
 			.create("mem");
 		Option explainOpt = OptionBuilder.withArgName("level")
-			.withDescription("explains plan levels; can be 'hops' / 'runtime'[default] / 'recompile_hops' / 'recompile_runtime'")
+			.withDescription("explains plan levels; can be 'hops' / 'runtime'[default] / 'recompile_hops' / 'recompile_runtime' / 'codegen' / 'codegen_recompile'")
 			.hasOptionalArg().create("explain");
 		Option execOpt = OptionBuilder.withArgName("mode")
 			.withDescription("sets execution mode; can be 'hadoop' / 'singlenode' / 'hybrid'[default] / 'HYBRID' / 'spark'")

src/main/java/org/apache/sysds/hops/codegen/SpoofCompiler.java

@@ -557,6 +557,11 @@ public static ArrayList<Hop> optimize(ArrayList<Hop> roots, boolean recompile)
 							LOG.info(CodegenUtils.printWithLineNumber(src_cuda));
 						}
 					}
+					if(DMLScript.EXPLAIN.isCodegenType()) {
+						System.out.print("JAVA Codegen EXPLAIN (generated code for HopID: " + cplan.getKey() +
+							", line m" + tmp.getValue().getBeginLine() + ", hash=" + tmp.getValue().hashCode() + "):");
+						System.out.println(CodegenUtils.printWithLineNumber(src));
+					}
 
 					//maintain plan cache
 					if( PLAN_CACHE_POLICY!=PlanCachePolicy.NONE )

src/main/java/org/apache/sysds/utils/Explain.java

@@ -86,13 +86,18 @@ public enum ExplainType {
 		HOPS,     // explain program and hops
 		RUNTIME,  // explain runtime program (default)
 		RECOMPILE_HOPS, // explain hops, incl recompile
-		RECOMPILE_RUNTIME;  // explain runtime program, incl recompile 
+		RECOMPILE_RUNTIME,  // explain runtime program, incl recompile
+		CODEGEN,	// show generated code, incl runtime explanation
+		CODEGEN_RECOMPILE;	// show generated code, incl runtime explanation and recompilation
 
 		public boolean isHopsType(boolean recompile) {
 			return (this==RECOMPILE_HOPS || (!recompile && this==HOPS));
 		}
 		public boolean isRuntimeType(boolean recompile) {
-			return (this==RECOMPILE_RUNTIME || (!recompile && this==RUNTIME));
+			return (this==RECOMPILE_RUNTIME || (!recompile && this==RUNTIME) || (this==CODEGEN_RECOMPILE) ||(!recompile && this==CODEGEN));
+		}
+		public boolean isCodegenType() {
+			return (this == CODEGEN || this == CODEGEN_RECOMPILE);
 		}
 	}
 
@@ -185,9 +190,11 @@ public static String explain(DMLProgram prog, Program rtprog, ExplainType type,
 			case HOPS:
 			case RECOMPILE_HOPS:
 				return explain(prog);
-			//explain runtime program	
+			//explain runtime program
 			case RUNTIME:
 			case RECOMPILE_RUNTIME:
+			case CODEGEN:
+			case CODEGEN_RECOMPILE:
 				return explain(rtprog, counts);
 			case NONE:
 				//do nothing

src/test/java/org/apache/sysds/test/functions/codegen/SumProductChainTest.java

@@ -114,8 +114,8 @@ private void testSumProductChain(String testname, boolean vectors, boolean spars
 
 			String HOME = SCRIPT_DIR + TEST_DIR;
 			fullDMLScriptName = HOME + testname + ".dml";
-			programArgs = new String[]{"hops", "-stats", 
-					"-args", input("X"), output("R") };
+			programArgs = new String[]{"-explain", "codegen",
+				"-stats", "-args", input("X"), output("R") };
 
 			fullRScriptName = HOME + testname + ".R";
 			rCmd = getRCmd(inputDir(), expectedDir());