GEODE-10231 : Add configuration for suppressing FunctionException logging

System property for suppressing FunctionException logging:
gemfire.logging.suppressFunctionExceptionLogging=true

Author: zsitole

geode-core/src/main/java/org/apache/geode/internal/cache/execute/AbstractExecution.java

@@ -56,6 +56,10 @@
 public abstract class AbstractExecution implements InternalExecution {
   private static final Logger logger = LogService.getLogger();
 
+  /** Whether to suppress logging of FunctionException */
+  public static final boolean SUPPRESS_FUNCTION_EXCEPTION_LOGGING =
+      Boolean.getBoolean(GeodeGlossary.GEMFIRE_PREFIX + "logging.suppressFunctionExceptionLogging");
+
   public static final int DEFAULT_CLIENT_FUNCTION_TIMEOUT = 0;
   private static final String CLIENT_FUNCTION_TIMEOUT_SYSTEM_PROPERTY =
       GeodeGlossary.GEMFIRE_PREFIX + "CLIENT_FUNCTION_TIMEOUT";
@@ -504,8 +508,16 @@ private void handleException(Throwable functionException, final Function fn,
         ((InternalResultSender) sender).setException(functionException);
       }
     } else {
-      logger.warn("Exception occurred on local node while executing Function:",
-          functionException);
+      if (AbstractExecution.SUPPRESS_FUNCTION_EXCEPTION_LOGGING
+          && functionException instanceof FunctionException) {
+        if (logger.isDebugEnabled()) {
+          logger.debug("Exception occurred on local node while executing Function:",
+              functionException);
+        }
+      } else {
+        logger.warn("Exception occurred on local node while executing Function:",
+            functionException);
+      }
     }
   }
 

geode-core/src/main/java/org/apache/geode/internal/cache/execute/DistributedRegionFunctionResultSender.java

@@ -224,8 +224,18 @@ public void setException(Throwable exception) {
     } else {
       ((LocalResultCollector) rc).setException(exception);
       // this.lastResult(exception);
-      logger.info("Unexpected exception during function execution on local node Distributed Region",
-          exception);
+      if (AbstractExecution.SUPPRESS_FUNCTION_EXCEPTION_LOGGING
+          && exception instanceof FunctionException) {
+        if (logger.isDebugEnabled()) {
+          logger.debug(
+              "Unexpected exception during function execution on local node Distributed Region",
+              exception);
+        }
+      } else {
+        logger.info(
+            "Unexpected exception during function execution on local node Distributed Region",
+            exception);
+      }
     }
     rc.endResults();
     localLastResultReceived = true;

geode-core/src/main/java/org/apache/geode/internal/cache/execute/MemberFunctionResultSender.java

@@ -232,8 +232,17 @@ public void sendException(Throwable exception) {
   public void setException(Throwable exception) {
     ((LocalResultCollector) rc).setException(exception);
     // this.lastResult(exception);
-    logger.info("Unexpected exception during function execution local member",
-        exception);
+    if (AbstractExecution.SUPPRESS_FUNCTION_EXCEPTION_LOGGING
+        && exception instanceof FunctionException) {
+      if (logger.isDebugEnabled()) {
+        logger.debug("Unexpected exception during function execution local member",
+            exception);
+      }
+    } else {
+      logger.info("Unexpected exception during function execution local member",
+          exception);
+    }
+
     rc.endResults();
     localLastResultReceived = true;
   }

geode-core/src/main/java/org/apache/geode/internal/cache/execute/PartitionedRegionFunctionResultSender.java

@@ -390,8 +390,18 @@ public void setException(Throwable exception) {
       serverSender.setException(exception);
     } else {
       ((LocalResultCollector) rc).setException(exception);
-      logger.info("Unexpected exception during function execution on local node Partitioned Region",
-          exception);
+      if (AbstractExecution.SUPPRESS_FUNCTION_EXCEPTION_LOGGING
+          && exception instanceof FunctionException) {
+        if (logger.isDebugEnabled()) {
+          logger.debug(
+              "Unexpected exception during function execution on local node Partitioned Region",
+              exception);
+        }
+      } else {
+        logger.info(
+            "Unexpected exception during function execution on local node Partitioned Region",
+            exception);
+      }
     }
     rc.endResults();
     localLastResultReceived = true;

geode-core/src/main/java/org/apache/geode/internal/cache/execute/ServerToClientFunctionResultSender.java

@@ -321,9 +321,17 @@ public synchronized void setException(Throwable exception) {
           }
           String exceptionMessage = exception.getMessage() != null ? exception.getMessage()
               : "Exception occurred during function execution";
-          logger.warn(String.format("Exception on server while executing function : %s",
-              fn),
-              exception);
+          if (AbstractExecution.SUPPRESS_FUNCTION_EXCEPTION_LOGGING
+              && exception instanceof FunctionException) {
+            if (logger.isDebugEnabled()) {
+              logger.debug(String.format("Exception on server while executing function : %s",
+                  fn), exception);
+            }
+          } else {
+            logger.warn(String.format("Exception on server while executing function : %s",
+                fn), exception);
+          }
+
           if (logger.isDebugEnabled()) {
             logger.debug("ServerToClientFunctionResultSender sending Function Exception : ");
           }

geode-core/src/main/java/org/apache/geode/internal/cache/properties.md

@@ -131,6 +131,7 @@ DistributionConfigImpl constructor prepends `gemfire.` to each valid attribute n
 | gemfire.EXPIRY_UNITS_MS | Boolean | `false` | See `org.apache.geode.internal.cache.LocalRegion#EXPIRY_MS_PROPERTY`.<p>Used by unit tests to set expiry to milliseconds instead of the default seconds. Used in ExpiryTask.|
 | gemfire.IDLE_THREAD_TIMEOUT | Integer | `30000 * 60` | See `org.apache.geode.distributed.internal.FunctionExecutionPooledExecutor` constructor.<p>See `org.apache.geode.internal.logging.CoreLoggingExecutors#getIdleThreadTimeoutMillis`<p>Units are in milliseconds.|
 | gemfire.locator-load-imbalance-threshold | Float | `10.0` | See `org.apache.geode.distributed.internal.LocatorLoadSnapshot#LOAD_IMBALANCE_THRESHOLD_PROPERTY_NAME`<p>Sets the connection count threshold for rebalancing clients.  When a client asks the locator whether it should switch to a less loaded server the locator will respond "no" if the connection-count gap between the highest-loaded server and the least-loaded server is within this threshold. If the threshold is reached the locator will aggressivley reassign clients until balance is re-established.|
+| gemfire.logging.suppressFunctionExceptionLogging | Boolean | `false` | See `org.apache.geode.internal.cache.execute.AbstractExecution#SUPPRESS_FUNCTION_EXCEPTION_LOGGING`.<p>Whether to suppress logging of FunctionException during function execution.|
 | gemfire.memoryEventTolerance | Integer | `0` | See `org.apache.geode.internal.cache.control.MemoryThresholds#memoryStateChangeTolerance`<p>Number of eviction or critical state changes that have to occur before the event is delivered.<p>The default is `0` so we will change states immediately by default.|
 | gemfire.MAX_PENDING_CANCELS | Integer | `10000` | See `org.apache.geode.internal.cache.ExpirationScheduler#MAX_PENDING_CANCELS`.|
 | gemfire.MAXIMUM_SHUTDOWN_PEEKS | Integer | `50` | See `org.apache.geode.internal.cache.tier.sockets.CacheClientProxy#MAXIMUM_SHUTDOWN_PEEKS`.<p>The number of times to peek on shutdown before giving up and shutting down.</p>|

geode-core/src/main/java/org/apache/geode/internal/cache/tier/sockets/command/ExecuteFunction70.java

@@ -431,6 +431,15 @@ private void executeFunctionLocally(final Function fn, final FunctionContext cx,
           if (logger.isDebugEnabled()) {
             logger.debug("Exception on server while executing function: {}", fn, e);
           }
+        } catch (FunctionException e) {
+          stats.endFunctionExecutionWithException(startExecution, fn.hasResult());
+          if (AbstractExecution.SUPPRESS_FUNCTION_EXCEPTION_LOGGING) {
+            if (logger.isDebugEnabled()) {
+              logger.debug("Exception on server while executing function: {}", fn, e);
+            }
+          } else {
+            logger.warn("Exception on server while executing function: {}", fn, e);
+          }
         } catch (Exception e) {
           stats.endFunctionExecutionWithException(startExecution, fn.hasResult());
           logger.warn("Exception on server while executing function: {}", fn, e);

geode-core/src/main/java/org/apache/geode/internal/cache/tier/sockets/command/ExecuteRegionFunction66.java

@@ -220,9 +220,17 @@ public void cmdExecute(final @NotNull Message clientMessage,
         resultSender.setException(fe);
       } else {
         if (setLastResultReceived(resultSender)) {
-          logger.warn(String.format("Exception on server while executing function : %s",
-              function),
-              fe);
+          if (AbstractExecution.SUPPRESS_FUNCTION_EXCEPTION_LOGGING) {
+            if (logger.isDebugEnabled()) {
+              logger.debug(String.format("Exception on server while executing function : %s",
+                  function),
+                  fe);
+            }
+          } else {
+            logger.warn(String.format("Exception on server while executing function : %s",
+                function),
+                fe);
+          }
           sendException(hasResult, clientMessage, message, serverConnection, fe);
         }
       }

geode-gfsh/src/main/java/org/apache/geode/management/internal/cli/functions/UserFunctionExecution.java

@@ -35,11 +35,13 @@
 import org.apache.geode.cache.execute.Execution;
 import org.apache.geode.cache.execute.Function;
 import org.apache.geode.cache.execute.FunctionContext;
+import org.apache.geode.cache.execute.FunctionException;
 import org.apache.geode.cache.execute.FunctionService;
 import org.apache.geode.cache.execute.ResultCollector;
 import org.apache.geode.cache.query.RegionNotFoundException;
 import org.apache.geode.distributed.DistributedMember;
 import org.apache.geode.internal.cache.InternalCache;
+import org.apache.geode.internal.cache.execute.AbstractExecution;
 import org.apache.geode.internal.cache.execute.InternalFunction;
 import org.apache.geode.internal.classloader.ClassPathLoader;
 import org.apache.geode.internal.security.SecurityService;
@@ -244,6 +246,17 @@ public void execute(FunctionContext<Object[]> context) {
               CliStrings.format(
                   CliStrings.EXECUTE_FUNCTION__MSG__RESULT_COLLECTOR_0_NOT_FOUND_ERROR_1,
                   resultCollectorName, e.getMessage())));
+    } catch (FunctionException e) {
+      if (AbstractExecution.SUPPRESS_FUNCTION_EXCEPTION_LOGGING) {
+        if (logger.isDebugEnabled()) {
+          logger.debug("error executing function " + functionId, e);
+        }
+      } else {
+        logger.error("error executing function " + functionId, e);
+      }
+
+      context.getResultSender().lastResult(
+          new CliFunctionResult(context.getMemberName(), ERROR, "Exception: " + e.getMessage()));
     } catch (Exception e) {
       logger.error("error executing function " + functionId, e);
       context.getResultSender().lastResult(