Merge pull request #232 from mrubanov/master

Revert the demand load memory check

Author: mrubanov

com.avaloq.tools.ddk.xtext.builder/src/com/avaloq/tools/ddk/xtext/builder/DefaultBuilderResourceLoadStrategy.java

@@ -18,7 +18,7 @@
 
 
 /**
- * Determine when to stop loading resources during the build.
+ * Determine when to stop loading resources during build.
  */
 @SuppressWarnings("nls")
 // CHECKSTYLE:CONSTANTS-OFF
@@ -27,10 +27,10 @@ public class DefaultBuilderResourceLoadStrategy implements IBuilderResourceLoadS
   /** Class-wide logger. */
   private static final Logger LOGGER = Logger.getLogger(DefaultBuilderResourceLoadStrategy.class);
 
-  /** Property for setting target free memory value. (in megabytes). */
+  /** Default value for target free memory in megabytes. */
   private static final String TARGET_FREE_MEMORY_PROPERTY = "com.avaloq.tools.ddk.xtext.builder.targetFreeMemory"; //$NON-NLS-1$
 
-  /** Property for setting minimum free memory value. (in megabytes) */
+  /** Property for setting minimum free memory setting. */
   private static final String MIN_FREE_MEMORY_PROPERTY = "com.avaloq.tools.ddk.xtext.builder.minFreeMemory"; //$NON-NLS-1$
 
   /** Property for setting the maximal cluster size. */
@@ -51,16 +51,16 @@ public class DefaultBuilderResourceLoadStrategy implements IBuilderResourceLoadS
   /** Default value for MINIMUM_FREE_MEMORY. */
   private static final long DEFAULT_MIN_FREE_MEMORY = 100 * ONE_MEGABYTE;
 
-  /** Cluster will always end if memory falls below this threshold. */
+  /** Cluster will always end if memory falls bellow this threshold. */
   private static final long MINIMUM_FREE_MEMORY = getMinimumFreeMemory();
 
-  /** Attempt to cap a cluster when memory reaches this limit, as long as we processed more than the minimum cluster size. */
+  /** Attempt to cap cluster when memory reaches this limit, as long as we processed more than the minimum cluster size. */
   private static final long TARGET_FREE_MEMORY = getTargetFreeMemory();
 
   /** Cluster size. */
   @Inject(optional = true)
   @Named("org.eclipse.xtext.builder.clustering.ClusteringBuilderState.clusterSize")
-  private int clusterSize = 50; // NOPMD Cannot be static because of the way Guice injection works.
+  private int clusterSize = 20; // NOPMD Cannot be static because of the way Guice injection works.
 
   /** The Java runtime; needed to get access to memory usage data. */
   private static final Runtime RUNTIME = Runtime.getRuntime();
@@ -100,6 +100,7 @@ public void setClusterSize(final int clusterSize) {
     this.clusterSize = clusterSize;
   }
 
+  /** {@inheritDoc} */
   @Override
   public boolean mayProcessAnotherResource(final ResourceSet resourceSet, final int alreadyProcessed) {
     if (alreadyProcessed == 0) {
@@ -114,13 +115,13 @@ public boolean mayProcessAnotherResource(final ResourceSet resourceSet, final in
       return false;
     }
 
-    if (isMemoryLimitReached()) {
+    final long freeMemory = RUNTIME.freeMemory();
+    if (freeMemory < MINIMUM_FREE_MEMORY) {
       return false;
     } else if (alreadyProcessed < clusterSize) {
       return true; // Process at least up to cluster size
     }
 
-    final long freeMemory = RUNTIME.freeMemory();
     if (freeMemory < TARGET_FREE_MEMORY) {
       // Running GC here might free memory, but would not significantly speed up processing.
       // We may use slightly smaller clusters than strictly necessary without GC, though.
@@ -133,10 +134,4 @@ public boolean mayProcessAnotherResource(final ResourceSet resourceSet, final in
     }
     return true; // Keep on loading resources into this resource set
   }
-
-  @Override
-  public boolean isMemoryLimitReached() {
-    return RUNTIME.freeMemory() < MINIMUM_FREE_MEMORY;
-  }
-
 }

com.avaloq.tools.ddk.xtext.builder/src/com/avaloq/tools/ddk/xtext/builder/DemandLoadFailedException.java

@@ -22,7 +22,7 @@
 public interface IBuilderResourceLoadStrategy {
 
   /**
-   * Determine whether the builder may process yet another resource.
+   * Determine whether the builder may, in its second phase, process yet another resource.
    *
    * @param resourceSet
    *          The builder's resource set, containing all currently loaded resources.
@@ -32,11 +32,4 @@ public interface IBuilderResourceLoadStrategy {
    */
   boolean mayProcessAnotherResource(ResourceSet resourceSet, int alreadyProcessed);
 
-  /**
-   * Checks whether the builder has reached the minimal amount of free memory, which prevents further progress without clearing the resource set.
-   *
-   * @return {@code true} if the described condition is met, {@code false} otherwise
-   */
-  boolean isMemoryLimitReached();
-
 }

com.avaloq.tools.ddk.xtext.builder/src/com/avaloq/tools/ddk/xtext/builder/IBuilderResourceLoadStrategy.java

@@ -15,9 +15,7 @@
 
 //CHECKSTYLE:OFF
 public class Messages extends NLS {
-
   private static final String BUNDLE_NAME = "com.avaloq.tools.ddk.xtext.builder.messages"; //$NON-NLS-1$
-
   public static String MonitoredClusteringBuilderState_CANNOT_LOAD_RESOURCE;
   public static String MonitoredClusteringBuilderState_NO_MORE_RESOURCES;
   public static String MonitoredClusteringBuilderState_PHASE_ONE_DONE;

com.avaloq.tools.ddk.xtext.builder/src/com/avaloq/tools/ddk/xtext/builder/Messages.java

@@ -852,14 +852,11 @@ protected boolean recordDeltaAsNew(final Delta newDelta) {
    *          The progress monitor used for user feedback
    * @return the list of {@link URI}s of loaded resources to be processed in the second phase
    */
-  private List<URI> writeResources(final Collection<URI> toWrite, final BuildData buildData, final IResourceDescriptions oldState, final CurrentDescriptions newState, final IProgressMonitor monitor) { // NOPMD
+  private List<URI> writeResources(final Collection<URI> toWrite, final BuildData buildData, final IResourceDescriptions oldState, final CurrentDescriptions newState, final IProgressMonitor monitor) {
     ResourceSet resourceSet = buildData.getResourceSet();
     IProject currentProject = getBuiltProject(buildData);
-    List<URI> toBuild = Lists.newArrayList();
-
+    List<URI> toBuild = Lists.newLinkedList();
     IResourceLoader.LoadOperation loadOperation = null;
-    URI resourceToRetry = null;
-
     try {
       int resourcesToWriteSize = toWrite.size();
       int index = 1;
@@ -869,23 +866,16 @@ private List<URI> writeResources(final Collection<URI> toWrite, final BuildData
 
       // Not using the loadingStrategy here; seems to work fine with a reasonable clusterSize (20 by default), even with
       // large resources and "scarce" memory (say, about 500MB).
-      while (loadOperation.hasNext() || resourceToRetry != null) {
+      while (loadOperation.hasNext()) {
         if (monitor.isCanceled()) {
           loadOperation.cancel();
           throw new OperationCanceledException();
         }
-
         URI uri = null;
         Resource resource = null;
         try {
-          if (resourceToRetry == null) {
-            resource = addResource(loadOperation.next().getResource(), resourceSet);
-          } else {
-            resource = resourceSet.getResource(resourceToRetry, true);
-            resourceToRetry = null;
-          }
+          resource = addResource(loadOperation.next().getResource(), resourceSet);
           uri = resource.getURI();
-
           final Object[] bindings = {Integer.valueOf(index), Integer.valueOf(resourcesToWriteSize), uri.fileExtension(), URI.decode(uri.lastSegment())};
           monitor.subTask(NLS.bind(Messages.MonitoredClusteringBuilderState_WRITE_ONE_DESCRIPTION, bindings));
           traceSet.started(ResourceIndexingEvent.class, uri);
@@ -901,11 +891,6 @@ private List<URI> writeResources(final Collection<URI> toWrite, final BuildData
             newState.register(intermediateDelta);
             toBuild.add(uri);
           }
-        } catch (final DemandLoadFailedException e) {
-          // Demand load failed because of memory shortage, save the uri to process again in the next cluster
-          // Resource set will be cleared at the end of this loop iteration
-          resourceToRetry = uri;
-          LOGGER.info("Demand load failed during resource indexing: " + resourceToRetry); //$NON-NLS-1$
         } catch (final WrappedException ex) {
           pollForCancellation(monitor);
           if (uri == null && ex instanceof LoadOperationException) { // NOPMD
@@ -940,13 +925,10 @@ private List<URI> writeResources(final Collection<URI> toWrite, final BuildData
           monitor.worked(1);
         }
 
-        if (!loadingStrategy.mayProcessAnotherResource(resourceSet, resourceSet.getResources().size()) || resourceToRetry != null) {
+        if (!loadingStrategy.mayProcessAnotherResource(resourceSet, resourceSet.getResources().size())) {
           clearResourceSet(resourceSet);
         }
-
-        if (resourceToRetry == null) {
-          index++;
-        }
+        index++;
       }
     } finally {
       if (loadOperation != null) {
@@ -956,32 +938,17 @@ private List<URI> writeResources(final Collection<URI> toWrite, final BuildData
     return toBuild;
   }
 
-  /**
-   * Writes resources descriptions to the database.
-   *
-   * @param buildData
-   *          builder's data, must not be {@code null}
-   * @param oldState
-   *          represents old index state, must not be {@code null}
-   * @param newState
-   *          represents the new index state, must not be {@code null}
-   * @param newData
-   *          new resource descriptions, must not be {@code null}
-   * @param monitor
-   *          progress monitor, must not be {@code null}
-   */
+  /** {@inheritDoc} */
   protected void writeNewResourceDescriptions(final BuildData buildData, final IResourceDescriptions oldState, final CurrentDescriptions newState, final ResourceDescriptionsData newData, final IProgressMonitor monitor) {
     final List<List<URI>> toWriteGroups = phaseOneBuildSorter.sort(buildData.getToBeUpdated(), oldState);
+    final List<URI> toBuild = Lists.newLinkedList();
     ResourceSet resourceSet = buildData.getResourceSet();
     BuildPhases.setIndexing(resourceSet, true);
-
     int totalSize = 0;
     for (List<URI> group : toWriteGroups) {
       totalSize = totalSize + group.size();
     }
-
     final SubMonitor subMonitor = SubMonitor.convert(monitor, Messages.MonitoredClusteringBuilderState_WRITE_DESCRIPTIONS, totalSize);
-    final List<URI> toBuild = Lists.newArrayListWithCapacity(totalSize);
 
     try {
       traceSet.started(BuildIndexingEvent.class);