Move binary-model related functionality to DDK

Maksim Rubanov · Maksim Rubanov · commit 26329699e6bb · 2019-11-27T15:20:55.000+01:00
The corresponding methods from StandaloneBuilderState moved to
MonitoredClusteringBuilderState.

IBinaryModelStore is now made a part of xtext target platform and
therefore moved to DDK.
diff --git a/com.avaloq.tools.ddk.xtext.builder/src/com/avaloq/tools/ddk/xtext/builder/IBinaryModelStore.java b/com.avaloq.tools.ddk.xtext.builder/src/com/avaloq/tools/ddk/xtext/builder/IBinaryModelStore.java
@@ -0,0 +1,108 @@
+/*
+ * Copyright (c) Avaloq Licence AG
+ * Schwerzistrasse 6, 8807 Freienbach, Switzerland, http://www.avaloq.com
+ * All Rights Reserved.
+ *
+ * This software is the confidential and proprietary information of Avaloq Licence AG.
+ * You shall not disclose whole or parts of it and shall use it only in accordance with the terms of the
+ * licence agreement you entered into with Avaloq Licence AG.
+ */
+
+package com.avaloq.tools.ddk.xtext.builder;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.Map;
+
+import org.eclipse.emf.common.util.URI;
+
+import com.avaloq.tools.ddk.xtext.builder.IBinaryModelStore.NullBinaryModelStore;
+import com.google.inject.ImplementedBy;
+
+
+/**
+ * Manages the binary resources.
+ */
+@ImplementedBy(NullBinaryModelStore.class)
+public interface IBinaryModelStore {
+
+  String SIZE_OPTION = "SIZE"; //$NON-NLS-1$
+
+  /**
+   * Creates an input stream from which a binary model can be read.
+   *
+   * @param uri
+   *          binary model URI, must not be {@code null}
+   * @param options
+   *          a map of options to influence the kind of stream that is returned; can be {@code null} and unrecognized options are ignored
+   * @return input stream which can be used to read binary model
+   * @throws IOException
+   *           in case of an I/O error
+   */
+  InputStream createInputStream(URI uri, Map<?, ?> options) throws IOException;
+
+  /**
+   * Creates an output stream to which a binary model can be written.
+   *
+   * @param uri
+   *          binary model URI, must not be {@code null}
+   * @param options
+   *          a map of options to influence the kind of stream that is returned; can be {@code null} and unrecognized options are ignored
+   * @return output stream which can be used to write binary model
+   * @throws IOException
+   *           if a database error occurred
+   */
+  OutputStream createOutputStream(URI uri, Map<?, ?> options) throws IOException;
+
+  /**
+   * Checks whether a binary model exists with the given URI.
+   *
+   * @param uri
+   *          binary model URI, must not be {@code null}
+   * @return {@code true} if binary model with given URI exists
+   * @throws IOException
+   *           if a database error occurred
+   */
+  boolean exists(URI uri) throws IOException;
+
+  /**
+   * Deletes the binary model for the given URI.
+   *
+   * @param uri
+   *          binary model URI, must not be {@code null}
+   * @throws IOException
+   *           if I/O exception occurred
+   */
+  void delete(URI uri) throws IOException;
+
+  /**
+   * Default no-op implementation throwing runtime exceptions when calling {@link #createInputStream(URI)} or {@link #createOutputStream(URI)} as that means
+   * the application has not been configured correctly.
+   */
+  class NullBinaryModelStore implements IBinaryModelStore {
+
+    @Override
+    public InputStream createInputStream(final URI uri, final Map<?, ?> options) throws IOException {
+      throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public OutputStream createOutputStream(final URI uri, final Map<?, ?> options) throws IOException {
+      throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public boolean exists(final URI uri) throws IOException {
+      return false;
+    }
+
+    @Override
+    public void delete(final URI uri) throws IOException {
+      throw new UnsupportedOperationException();
+    }
+
+  }
+}
+
+/* Copyright (c) Avaloq Licence AG */
diff --git a/com.avaloq.tools.ddk.xtext.builder/src/com/avaloq/tools/ddk/xtext/builder/MonitoredClusteringBuilderState.java b/com.avaloq.tools.ddk.xtext.builder/src/com/avaloq/tools/ddk/xtext/builder/MonitoredClusteringBuilderState.java
@@ -17,6 +17,8 @@
 import java.util.Optional;
 import java.util.Queue;
 import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ForkJoinPool;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 import java.util.stream.Collectors;
@@ -38,6 +40,8 @@
 import org.eclipse.xtext.builder.impl.BuildData;
 import org.eclipse.xtext.builder.resourceloader.IResourceLoader;
 import org.eclipse.xtext.builder.resourceloader.IResourceLoader.LoadOperationException;
+import org.eclipse.xtext.generator.IFileSystemAccess;
+import org.eclipse.xtext.generator.IFileSystemAccessExtension3;
 import org.eclipse.xtext.resource.IEObjectDescription;
 import org.eclipse.xtext.resource.IReferenceDescription;
 import org.eclipse.xtext.resource.IResourceDescription;
@@ -49,6 +53,7 @@
 import org.eclipse.xtext.resource.impl.DefaultResourceDescriptionDelta;
 import org.eclipse.xtext.resource.impl.ResourceDescriptionChangeEvent;
 import org.eclipse.xtext.resource.impl.ResourceDescriptionsData;
+import org.eclipse.xtext.resource.persistence.IResourceStorageFacade;
 import org.eclipse.xtext.resource.persistence.StorageAwareResource;
 import org.eclipse.xtext.service.OperationCanceledManager;
 import org.eclipse.xtext.util.CancelIndicator;
@@ -79,6 +84,7 @@
 import com.avaloq.tools.ddk.xtext.resource.AbstractResourceDescriptionDelta;
 import com.avaloq.tools.ddk.xtext.resource.extensions.ForwardingResourceDescriptions;
 import com.avaloq.tools.ddk.xtext.resource.extensions.IResourceDescriptions2;
+import com.avaloq.tools.ddk.xtext.resource.persistence.DirectLinkingResourceStorageFacade;
 import com.avaloq.tools.ddk.xtext.resource.persistence.DirectLinkingSourceLevelURIsAdapter;
 import com.avaloq.tools.ddk.xtext.scoping.ImplicitReferencesAdapter;
 import com.avaloq.tools.ddk.xtext.tracing.ITraceSet;
@@ -162,6 +168,11 @@ public class MonitoredClusteringBuilderState extends ClusteringBuilderState
   @Inject
   private IResourceServiceProvider.Registry resourceServiceProviderRegistry;
 
+  @Inject(optional = true)
+  private IFileSystemAccess fileSystemAccess;
+
+  private final ForkJoinPool binaryStorageExecutor = new ForkJoinPool(4);
+
   /**
    * Handle to the ResourceDescriptionsData we use viewed as a IResourceDescriptions2 (with findReferences()). Parent class does not provide direct access to
    * its ResourceDescriptionsData, and anyway it wouldn't know about out new interfaces.
@@ -413,6 +424,7 @@ protected Collection<Delta> doUpdate(final BuildData buildData, final ResourceDe
     final Map<URI, DerivedObjectAssociations> oldDerivedObjectAssociations = saveOldDerivedObjectAssociations(buildData);
 
     buildData.getSourceLevelURICache().cacheAsSourceURIs(toBeDeleted);
+    deleteBinaryResources(toBeDeleted);
     installSourceLevelURIs(buildData);
 
     // Step 3: Create a queue; write new temporary resource descriptions for the added or updated resources
@@ -611,6 +623,7 @@ public boolean isCanceled() {
           if (resource instanceof XtextResource) {
             ((XtextResource) resource).getCache().clear(resource);
           }
+          storeBinaryResource(resource, buildData);
           traceSet.ended(ResourceProcessingEvent.class);
           buildData.getSourceLevelURICache().getSources().remove(changedURI);
           subProgress.worked(1);
@@ -643,6 +656,7 @@ public boolean isCanceled() {
       }
       traceSet.ended(BuildLinkingEvent.class);
       watchdog.interrupt();
+      awaitBinaryStorageExecutorQuiescence();
     }
     return allDeltas;
     // CHECKSTYLE:CHECK-ON NestedTryDepth
@@ -665,6 +679,70 @@ protected Resource addResource(final Resource resource, final ResourceSet resour
     }
   }
 
+  /**
+   * Stores the process resource as a binary if it doesn't contain syntax or linking errors.
+   *
+   * @param resource
+   *          resource to store, must not be {@code null}
+   * @param buildData
+   *          build data, must not be {@code null}
+   */
+  protected void storeBinaryResource(final Resource resource, final BuildData buildData) {
+    if (resource instanceof StorageAwareResource && ((StorageAwareResource) resource).getResourceStorageFacade() != null
+        && fileSystemAccess instanceof IFileSystemAccessExtension3) {
+      CompletableFuture.runAsync(() -> {
+        final long maxTaskExecutionNanos = TimeUnit.NANOSECONDS.convert(1, TimeUnit.SECONDS);
+
+        try {
+          long elapsed = System.nanoTime();
+
+          IResourceStorageFacade storageFacade = ((StorageAwareResource) resource).getResourceStorageFacade();
+          storageFacade.saveResource((StorageAwareResource) resource, (IFileSystemAccessExtension3) fileSystemAccess);
+          buildData.getSourceLevelURICache().getSources().remove(resource.getURI());
+
+          elapsed = System.nanoTime() - elapsed;
+          if (elapsed > maxTaskExecutionNanos) {
+            LOGGER.info("saving binary taking longer than expected (" + elapsed + " ns) : " + resource.getURI()); //$NON-NLS-1$ //$NON-NLS-2$
+          }
+          // CHECKSTYLE:OFF
+        } catch (Throwable ex) {
+          // CHECKSTYLE:ON
+          LOGGER.error("Failed to save binary for " + resource.getURI(), ex); //$NON-NLS-1$
+        }
+      }, binaryStorageExecutor);
+    }
+  }
+
+  /**
+   * Deletes the binary models for the given set of URIs.
+   *
+   * @param toBeDeleted
+   *          set of URIs, must not be {@code null}
+   */
+  protected void deleteBinaryResources(final Set<URI> toBeDeleted) {
+    if (fileSystemAccess == null) {
+      return;
+    }
+    for (URI uri : toBeDeleted) {
+      IResourceStorageFacade resourceStorageFacade = resourceServiceProviderRegistry.getResourceServiceProvider(uri).get(IResourceStorageFacade.class);
+      if (resourceStorageFacade instanceof DirectLinkingResourceStorageFacade) {
+        ((DirectLinkingResourceStorageFacade) resourceStorageFacade).deleteStorage(uri, fileSystemAccess);
+      }
+    }
+  }
+
+  /**
+   * Waits until binary models are stored.
+   */
+  protected void awaitBinaryStorageExecutorQuiescence() {
+    int activeThreadCount = binaryStorageExecutor.getActiveThreadCount();
+    long queuedTaskCount = binaryStorageExecutor.getQueuedTaskCount();
+    if (!binaryStorageExecutor.awaitQuiescence(1, TimeUnit.MINUTES)) {
+      LOGGER.warn(String.format("Binary resource storage tasks not completed in time, start with task / queue %d / %d; now have %d / %d", //$NON-NLS-1$
+          activeThreadCount, queuedTaskCount, binaryStorageExecutor.getActiveThreadCount(), binaryStorageExecutor.getQueuedTaskCount()));
+    }
+  }
+
   /**
    * Log the first and last 10 StackOverflowError Stack Trace.
    *
@@ -840,7 +918,9 @@ protected boolean recordDeltaAsNew(final Delta newDelta) {
    *          The progress monitor used for user feedback
    * @return the list of {@link URI}s of loaded resources to be processed in the second phase
    */
-  private List<URI> writeResources(final Collection<URI> toWrite, final BuildData buildData, final IResourceDescriptions oldState, final CurrentDescriptions newState, final IProgressMonitor monitor) { // NOPMD NPath Complexity
+  private List<URI> writeResources(final Collection<URI> toWrite, final BuildData buildData, final IResourceDescriptions oldState, final CurrentDescriptions newState, final IProgressMonitor monitor) { // NOPMD
+                                                                                                                                                                                                         // NPath
+                                                                                                                                                                                                         // Complexity
     ResourceSet resourceSet = buildData.getResourceSet();
     IProject currentProject = getBuiltProject(buildData);
     List<URI> toBuild = Lists.newLinkedList();
@@ -884,7 +964,7 @@ private List<URI> writeResources(final Collection<URI> toWrite, final BuildData
           if (uri == null && ex instanceof LoadOperationException) { // NOPMD
             uri = ((LoadOperationException) ex).getUri();
           }
-          LOGGER.error(NLS.bind(Messages.MonitoredClusteringBuilderState_CANNOT_LOAD_RESOURCE, uri != null ? uri: "unknown uri"), ex); //$NON-NLS-1$
+          LOGGER.error(NLS.bind(Messages.MonitoredClusteringBuilderState_CANNOT_LOAD_RESOURCE, uri != null ? uri : "unknown uri"), ex); //$NON-NLS-1$
           if (resource != null) {
             resourceSet.getResources().remove(resource);
           }
@@ -970,6 +1050,10 @@ protected void writeNewResourceDescriptions(final BuildData buildData, final IRe
    */
   @Override
   protected void clearResourceSet(final ResourceSet resourceSet) {
+    // this is important as otherwise the resources would unexpectedly become detached from the resource set
+    while (!binaryStorageExecutor.awaitQuiescence(1, TimeUnit.SECONDS)) {
+      LOGGER.warn("Waiting for binary resource storage tasks to complete."); //$NON-NLS-1$
+    }
     traceSet.started(BuildResourceSetClearEvent.class, resourceSet.getResources().size());
     try {
       EmfResourceSetUtil.clearResourceSetWithoutNotifications(resourceSet);
diff --git a/com.avaloq.tools.ddk.xtext.builder/src/com/avaloq/tools/ddk/xtext/builder/layered/DefaultXtextTargetPlatform.java b/com.avaloq.tools.ddk.xtext.builder/src/com/avaloq/tools/ddk/xtext/builder/layered/DefaultXtextTargetPlatform.java
@@ -12,16 +12,17 @@
 
 import java.io.IOException;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.Map;
 
 import org.eclipse.core.runtime.IProgressMonitor;
 import org.eclipse.xtext.builder.builderState.PersistedStateProvider;
 import org.eclipse.xtext.resource.impl.ResourceDescriptionsData;
 
+import com.avaloq.tools.ddk.xtext.builder.IBinaryModelStore;
 import com.avaloq.tools.ddk.xtext.builder.IDerivedObjectAssociationsStore;
 import com.avaloq.tools.ddk.xtext.extensions.DelegatingResourceDescriptionsData;
 import com.avaloq.tools.ddk.xtext.extensions.IResourceDescriptionsData;
-import com.google.common.collect.ImmutableMap;
 import com.google.inject.Inject;
 
 
@@ -73,12 +74,17 @@ public IDerivedObjectAssociationsStore getAssociationsStore() {
   /** {@inheritDoc} */
   @Override
   public Map<String, String> getMetadata(final Collection<String> keys, final IProgressMonitor monitor) {
-    return ImmutableMap.of();
+    return Collections.emptyMap();
   }
 
   /** {@inheritDoc} */
   @Override
   public void setMetadata(final Map<String, String> options) {
     // nothing to do
   }
+
+  @Override
+  public IBinaryModelStore getBinaryModelStore() {
+    return null;
+  }
 }
diff --git a/com.avaloq.tools.ddk.xtext.builder/src/com/avaloq/tools/ddk/xtext/builder/layered/IXtextTargetPlatform.java b/com.avaloq.tools.ddk.xtext.builder/src/com/avaloq/tools/ddk/xtext/builder/layered/IXtextTargetPlatform.java
@@ -16,6 +16,7 @@
 
 import org.eclipse.core.runtime.IProgressMonitor;
 
+import com.avaloq.tools.ddk.xtext.builder.IBinaryModelStore;
 import com.avaloq.tools.ddk.xtext.builder.IDerivedObjectAssociationsStore;
 import com.avaloq.tools.ddk.xtext.extensions.IResourceDescriptionsData;
 import com.google.inject.ImplementedBy;
@@ -109,6 +110,13 @@ public interface IXtextTargetPlatform {
    */
   IDerivedObjectAssociationsStore getAssociationsStore();
 
+  /**
+   * Returns the store of binary resources.
+   *
+   * @return the platform's store, can be {@code null} if not supported
+   */
+  IBinaryModelStore getBinaryModelStore();
+
   /**
    * Platforms are supposed to be closed when no longer in use. At any time, there must be only one open platform.
    * The platform manager should ensure that there are never two platforms open at the same time.
diff --git a/com.avaloq.tools.ddk.xtext.builder/src/com/avaloq/tools/ddk/xtext/builder/layered/NullXtextTargetPlatform.java b/com.avaloq.tools.ddk.xtext.builder/src/com/avaloq/tools/ddk/xtext/builder/layered/NullXtextTargetPlatform.java
@@ -15,6 +15,7 @@
 
 import org.eclipse.core.runtime.IProgressMonitor;
 
+import com.avaloq.tools.ddk.xtext.builder.IBinaryModelStore;
 import com.avaloq.tools.ddk.xtext.builder.IDerivedObjectAssociationsStore;
 import com.avaloq.tools.ddk.xtext.extensions.IResourceDescriptionsData;
 
@@ -53,6 +54,11 @@ public IDerivedObjectAssociationsStore getAssociationsStore() {
     return null;
   }
 
+  @Override
+  public IBinaryModelStore getBinaryModelStore() {
+    return null;
+  }
+
   /** {@inheritDoc} */
   @Override
   public Map<String, String> getMetadata(final Collection<String> keys, final IProgressMonitor monitor) {
@@ -64,4 +70,5 @@ public Map<String, String> getMetadata(final Collection<String> keys, final IPro
   public void setMetadata(final Map<String, String> options) {
     // Nothing to do.
   }
+
 }
