closes #66 ChainLP: Storage support: DB

anshooarora · anshooarora · commit 6d7303f6de0b · 2025-02-03T19:18:16.000-05:00
diff --git a/Config.md b/Config.md
@@ -21,6 +21,7 @@ chaintest.generator.chainlp.host.url=http://localhost/
 chaintest.generator.chainlp.client.request-timeout-s=30
 chaintest.generator.chainlp.client.expect-continue=false
 chaintest.generator.chainlp.client.max-retries=3
+chaintest.generator.chainlp.persist-embeds=true
 
 ## simple
 chaintest.generator.simple.enabled=true
diff --git a/chainlp/backend/chainlp/src/main/java/com/aventstack/chainlp/ChainApp.java b/chainlp/backend/chainlp/src/main/java/com/aventstack/chainlp/ChainApp.java
@@ -2,7 +2,9 @@
 
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.cache.annotation.EnableCaching;
 
+@EnableCaching
 @SpringBootApplication
 public class ChainApp {
 
diff --git a/chainlp/backend/chainlp/src/main/java/com/aventstack/chainlp/api/test/Embed.java b/chainlp/backend/chainlp/src/main/java/com/aventstack/chainlp/api/test/Embed.java
@@ -6,6 +6,7 @@
 import jakarta.persistence.GeneratedValue;
 import jakarta.persistence.GenerationType;
 import jakarta.persistence.Id;
+import jakarta.persistence.Lob;
 import jakarta.persistence.ManyToOne;
 import jakarta.persistence.Table;
 import jakarta.persistence.Transient;
@@ -35,7 +36,17 @@ public class Embed {
     @Column
     private String url;
 
+    @Column(columnDefinition = "longblob")
+    @Lob
+    private byte[] bytes;
+
+    @Column
+    private String base64;
+
     @Transient
     private String presigned;
 
+    @Transient
+    private boolean store;
+
 }
diff --git a/chainlp/backend/chainlp/src/main/java/com/aventstack/chainlp/api/test/Test.java b/chainlp/backend/chainlp/src/main/java/com/aventstack/chainlp/api/test/Test.java
@@ -22,6 +22,7 @@
 import lombok.ToString;
 import org.springframework.data.annotation.CreatedDate;
 
+import java.util.Base64;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
@@ -31,7 +32,7 @@
 @Builder
 @NoArgsConstructor
 @AllArgsConstructor
-@ToString(exclude = { "parent" })
+@ToString(exclude = { "parent", "embeds" })
 @Table(name = "test")
 public class Test {
 
@@ -125,7 +126,19 @@ public void setChildren(List<Test> children) {
     public void setEmbeds(final List<Embed> embeds) {
         this.embeds = embeds;
         if (embeds != null) {
-            embeds.forEach(x -> x.setTest(this));
+            for (final Embed embed : embeds) {
+                embed.setTest(this);
+                if (embed.isStore()) {
+                    if (null != embed.getBase64() && !embed.getBase64().isEmpty()) {
+                        final byte[] data = Base64.getDecoder().decode(embed.getBase64().getBytes());
+                        embed.setBytes(data);
+                        embed.setBase64(null);
+                    }
+                } else {
+                    embed.setBase64(null);
+                    embed.setBytes(null);
+                }
+            }
         }
     }
 
diff --git a/chainlp/backend/chainlp/src/main/java/com/aventstack/chainlp/api/test/TestService.java b/chainlp/backend/chainlp/src/main/java/com/aventstack/chainlp/api/test/TestService.java
@@ -63,10 +63,12 @@ public Page<Test> findAll(final Test test, final String op, final Pageable pagea
     }
 
     private Test resolveEmbeds(final Test test) {
-        for (final Embed embed : test.getEmbeds()) {
-            final String presigned = embedResolver.getResolver(embed.getUrl())
-                    .resolve(embed.getUrl());
-            embed.setPresigned(presigned);
+        if (test.getEmbeds().stream().anyMatch(x -> null != x.getUrl() && !x.getUrl().isBlank())) {
+            for (final Embed embed : test.getEmbeds()) {
+                final String presigned = embedResolver.getResolver(embed.getUrl())
+                        .resolve(embed.getUrl());
+                embed.setPresigned(presigned);
+            }
         }
         test.getChildren().forEach(this::resolveEmbeds);
         return test;
diff --git a/chainlp/backend/chainlp/src/main/java/com/aventstack/chainlp/embed/AwsS3SignedUrlResolver.java b/chainlp/backend/chainlp/src/main/java/com/aventstack/chainlp/embed/AwsS3SignedUrlResolver.java
@@ -40,7 +40,7 @@ public String createPresignedGetUrl(final String bucket, final String key) {
                     .key(key)
                     .build();
             final GetObjectPresignRequest presignRequest = GetObjectPresignRequest.builder()
-                    .signatureDuration(Duration.ofMinutes(10))
+                    .signatureDuration(Duration.ofDays(1))
                     .getObjectRequest(objectRequest)
                     .build();
             final PresignedGetObjectRequest presignedRequest = presigner.presignGetObject(presignRequest);
diff --git a/chainlp/frontend/chainlp/src/app/components/test/test.component.html b/chainlp/frontend/chainlp/src/app/components/test/test.component.html
@@ -60,12 +60,18 @@ <h5 class="mb-3 testname" [ngClass]="test.result.toLowerCase()">{{test.name}}</h
                 <pre>{{leaf.error}}</pre>
               </div>
               <div class="mt-2 screenshots" *ngIf="leaf.embeds && leaf.embeds.length">
-                <img *ngFor="let s of leaf.embeds" [src]="s.presigned || s.url" class="img-fluid" alt="screenshot" />
+                <ng-container *ngFor="let s of leaf.embeds">
+                  <img *ngIf="s.presigned || s.url" [src]="s.presigned || s.url" class="img-fluid" alt="screenshot" />
+                  <img *ngIf="s.bytes || s.base64" [src]="'data:image/png;base64,' + s.bytes || s.base64" class="img-fluid" alt="screenshot" />
+                </ng-container>
               </div>
             </div>
           </div>
           <div class="mt-2 screenshots" *ngIf="child.embeds && child.embeds.length">
-            <img *ngFor="let s of child.embeds" [src]="s.presigned || s.url" class="img-fluid" alt="screenshot" />
+            <ng-container *ngFor="let s of child.embeds">
+              <img *ngIf="s.presigned || s.url" [src]="s.presigned || s.url" class="img-fluid" alt="screenshot" />
+              <img *ngIf="s.bytes || s.base64" [src]="'data:image/png;base64,' + s.bytes || s.base64" class="img-fluid" alt="screenshot" />
+            </ng-container>
           </div>
         </div>
       </div>
diff --git a/chainlp/frontend/chainlp/src/app/model/embed.model.ts b/chainlp/frontend/chainlp/src/app/model/embed.model.ts
@@ -1,4 +1,6 @@
 export class Embed {
   presigned: string;
   url: string;
+  bytes: any;
+  base64: string;
 }
diff --git a/chainlp/frontend/chainlp/src/app/pages/build/build/build.component.html b/chainlp/frontend/chainlp/src/app/pages/build/build/build.component.html
@@ -220,7 +220,10 @@ <h5 class="mb-3 testname" [ngClass]="test.result.toLowerCase()">{{test.name}}</h
                         <pre>{{leaf.error}}</pre>
                       </div>
                       <div class="mt-2 mb-4 screenshots" *ngIf="leaf.embeds && leaf.embeds.length">
-                        <img *ngFor="let s of leaf.embeds" [src]="s.presigned || s.url" class="img-fluid" alt="screenshot" />
+                        <ng-container *ngFor="let s of leaf.embeds">
+                          <img *ngIf="s.presigned || s.url" [src]="s.presigned || s.url" class="img-fluid" alt="screenshot" />
+                          <img *ngIf="s.bytes || s.base64" [src]="'data:image/png;base64,' + s.bytes || s.base64" class="img-fluid" alt="screenshot" />
+                        </ng-container>
                       </div>
                     </div>
                   </div>
@@ -231,7 +234,10 @@ <h5 class="mb-3 testname" [ngClass]="test.result.toLowerCase()">{{test.name}}</h
                     <pre>{{child.error}}</pre>
                   </div>
                   <div class="mt-2 screenshots" *ngIf="child.embeds && child.embeds.length">
-                    <img *ngFor="let s of child.embeds" [src]="s.presigned || s.url" class="img-fluid" alt="screenshot" />
+                    <ng-container *ngFor="let s of child.embeds">
+                      <img *ngIf="s.presigned || s.url" [src]="s.presigned || s.url" class="img-fluid" alt="screenshot" />
+                      <img *ngIf="s.bytes || s.base64" [src]="'data:image/png;base64,' + s.bytes || s.base64" class="img-fluid" alt="screenshot" />
+                    </ng-container>
                   </div>
                 </div>
               </div>
diff --git a/core/chaintest-core-java/src/main/java/com/aventstack/chaintest/domain/Embed.java b/core/chaintest-core-java/src/main/java/com/aventstack/chaintest/domain/Embed.java
@@ -1,7 +1,9 @@
 package com.aventstack.chaintest.domain;
 
-import lombok.Data;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import lombok.Getter;
 import lombok.NoArgsConstructor;
+import lombok.Setter;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -13,8 +15,10 @@
 import java.util.List;
 import java.util.UUID;
 
-@Data
+@Getter
+@Setter
 @NoArgsConstructor
+@JsonIgnoreProperties(ignoreUnknown = true)
 public class Embed {
 
     private static final Logger log = LoggerFactory.getLogger(Embed.class);
@@ -30,10 +34,12 @@ public class Embed {
     );
 
     private final UUID uuid = UUID.randomUUID();
+    private String url;
     private String base64;
     private File file;
     private byte[] bytes;
     private String mediaType;
+    private boolean store = true;
 
     public Embed(final String base64, final String mediaType) {
         this.base64 = base64;
diff --git a/core/chaintest-core-java/src/main/java/com/aventstack/chaintest/domain/Test.java b/core/chaintest-core-java/src/main/java/com/aventstack/chaintest/domain/Test.java
@@ -4,7 +4,6 @@
 import com.aventstack.chaintest.util.TimeUtil;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
-import com.fasterxml.jackson.annotation.JsonProperty;
 import lombok.Getter;
 import lombok.Setter;
 
@@ -28,9 +27,9 @@
  * class name, start and end times, result, tags, error details, and child tests.
  * It also provides methods to complete the test and update its statistics.
  */
-@JsonIgnoreProperties(ignoreUnknown = true)
 @Getter
 @Setter
+@JsonIgnoreProperties(ignoreUnknown = true)
 public class Test implements ChainTestEntity {
 
     /**
@@ -116,7 +115,6 @@ public class Test implements ChainTestEntity {
     /**
      * The embedded media associated with the test.
      */
-    @JsonIgnore
     private final List<Embed> embeds = new ArrayList<>();
 
     /**
@@ -132,9 +130,6 @@ public class Test implements ChainTestEntity {
     @JsonIgnore
     private Test parent;
 
-    @JsonProperty("embeds")
-    private final List<BasicEmbed> basicEmbeds = new ArrayList<>();
-
     /**
      * Default constructor.
      */
@@ -453,12 +448,4 @@ public void addEmbed(final Embed embed) {
         embeds.add(embed);
     }
 
-    public void addScreenshotURL(final String url) {
-        addBasicEmbed(url);
-    }
-
-    public void addBasicEmbed(final String url) {
-        basicEmbeds.add(new BasicEmbed(url));
-    }
-
 }
diff --git a/core/chaintest-core-java/src/main/java/com/aventstack/chaintest/generator/ChainLPGenerator.java b/core/chaintest-core-java/src/main/java/com/aventstack/chaintest/generator/ChainLPGenerator.java
@@ -23,6 +23,7 @@ public class ChainLPGenerator implements Generator {
     private static final Logger log = LoggerFactory.getLogger(ChainLPGenerator.class);
     private static final String NAME = "chainlp";
     private static final String HTTP_CLIENT_ENABLED = "chaintest.generator.chainlp.enabled";
+    private static final String SEND_EMBEDS = "chaintest.generator.chainlp.persist-embeds";
     private static final ConcurrentHashMap<String, WrappedResponseAsync<Test>> _wrappedResponses = new ConcurrentHashMap<>();
     private static final AtomicBoolean CALLBACK_INVOKED = new AtomicBoolean();
 
@@ -32,6 +33,7 @@ public class ChainLPGenerator implements Generator {
     private ChainTestApiClient _client;
     private Build _build;
     private Queue<Test> _tests;
+    private boolean _sendEmbeds = true;
 
     public ChainLPGenerator(final String testRunner) {
         _testRunner = testRunner;
@@ -101,6 +103,11 @@ public void start(final Optional<Map<String, String>> config, final String testR
                 _build.setProjectId(buildReq.getProjectId());
                 CALLBACK_INVOKED.set(true);
                 log.debug("All tests in this run will be associated with buildId: {}", _build.getId());
+
+                final String sendEmbeds = config.get().get(SEND_EMBEDS);
+                if (null != sendEmbeds && !sendEmbeds.isEmpty()) {
+                    _sendEmbeds = Boolean.parseBoolean(sendEmbeds);
+                }
             }
         } catch (final IOException | InterruptedException e) {
             log.debug("Failed to send Build. PluginService will shutdown and " +
@@ -149,6 +156,10 @@ public void afterTest(final Test test, final Optional<Throwable> throwable) {
 
         updateAttributes(test);
 
+        if (!_sendEmbeds) {
+            setEmbedStorableState(test);
+        }
+
         final WrappedResponseAsync<Test> wrapper = new WrappedResponseAsync<>(test);
         _wrappedResponses.put(test.getClientId().toString(), wrapper);
         try {
@@ -159,6 +170,11 @@ public void afterTest(final Test test, final Optional<Throwable> throwable) {
         }
     }
 
+    private void setEmbedStorableState(final Test test) {
+        test.getEmbeds().forEach(embed -> embed.setStore(false));
+        test.getChildren().forEach(this::setEmbedStorableState);
+    }
+
     private void updateAttributes(final Test test) {
         test.setBuildId(_build.getId());
         test.setProjectId(_build.getProjectId());
diff --git a/core/chaintest-core-java/src/main/java/com/aventstack/chaintest/storage/AWSS3Client.java b/core/chaintest-core-java/src/main/java/com/aventstack/chaintest/storage/AWSS3Client.java
@@ -81,45 +81,46 @@ private void createBucket(final String bucketName) {
     }
 
     @Override
-    public void upload(final Test test, final String key, final byte[] data) {
+    public String upload(final Test test, final String key, final byte[] data) {
         final String prefixKey = getPrefixKey(_prefix, key);
         _client.putObject(PutObjectRequest.builder().bucket(_bucket).key(prefixKey)
                         .build(),
                 RequestBody.fromBytes(data));
-        assignURL(test, prefixKey);
+        return getUrl(test, prefixKey);
     }
 
-    private void assignURL(final Test test, final String key) {
-        test.addScreenshotURL(
-                _client.utilities().getUrl(builder -> builder.bucket(_bucket).key(key)).toExternalForm());
+    private String getUrl(final Test test, final String key) {
+        return _client.utilities().getUrl(builder -> builder.bucket(_bucket).key(key)).toExternalForm();
     }
 
     @Override
-    public void upload(final Test test, String key, String base64) {
+    public String upload(final Test test, String key, String base64) {
         final byte[] data = Base64.getDecoder().decode(base64.getBytes());
-        upload(test, key, data);
+        return upload(test, key, data);
     }
 
     @Override
-    public void upload(final Test test, final String key, final File file) {
+    public String upload(final Test test, final String key, final File file) {
         final String prefixKey = getPrefixKey(_prefix, key);
         _client.putObject(PutObjectRequest.builder().bucket(_bucket).key(prefixKey)
                         .build(),
                 RequestBody.fromFile(file));
-        assignURL(test, prefixKey);
+        return getUrl(test, prefixKey);
     }
 
     @Override
     public void upload(final Test test, Embed embed) {
+        String url = null;
         if (null != embed.getBytes()) {
-            upload(test, embed.getName(), embed.getBytes());
+            url = upload(test, embed.getName(), embed.getBytes());
         } else if (null != embed.getBase64() && !embed.getBase64().isBlank()) {
-            upload(test, embed.getName(), embed.getBase64());
+            url = upload(test, embed.getName(), embed.getBase64());
         } else if (null != embed.getFile()) {
-            upload(test, embed.getName(), embed.getFile());
+            url = upload(test, embed.getName(), embed.getFile());
         } else {
             log.error("Unable to upload Embed to Azure Blob Storage. Source missing");
         }
+        embed.setUrl(url);
     }
 
     @Override
diff --git a/core/chaintest-core-java/src/main/java/com/aventstack/chaintest/storage/AzureBlobClient.java b/core/chaintest-core-java/src/main/java/com/aventstack/chaintest/storage/AzureBlobClient.java
diff --git a/core/chaintest-core-java/src/main/java/com/aventstack/chaintest/storage/StorageService.java b/core/chaintest-core-java/src/main/java/com/aventstack/chaintest/storage/StorageService.java
diff --git a/examples/chaintest-cucumber-jvm-example/src/test/resources/chaintest.properties b/examples/chaintest-cucumber-jvm-example/src/test/resources/chaintest.properties
diff --git a/examples/chaintest-testng-example/src/test/resources/chaintest.properties b/examples/chaintest-testng-example/src/test/resources/chaintest.properties

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,6 @@`
`1`	`1`	`export class Embed {`
`2`	`2`	`presigned: string;`
`3`	`3`	`url: string;`
	`4`	`+ bytes: any;`
	`5`	`+ base64: string;`
`4`	`6`	`}`