Skip to content

Commit 1773c89

Browse files
UbuntuUbuntu
Ubuntu
authored and
Ubuntu
committed
Add configurable filesystem binary storage (fix #860)
1 parent a499603 commit 1773c89

File tree

5 files changed

+172
-6
lines changed

5 files changed

+172
-6
lines changed

‎README.md‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,21 @@ docker run -p 8080:8080 -e hapi.fhir.default_encoding=xml hapiproject/hapi:lates
4949

5050
HAPI looks in the environment variables for properties in the [application.yaml](https://github.com/hapifhir/hapi-fhir-jpaserver-starter/blob/master/src/main/resources/application.yaml) file for defaults.
5151

52+
### Binary storage configuration
53+
54+
To stream large `Binary` payloads to disk instead of the database, configure the starter with filesystem storage properties:
55+
56+
```
57+
hapi:
58+
fhir:
59+
binary_storage_enabled: true
60+
binary_storage_mode: FILESYSTEM
61+
binary_storage_filesystem_base_directory: /binstore
62+
# inline_resource_storage_below_size: 131072 # optional override
63+
```
64+
65+
When `binary_storage_mode` is set to `FILESYSTEM` and `inline_resource_storage_below_size` is omitted, the starter automatically applies a 102400 byte (100 KB) inline threshold so smaller payloads remain in the database. Ensure the directory you point to is writable by the process (for Docker builds, mount it into the container with appropriate permissions).
66+
5267
### Configuration via overridden application.yaml file and using Docker
5368

5469
You can customize HAPI by telling HAPI to look for the configuration file in a different location, e.g.:

‎src/main/java/ca/uhn/fhir/jpa/starter/AppProperties.java‎

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,15 @@ public class AppProperties {
6161
private Boolean filter_search_enabled = true;
6262
private Boolean graphql_enabled = false;
6363
private Boolean binary_storage_enabled = false;
64-
private Integer inline_resource_storage_below_size = 0;
64+
65+
public enum BinaryStorageMode {
66+
DATABASE,
67+
FILESYSTEM
68+
}
69+
70+
private BinaryStorageMode binary_storage_mode = BinaryStorageMode.DATABASE;
71+
private String binary_storage_filesystem_base_directory;
72+
private Integer inline_resource_storage_below_size;
6573
private Boolean bulk_export_enabled = false;
6674
private Boolean bulk_import_enabled = false;
6775
private Boolean default_pretty_print = true;
@@ -483,6 +491,22 @@ public void setBinary_storage_enabled(Boolean binary_storage_enabled) {
483491
this.binary_storage_enabled = binary_storage_enabled;
484492
}
485493

494+
public BinaryStorageMode getBinary_storage_mode() {
495+
return binary_storage_mode;
496+
}
497+
498+
public void setBinary_storage_mode(BinaryStorageMode binary_storage_mode) {
499+
this.binary_storage_mode = binary_storage_mode;
500+
}
501+
502+
public String getBinary_storage_filesystem_base_directory() {
503+
return binary_storage_filesystem_base_directory;
504+
}
505+
506+
public void setBinary_storage_filesystem_base_directory(String binary_storage_filesystem_base_directory) {
507+
this.binary_storage_filesystem_base_directory = binary_storage_filesystem_base_directory;
508+
}
509+
486510
public Integer getInline_resource_storage_below_size() {
487511
return inline_resource_storage_below_size;
488512
}

‎src/main/java/ca/uhn/fhir/jpa/starter/common/FhirServerConfigCommon.java‎

Lines changed: 37 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
import ca.uhn.fhir.jpa.api.config.JpaStorageSettings;
44
import ca.uhn.fhir.jpa.binary.api.IBinaryStorageSvc;
55
import ca.uhn.fhir.jpa.binstore.DatabaseBinaryContentStorageSvcImpl;
6+
import ca.uhn.fhir.jpa.binstore.FilesystemBinaryStorageSvcImpl;
67
import ca.uhn.fhir.jpa.config.HibernatePropertiesProvider;
78
import ca.uhn.fhir.jpa.model.config.PartitionSettings;
89
import ca.uhn.fhir.jpa.model.config.PartitionSettings.CrossPartitionReferenceMode;
@@ -23,6 +24,7 @@
2324
import org.springframework.context.annotation.*;
2425
import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean;
2526
import org.springframework.transaction.annotation.EnableTransactionManagement;
27+
import org.springframework.util.Assert;
2628

2729
import java.util.HashSet;
2830
import java.util.stream.Collectors;
@@ -38,6 +40,7 @@
3840
public class FhirServerConfigCommon {
3941

4042
private static final Logger ourLog = LoggerFactory.getLogger(FhirServerConfigCommon.class);
43+
private static final int DEFAULT_FILESYSTEM_INLINE_THRESHOLD = 102_400;
4144

4245
public FhirServerConfigCommon(AppProperties appProperties) {
4346
ourLog.info(
@@ -222,8 +225,13 @@ public JpaStorageSettings jpaStorageSettings(AppProperties appProperties) {
222225
jpaStorageSettings.setLastNEnabled(true);
223226
}
224227

225-
if (appProperties.getInline_resource_storage_below_size() != 0) {
226-
jpaStorageSettings.setInlineResourceTextBelowSize(appProperties.getInline_resource_storage_below_size());
228+
Integer inlineResourceThreshold = appProperties.getInline_resource_storage_below_size();
229+
if (inlineResourceThreshold == null
230+
&& appProperties.getBinary_storage_mode() == AppProperties.BinaryStorageMode.FILESYSTEM) {
231+
inlineResourceThreshold = resolveFilesystemMinimumBinarySize(appProperties);
232+
}
233+
if (inlineResourceThreshold != null && inlineResourceThreshold != 0) {
234+
jpaStorageSettings.setInlineResourceTextBelowSize(inlineResourceThreshold);
227235
}
228236

229237
jpaStorageSettings.setStoreResourceInHSearchIndex(appProperties.getStore_resource_in_lucene_index_enabled());
@@ -342,15 +350,39 @@ public HibernatePropertiesProvider jpaStarterDialectProvider(
342350
@Lazy
343351
@Bean
344352
public IBinaryStorageSvc binaryStorageSvc(AppProperties appProperties) {
345-
DatabaseBinaryContentStorageSvcImpl binaryStorageSvc = new DatabaseBinaryContentStorageSvcImpl();
353+
IBinaryStorageSvc binaryStorageSvc;
354+
355+
if (appProperties.getBinary_storage_mode() == AppProperties.BinaryStorageMode.FILESYSTEM) {
356+
String baseDirectory = appProperties.getBinary_storage_filesystem_base_directory();
357+
Assert.hasText(
358+
baseDirectory,
359+
"binary_storage_filesystem_base_directory must be provided when binary_storage_mode=FILESYSTEM");
360+
361+
FilesystemBinaryStorageSvcImpl filesystemSvc = new FilesystemBinaryStorageSvcImpl(baseDirectory);
362+
int minimumBinarySize = resolveFilesystemMinimumBinarySize(appProperties);
363+
filesystemSvc.setMinimumBinarySize(minimumBinarySize);
364+
365+
if (appProperties.getMax_binary_size() != null) {
366+
filesystemSvc.setMaximumBinarySize(appProperties.getMax_binary_size().longValue());
367+
}
346368

347-
if (appProperties.getMax_binary_size() != null) {
348-
binaryStorageSvc.setMaximumBinarySize(appProperties.getMax_binary_size());
369+
binaryStorageSvc = filesystemSvc;
370+
} else {
371+
DatabaseBinaryContentStorageSvcImpl databaseSvc = new DatabaseBinaryContentStorageSvcImpl();
372+
if (appProperties.getMax_binary_size() != null) {
373+
databaseSvc.setMaximumBinarySize(appProperties.getMax_binary_size());
374+
}
375+
binaryStorageSvc = databaseSvc;
349376
}
350377

351378
return binaryStorageSvc;
352379
}
353380

381+
private int resolveFilesystemMinimumBinarySize(AppProperties appProperties) {
382+
Integer inlineThreshold = appProperties.getInline_resource_storage_below_size();
383+
return inlineThreshold == null ? DEFAULT_FILESYSTEM_INLINE_THRESHOLD : inlineThreshold;
384+
}
385+
354386
@Bean
355387
public IEmailSender emailSender(AppProperties appProperties) {
356388
if (appProperties.getSubscription() != null

‎src/main/resources/application.yaml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -381,6 +381,14 @@ hapi:
381381
# max_page_size: 200
382382
# retain_cached_searches_mins: 60
383383
# reuse_cached_search_results_millis: 60000
384+
# validation:
385+
# requests_enabled: true
386+
# responses_enabled: true
387+
# binary_storage_enabled: true
388+
# binary_storage_mode: FILESYSTEM
389+
# binary_storage_filesystem_base_directory: /binstore
390+
# When binary_storage_mode is FILESYSTEM and this value is not set,
391+
# the starter defaults to 102400 bytes so smaller binaries stay inline.
384392
inline_resource_storage_below_size: 4000
385393

386394
# -------------------------------------------------------------------------------

‎src/test/java/ca/uhn/fhir/jpa/starter/common/FhirServerConfigCommonBinaryStorageTest.java‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
package ca.uhn.fhir.jpa.starter.common;
2+
3+
import ca.uhn.fhir.jpa.binary.api.IBinaryStorageSvc;
4+
import ca.uhn.fhir.jpa.binstore.DatabaseBinaryContentStorageSvcImpl;
5+
import ca.uhn.fhir.jpa.binstore.FilesystemBinaryStorageSvcImpl;
6+
import ca.uhn.fhir.jpa.starter.AppProperties;
7+
import org.junit.jupiter.api.Test;
8+
import org.junit.jupiter.api.io.TempDir;
9+
10+
import java.nio.file.Files;
11+
import java.nio.file.Path;
12+
13+
import static org.assertj.core.api.Assertions.assertThat;
14+
import static org.assertj.core.api.Assertions.assertThatThrownBy;
15+
16+
class FhirServerConfigCommonBinaryStorageTest {
17+
18+
@TempDir
19+
Path tempDir;
20+
21+
private FhirServerConfigCommon newConfig() {
22+
return new FhirServerConfigCommon(new AppProperties());
23+
}
24+
25+
@Test
26+
void defaultsToDatabaseImplementation() {
27+
AppProperties props = new AppProperties();
28+
29+
IBinaryStorageSvc svc = newConfig().binaryStorageSvc(props);
30+
31+
assertThat(svc).isInstanceOf(DatabaseBinaryContentStorageSvcImpl.class);
32+
}
33+
34+
@Test
35+
void filesystemModeUsesDefaultMinimumWhenUnspecified() throws Exception {
36+
AppProperties props = new AppProperties();
37+
props.setBinary_storage_mode(AppProperties.BinaryStorageMode.FILESYSTEM);
38+
Path baseDir = tempDir.resolve("fs-default");
39+
Files.createDirectories(baseDir);
40+
props.setBinary_storage_filesystem_base_directory(baseDir.toString());
41+
42+
FilesystemBinaryStorageSvcImpl svc =
43+
(FilesystemBinaryStorageSvcImpl) newConfig().binaryStorageSvc(props);
44+
45+
assertThat(svc.getMinimumBinarySize()).isEqualTo(102_400);
46+
}
47+
48+
@Test
49+
void filesystemModeHonoursExplicitMinimum() throws Exception {
50+
AppProperties props = new AppProperties();
51+
props.setBinary_storage_mode(AppProperties.BinaryStorageMode.FILESYSTEM);
52+
props.setInline_resource_storage_below_size(4096);
53+
Path baseDir = tempDir.resolve("fs-min-explicit");
54+
Files.createDirectories(baseDir);
55+
props.setBinary_storage_filesystem_base_directory(baseDir.toString());
56+
57+
FilesystemBinaryStorageSvcImpl svc =
58+
(FilesystemBinaryStorageSvcImpl) newConfig().binaryStorageSvc(props);
59+
60+
assertThat(svc.getMinimumBinarySize()).isEqualTo(4096);
61+
}
62+
63+
@Test
64+
void filesystemModeSupportsZeroMinimumWhenExplicit() throws Exception {
65+
AppProperties props = new AppProperties();
66+
props.setBinary_storage_mode(AppProperties.BinaryStorageMode.FILESYSTEM);
67+
props.setInline_resource_storage_below_size(0);
68+
Path baseDir = tempDir.resolve("fs-zero");
69+
Files.createDirectories(baseDir);
70+
props.setBinary_storage_filesystem_base_directory(baseDir.toString());
71+
72+
FilesystemBinaryStorageSvcImpl svc =
73+
(FilesystemBinaryStorageSvcImpl) newConfig().binaryStorageSvc(props);
74+
75+
assertThat(svc.getMinimumBinarySize()).isZero();
76+
}
77+
78+
@Test
79+
void filesystemModeRequiresBaseDirectory() {
80+
AppProperties props = new AppProperties();
81+
props.setBinary_storage_mode(AppProperties.BinaryStorageMode.FILESYSTEM);
82+
83+
assertThatThrownBy(() -> newConfig().binaryStorageSvc(props))
84+
.isInstanceOf(IllegalArgumentException.class)
85+
.hasMessageContaining("binary_storage_filesystem_base_directory");
86+
}
87+
}

0 commit comments

Comments
 (0)