Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ jobs:
- name: Set up Maven Central Repository
uses: actions/setup-java@v4
with:
java-version: 8
java-version: 17
distribution: "adopt"
- uses: s4u/maven-settings-action@v3.0.0
with:
Expand All @@ -58,7 +58,7 @@ jobs:
"password": "${{ secrets.GH_PACKAGES_PASSWORD }}"
}]
- name: Test
run: mvn package -DskipTests
run: mvn package
test-sonar-package:
name: Run Trevas tests with coverage & sonar checks
# Trevas main repo commit branch or merged PR
Expand Down
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ If you want to contribute, see this [guide](docs/CONTRIBUTING.md).

## Requirements

Open JDK 8+ is required.
Open JDK 17 + is required.

## References

Expand Down
2 changes: 1 addition & 1 deletion docs/docs/introduction/index-introduction.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,6 @@ import Video from '@theme/Video';

## Technical description

Trevas provides VTL engines for different execution contexts, in particular a Java 11 engine and an [Apache Spark](https://spark.apache.org/) engine.
Trevas provides VTL engines for different execution contexts, in particular a Java 17 engine and an [Apache Spark](https://spark.apache.org/) engine.

Trevas engines are based on the [JSR 223](https://jcp.org/en/jsr/detail?id=223) specification (Scripting for the Java Platform), which describes the use of scripting languages in Java.
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,6 @@ import Video from '@theme/Video';

## Description technique

Trevas fournit des moteurs VTL pour différents environnemens d'exécution, notamment un moteur Java 11 et un moteur [Apache Spark](https://spark.apache.org/).
Trevas fournit des moteurs VTL pour différents environnemens d'exécution, notamment un moteur Java 17 et un moteur [Apache Spark](https://spark.apache.org/).

Les moteurs Trevas se basent sur la spécification [JSR 223](https://jcp.org/en/jsr/detail?id=223) qui décrit l'utilisation de langages de scripts en Java.
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,6 @@ import Video from '@theme/Video';

## Teknisk beskrivelse

Trevas leverer VTL-motorer for ulike driftsmiljøer, inkludert en Java 11-motor og en [Apache Spark](https://spark.apache.org/) motor.
Trevas leverer VTL-motorer for ulike driftsmiljøer, inkludert en Java 17-motor og en [Apache Spark](https://spark.apache.org/) motor.

Trevas-motorer er basert på [JSR 223](https://jcp.org/en/jsr/detail?id=223) spesifikasjonen som beskriver bruken av skriptspråk i Java.
22 changes: 14 additions & 8 deletions pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -69,12 +69,14 @@
</distributionManagement>

<properties>
<jdk.version>1.8</jdk.version>
<jdk.version>17</jdk.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<jacoco.version>0.8.7</jacoco.version>
<jacoco.version>0.8.13</jacoco.version>
<sonar.coverage.jacoco.xmlReportPaths>
${project.basedir}/coverage/target/site/jacoco-aggregate/jacoco.xml
</sonar.coverage.jacoco.xmlReportPaths>
<maven.compiler.source>${jdk.version}</maven.compiler.source>
<maven.compiler.target>${jdk.version}</maven.compiler.target>
</properties>

<dependencies>
Expand Down Expand Up @@ -110,7 +112,7 @@
<configuration>
<rules>
<requireJavaVersion>
<version>1.8</version>
<version>17.0.0</version>
</requireJavaVersion>
<requireMavenVersion>
<version>3.3.9</version>
Expand Down Expand Up @@ -171,16 +173,20 @@
<artifactId>sonar-maven-plugin</artifactId>
<version>3.9.1.2184</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
</plugin>
</plugins>
<pluginManagement>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.13.0</version>
<version>3.14.0</version>
<configuration>
<target>${jdk.version}</target>
<source>${jdk.version}</source>
<release>${jdk.version}</release>
<testRelease>${jdk.version}</testRelease>
</configuration>
</plugin>
</plugins>
Expand Down Expand Up @@ -220,7 +226,7 @@
<artifactId>maven-javadoc-plugin</artifactId>
<version>3.8.0</version>
<configuration>
<source>8</source>
<source>${jdk.version}</source>
<detectJavaApiLink>false</detectJavaApiLink>
<doclint>none</doclint>
</configuration>
Expand Down Expand Up @@ -280,7 +286,7 @@
<artifactId>maven-javadoc-plugin</artifactId>
<version>3.7.0</version>
<configuration>
<source>8</source>
<source>${jdk.version}</source>
<detectJavaApiLink>false</detectJavaApiLink>
<doclint>none</doclint>
</configuration>
Expand Down
9 changes: 4 additions & 5 deletions vtl-csv/src/main/java/fr/insee/vtl/csv/CSVDataset.java
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
package fr.insee.vtl.csv;

import fr.insee.vtl.model.Dataset;
import fr.insee.vtl.model.utils.Java8Helpers;
import org.supercsv.cellprocessor.Optional;
import org.supercsv.cellprocessor.ParseBool;
import org.supercsv.cellprocessor.ParseDouble;
Expand All @@ -27,8 +26,8 @@ public class CSVDataset implements Dataset {
public CSVDataset(DataStructure structure, Reader csv) throws IOException {
this.structure = structure;
this.csvReader = new CsvMapReader(csv, CsvPreference.EXCEL_NORTH_EUROPE_PREFERENCE);
String[] columns = this.csvReader.getHeader(true);
if (!this.structure.keySet().containsAll(Java8Helpers.listOf(columns))) {
var columns = this.csvReader.getHeader(true);
if (!this.structure.keySet().containsAll(List.of(columns))) {
throw new RuntimeException("missing columns in CSV");
}
}
Expand Down Expand Up @@ -69,8 +68,8 @@ public List<DataPoint> getDataPoints() {
if (this.data == null) {
this.data = new ArrayList<>();
try {
String[] header = getNameMapping();
CellProcessor[] processors = getProcessors();
var header = getNameMapping();
var processors = getProcessors();
Map<String, Object> datum;
while ((datum = this.csvReader.read(header, processors)) != null) {
this.data.add(new DataPoint(this.structure, datum));
Expand Down
27 changes: 3 additions & 24 deletions vtl-engine/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -44,16 +44,10 @@
<artifactId>commons-text</artifactId>
<version>1.10.0</version>
</dependency>
<!-- Because of safety mirror -->
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<version>1.18.34</version>
</dependency>
<dependency>
<groupId>org.jetbrains</groupId>
<artifactId>annotations</artifactId>
<version>20.1.0</version>
<groupId>com.github.hervian</groupId>
<artifactId>safety-mirror</artifactId>
<version>4.0.1</version>
</dependency>
</dependencies>
<build>
Expand All @@ -77,21 +71,6 @@
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<source>8</source>
<target>8</target>
<annotationProcessorPaths>
<path>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<version>1.18.34</version>
</path>
</annotationProcessorPaths>
</configuration>
</plugin>
</plugins>
</build>

Expand Down
70 changes: 30 additions & 40 deletions vtl-engine/src/main/java/fr/insee/vtl/engine/TemporalFunctions.java
Original file line number Diff line number Diff line change
@@ -1,10 +1,14 @@
package fr.insee.vtl.engine;

import fr.insee.vtl.engine.exceptions.VtlRuntimeException;
import fr.insee.vtl.model.exceptions.VtlScriptException;
import org.threeten.extra.Interval;
import org.threeten.extra.PeriodDuration;

import javax.script.ScriptEngine;
import javax.script.ScriptEngineManager;
import java.time.*;
import java.time.temporal.ChronoUnit;
import java.time.temporal.*;

/**
* This comment explains the temporal functionality supported by Trevas, as defined in the VTL 2.0 specification.
Expand Down Expand Up @@ -184,19 +188,19 @@ public static PeriodDuration period_indicator(Interval timePeriod) {
public static Interval timeshift(Interval time, Long n) {
OffsetDateTime from = time.getStart().atOffset(ZoneOffset.UTC);
OffsetDateTime to = time.getEnd().atOffset(ZoneOffset.UTC);
PeriodDuration dur = PeriodDuration.between(from, to)
var dur = PeriodDuration.between(from, to)
.multipliedBy(n.intValue());
return Interval.of(from.plus(dur.getPeriod()).toInstant(), to.plus(dur.getPeriod()).toInstant());
}

public static ZonedDateTime at_zone(Instant op, String zone) {
ZoneId zid = ZoneId.of(zone);
var zid = ZoneId.of(zone);
return op.atZone(zid);
}


private static Interval truncate_time(Interval op, ChronoUnit unit, ZoneId zone) {
Instant start = truncate_time(op.getStart(), unit, zone);
var start = truncate_time(op.getStart(), unit, zone);
return Interval.of(start, unit.getDuration());
}

Expand All @@ -209,25 +213,18 @@ public static Interval truncate_time(Interval op, String unit) {
}

private static Instant truncate_time(Instant op, ChronoUnit unit, ZoneId zone) {
ZonedDateTime zonedOp = op.atZone(zone);
switch (unit) {
case DAYS:
return zonedOp.truncatedTo(ChronoUnit.DAYS).toInstant();
case MONTHS:
return zonedOp.withDayOfMonth(1)
.truncatedTo(ChronoUnit.DAYS).toInstant();
case YEARS:
return zonedOp.withDayOfYear(1)
.truncatedTo(ChronoUnit.DAYS).toInstant();
case HOURS:
return zonedOp.truncatedTo(ChronoUnit.HOURS).toInstant();
case MINUTES:
return zonedOp.truncatedTo(ChronoUnit.MINUTES).toInstant();
case SECONDS:
return zonedOp.truncatedTo(ChronoUnit.SECONDS).toInstant();
default:
throw new IllegalArgumentException("Unsupported unit: " + unit);
}
var zonedOp = op.atZone(zone);
return switch (unit) {
case DAYS -> zonedOp.truncatedTo(ChronoUnit.DAYS).toInstant();
case MONTHS -> zonedOp.withDayOfMonth(1)
.truncatedTo(ChronoUnit.DAYS).toInstant();
case YEARS -> zonedOp.withDayOfYear(1)
.truncatedTo(ChronoUnit.DAYS).toInstant();
case HOURS -> zonedOp.truncatedTo(ChronoUnit.HOURS).toInstant();
case MINUTES -> zonedOp.truncatedTo(ChronoUnit.MINUTES).toInstant();
case SECONDS -> zonedOp.truncatedTo(ChronoUnit.SECONDS).toInstant();
default -> throw new IllegalArgumentException("Unsupported unit: " + unit);
};
}

public static Instant truncate_time(Instant op, String unit, String zone) {
Expand All @@ -244,26 +241,19 @@ public static ZonedDateTime truncate_time(ZonedDateTime op, String unit) {
}

public static OffsetDateTime truncate_time(OffsetDateTime op, String unit) {
ZonedDateTime zoned = op.toZonedDateTime();
var zoned = op.toZonedDateTime();
return truncate_time(zoned.toInstant(), toChronoUnit(unit), zoned.getZone()).atOffset(op.getOffset());
}

private static ChronoUnit toChronoUnit(String unit) {
switch (unit.toLowerCase()) {
case "day":
return ChronoUnit.DAYS;
case "month":
return ChronoUnit.MONTHS;
case "year":
return ChronoUnit.YEARS;
case "hour":
return ChronoUnit.HOURS;
case "minute":
return ChronoUnit.MINUTES;
case "second":
return ChronoUnit.SECONDS;
default:
throw new IllegalArgumentException("Unsupported unit: " + unit);
}
return switch (unit.toLowerCase()) {
case "day" -> ChronoUnit.DAYS;
case "month" -> ChronoUnit.MONTHS;
case "year" -> ChronoUnit.YEARS;
case "hour" -> ChronoUnit.HOURS;
case "minute" -> ChronoUnit.MINUTES;
case "second" -> ChronoUnit.SECONDS;
default -> throw new IllegalArgumentException("Unsupported unit: " + unit);
};
}
}
Original file line number Diff line number Diff line change
@@ -1,24 +1,22 @@
package fr.insee.vtl.engine;

import fr.insee.vtl.engine.utils.safetymirror.Fun;
import com.github.hervian.reflection.Fun;
import fr.insee.vtl.engine.visitors.expression.*;
import fr.insee.vtl.engine.visitors.expression.functions.ComparisonFunctionsVisitor;
import fr.insee.vtl.engine.visitors.expression.functions.DistanceFunctionsVisitor;
import fr.insee.vtl.engine.visitors.expression.functions.NumericFunctionsVisitor;
import fr.insee.vtl.engine.visitors.expression.functions.StringFunctionsVisitor;
import fr.insee.vtl.model.utils.Java8Helpers;
import org.threeten.extra.Interval;
import org.threeten.extra.PeriodDuration;

import java.lang.reflect.Method;
import java.time.Instant;
import java.time.OffsetDateTime;
import java.time.ZonedDateTime;
import java.time.*;
import java.time.temporal.ChronoUnit;
import java.util.Set;

public class VtlNativeMethods {

public static final Set<Method> NATIVE_METHODS = Java8Helpers.setOf(
public static final Set<Method> NATIVE_METHODS = Set.of(
// NumericFunctionsVisitor
Fun.toMethod(NumericFunctionsVisitor::ceil),
Fun.toMethod(NumericFunctionsVisitor::floor),
Expand Down
Loading
Loading