Skip to content

Commit 25da1eb

Browse files
annajgibsonannajgibson
authored
Merge branch 'main' into di-447-amend-dq-tests-for-housing
2 parents 15dc8dc + 8fdb87d commit 25da1eb

File tree

60 files changed

+1747
-2656
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

60 files changed

+1747
-2656
lines changed

‎.github/CODEOWNERS‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
# This file specifies owners for pull request approval
2+
# See https://help.github.com/articles/about-code-owners/
3+
4+
* @LBHackney-IT/data-analytics-platform

‎docker/sql-to-parquet/entrypoint.sh‎

Lines changed: 4 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -19,23 +19,18 @@ echo "Snapshot Id - $SNAPSHOT_ID"
1919
FILENAME="liberator_dump_${DATE}"
2020
DBNAME="liberator"
2121

22-
MYSQL_CONN_PARAMS="--user=${MYSQL_USER} --password=${MYSQL_PASS} --host=${MYSQL_HOST}"
22+
echo "Retrieving pem file..."
23+
wget -O /tmp/rds-combined-ca-bundle.pem https://truststore.pki.rds.amazonaws.com/eu-west-2/eu-west-2-bundle.pem
2324

24-
SSL_MODE=${MYSQL_SSL_MODE:-""}
25-
OTHER_FLAGS=${MYSQL_OTHER_FLAGS:-""}
25+
MYSQL_CONN_PARAMS="--user=${MYSQL_USER} --password=${MYSQL_PASS} --host=${MYSQL_HOST} --ssl-ca=/tmp/rds-combined-ca-bundle.pem"
2626

27-
if [ -n "$SSL_MODE" ]; then
28-
MYSQL_CONN_PARAMS="$MYSQL_CONN_PARAMS --ssl-mode=$SSL_MODE"
29-
echo "SSL Mode set to $SSL_MODE"
30-
fi
27+
OTHER_FLAGS=${MYSQL_OTHER_FLAGS:-""}
3128

3229
if [ -n "$OTHER_FLAGS" ]; then
3330
MYSQL_CONN_PARAMS="$MYSQL_CONN_PARAMS $OTHER_FLAGS"
3431
echo "Additional MySQL flags: $OTHER_FLAGS"
3532
fi
3633

37-
echo "MYSQL connection params: $MYSQL_CONN_PARAMS"
38-
3934
echo "Deleting old snapshots in database..."
4035
python3 delete_db_snapshots_in_db.py
4136

‎external-lib/Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
all: target/java-lib-1.0-SNAPSHOT-jar-with-dependencies.jar target/deequ-1.0.3.jar target/pydeequ-1.0.1.zip
44

55
target/java-lib-1.0-SNAPSHOT-jar-with-dependencies.jar: pom.xml
6-
mvn assembly:assembly -DdescriptorId=jar-with-dependencies
6+
mvn clean package
77

88
target/deequ-1.0.3.jar:
99
wget https://repo1.maven.org/maven2/com/amazon/deequ/deequ/1.0.3/deequ-1.0.3.jar -O target/deequ-1.0.3.jar

‎external-lib/pom.xml‎

Lines changed: 33 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -5,13 +5,14 @@
55
<artifactId>java-lib</artifactId>
66
<version>1.0-SNAPSHOT</version>
77
<name>java-lib</name>
8-
<!-- FIXME change it to the project's website -->
98
<url>http://www.example.com</url>
9+
1010
<properties>
1111
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
1212
<maven.compiler.source>1.7</maven.compiler.source>
1313
<maven.compiler.target>1.7</maven.compiler.target>
1414
</properties>
15+
1516
<dependencies>
1617
<dependency>
1718
<groupId>junit</groupId>
@@ -25,41 +26,44 @@
2526
<version>0.13.7</version>
2627
</dependency>
2728
</dependencies>
29+
2830
<build>
31+
<plugins>
32+
<plugin>
33+
<groupId>org.apache.maven.plugins</groupId>
34+
<artifactId>maven-assembly-plugin</artifactId>
35+
<version>3.3.0</version>
36+
<configuration>
37+
<descriptorRefs>
38+
<descriptorRef>jar-with-dependencies</descriptorRef>
39+
</descriptorRefs>
40+
<archive>
41+
<manifest>
42+
<addClasspath>true</addClasspath>
43+
</manifest>
44+
</archive>
45+
<appendAssemblyId>true</appendAssemblyId>
46+
<ignoreDirFormatExtensions>false</ignoreDirFormatExtensions>
47+
<includeProjectArtifact>false</includeProjectArtifact>
48+
</configuration>
49+
<executions>
50+
<execution>
51+
<id>make-assembly</id>
52+
<phase>package</phase>
53+
<goals>
54+
<goal>single</goal>
55+
</goals>
56+
</execution>
57+
</executions>
58+
</plugin>
59+
</plugins>
60+
2961
<pluginManagement>
30-
<!-- lock down plugins versions to avoid using Maven defaults (may be moved to parent pom) -->
3162
<plugins>
32-
<plugin>
33-
<groupId>org.apache.maven.plugins</groupId>
34-
<artifactId>maven-shade-plugin</artifactId>
35-
<version>3.2.4</version>
36-
<executions>
37-
<execution>
38-
<phase>package</phase>
39-
<goals>
40-
<goal>shade</goal>
41-
</goals>
42-
<configuration>
43-
<artifactSet>
44-
<excludes>
45-
<exclude>classworlds:classworlds</exclude>
46-
<exclude>junit:junit</exclude>
47-
<exclude>jmock:*</exclude>
48-
<exclude>*:xml-apis</exclude>
49-
<exclude>org.apache.maven:lib:tests</exclude>
50-
<exclude>log4j:log4j:jar:</exclude>
51-
</excludes>
52-
</artifactSet>
53-
</configuration>
54-
</execution>
55-
</executions>
56-
</plugin>
57-
<!-- clean lifecycle, see https://maven.apache.org/ref/current/maven-core/lifecycles.html#clean_Lifecycle -->
5863
<plugin>
5964
<artifactId>maven-clean-plugin</artifactId>
6065
<version>3.1.0</version>
6166
</plugin>
62-
<!-- default lifecycle, jar packaging: see https://maven.apache.org/ref/current/maven-core/default-bindings.html#Plugin_bindings_for_jar_packaging -->
6367
<plugin>
6468
<artifactId>maven-resources-plugin</artifactId>
6569
<version>3.0.2</version>
@@ -84,7 +88,6 @@
8488
<artifactId>maven-deploy-plugin</artifactId>
8589
<version>2.8.2</version>
8690
</plugin>
87-
<!-- site lifecycle, see https://maven.apache.org/ref/current/maven-core/lifecycles.html#site_Lifecycle -->
8891
<plugin>
8992
<artifactId>maven-site-plugin</artifactId>
9093
<version>3.7.1</version>

‎lambdas/g_drive_folder_to_s3/Dockerfile‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ COPY Pipfile Pipfile.lock /app/
1818
COPY main.py ./source/
1919

2020
# Install Python dependencies using pipenv
21-
RUN pipenv install
21+
RUN pipenv install
2222
RUN pipenv requirements > requirements.txt
2323
RUN pip install -t ./source/lib -r requirements.txt
2424

@@ -27,7 +27,3 @@ WORKDIR /app/source
2727
RUN zip -r g_drive_folder_to_s3.zip .
2828

2929
CMD "pyhon3", "main.py"
30-
31-
32-
33-

‎lambdas/lambda_layers/requirements3.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
notifications-python-client==10.0.0
1+
notifications-python-client==10.0.1

‎lambdas/lambda_layers/requirements4.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
numpy==2.1.3
1+
numpy==2.2.2

‎lambdas/lambda_layers/requirements5.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
gspread==6.1.4
22
oauth2client==4.1.3
3-
google-api-python-client==2.154.0
3+
google-api-python-client==2.160.0
44
yagmail==0.15.293

‎lambdas/lambda_layers/requirements6.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
urllib3==2.2.3
1+
urllib3==2.3.0

‎lambdas/lambda_layers/requirements7.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
s3fs==2024.10.0
1+
s3fs==2025.2.0

0 commit comments

Comments
 (0)