Skip to content

Commit b3c5d7d

Browse files
authored
Merge pull request #306 from aws/develop
chore: Lambda Builder 1.9.0 Release Merge to Master
2 parents 26c8265 + 0fdbcdd commit b3c5d7d

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

56 files changed

+1368
-356
lines changed

.appveyor.yml

Lines changed: 63 additions & 74 deletions
Original file line numberDiff line numberDiff line change
@@ -9,92 +9,81 @@ environment:
99
nodejs_version: "10.10.0"
1010

1111
matrix:
12-
13-
- PYTHON: "C:\\Python27-x64"
14-
PYTHON_VERSION: '2.7'
15-
PYTHON_ARCH: '64'
16-
LINE_COVERAGE: '91'
17-
NEW_FLAKE8: 0
18-
JAVA_HOME: "C:\\Program Files\\Java\\jdk11"
19-
- PYTHON: "C:\\Python36-x64"
20-
PYTHON_VERSION: '3.6'
21-
PYTHON_ARCH: '64'
22-
LINE_COVERAGE: '91'
23-
NEW_FLAKE8: 0
24-
JAVA_HOME: "C:\\Program Files\\Java\\jdk11"
25-
- PYTHON: "C:\\Python37-x64"
26-
PYTHON_VERSION: '3.7'
27-
PYTHON_ARCH: '64'
28-
LINE_COVERAGE: '91'
29-
NEW_FLAKE8: 0
30-
JAVA_HOME: "C:\\Program Files\\Java\\jdk11"
31-
- PYTHON: "C:\\Python38-x64"
32-
PYTHON_VERSION: '3.8'
33-
PYTHON_ARCH: '64'
34-
LINE_COVERAGE: '72'
35-
NEW_FLAKE8: 1
36-
JAVA_HOME: "C:\\Program Files\\Java\\jdk11"
37-
- PYTHON: "C:\\Python39-x64"
38-
PYTHON_VERSION: '3.9'
39-
PYTHON_ARCH: '64'
40-
LINE_COVERAGE: '72'
41-
NEW_FLAKE8: 1
42-
JAVA_HOME: "C:\\Program Files\\Java\\jdk11"
43-
44-
12+
- PYTHON: "C:\\Python36-x64"
13+
PYTHON_VERSION: "3.6"
14+
PYTHON_ARCH: "64"
15+
LINE_COVERAGE: "91"
16+
NEW_FLAKE8: 0
17+
JAVA_HOME: "C:\\Program Files\\Java\\jdk11"
18+
- PYTHON: "C:\\Python37-x64"
19+
PYTHON_VERSION: "3.7"
20+
PYTHON_ARCH: "64"
21+
LINE_COVERAGE: "91"
22+
NEW_FLAKE8: 0
23+
JAVA_HOME: "C:\\Program Files\\Java\\jdk11"
24+
- PYTHON: "C:\\Python38-x64"
25+
PYTHON_VERSION: "3.8"
26+
PYTHON_ARCH: "64"
27+
LINE_COVERAGE: "72"
28+
NEW_FLAKE8: 1
29+
JAVA_HOME: "C:\\Program Files\\Java\\jdk11"
30+
- PYTHON: "C:\\Python39-x64"
31+
PYTHON_VERSION: "3.9"
32+
PYTHON_ARCH: "64"
33+
LINE_COVERAGE: "72"
34+
NEW_FLAKE8: 1
35+
JAVA_HOME: "C:\\Program Files\\Java\\jdk11"
4536

4637
build: off
4738

4839
for:
49-
-
50-
matrix:
40+
- matrix:
5141
only:
5242
- image: Visual Studio 2019
5343

5444
environment:
5545
GOPATH: c:\gopath
5646

5747
install:
58-
# To run Nodejs workflow integ tests
59-
- ps: Install-Product node $env:nodejs_version
60-
61-
- "set PATH=%PYTHON%;%PYTHON%\\Scripts;%PYTHON%\\bin;%PATH%"
62-
- "%PYTHON%\\python.exe -m pip install -r requirements/dev.txt"
63-
- "%PYTHON%\\python.exe -m pip install -e ."
64-
- "set PATH=C:\\Ruby25-x64\\bin;%PATH%"
65-
- "gem --version"
66-
- "gem install bundler -v 1.17.3"
67-
- "bundler --version"
68-
- "echo %PATH%"
69-
70-
# setup go
71-
- rmdir c:\go /s /q
72-
- "choco install golang --version 1.15.7"
73-
- "choco install bzr"
74-
- "choco install dep"
75-
- setx PATH "C:\go\bin;C:\gopath\bin;C:\Program Files (x86)\Bazaar\;C:\Program Files\Mercurial;%PATH%;"
76-
- "go version"
77-
# set set GO111MODULE to auto to enable module-aware mode only when a go.mod file is present in the current directory or any parent directory
78-
# https://blog.golang.org/go116-module-changes#TOC_2.
79-
# This is required for the go dep integration tests
80-
- "go env -w GO111MODULE=auto"
81-
- "go env"
82-
83-
# setup Gradle
84-
- "choco install gradle"
85-
86-
# setup make
87-
- "choco install make"
88-
89-
# Echo final Path
90-
- "echo %PATH%"
48+
# To run Nodejs workflow integ tests
49+
- ps: Install-Product node $env:nodejs_version
50+
51+
- "set PATH=%PYTHON%;%PYTHON%\\Scripts;%PYTHON%\\bin;%PATH%"
52+
- "%PYTHON%\\python.exe -m pip install -r requirements/dev.txt"
53+
- "%PYTHON%\\python.exe -m pip install -e ."
54+
- "set PATH=C:\\Ruby25-x64\\bin;%PATH%"
55+
- "gem --version"
56+
- "gem install bundler -v 1.17.3"
57+
- "bundler --version"
58+
- "echo %PATH%"
59+
60+
# setup go
61+
- rmdir c:\go /s /q
62+
- "choco install golang --version 1.15.7"
63+
- "choco install bzr"
64+
- "choco install dep"
65+
- setx PATH "C:\go\bin;C:\gopath\bin;C:\Program Files (x86)\Bazaar\;C:\Program Files\Mercurial;%PATH%;"
66+
- "go version"
67+
# set set GO111MODULE to auto to enable module-aware mode only when a go.mod file is present in the current directory or any parent directory
68+
# https://blog.golang.org/go116-module-changes#TOC_2.
69+
# This is required for the go dep integration tests
70+
- "go env -w GO111MODULE=auto"
71+
- "go env"
72+
73+
# setup Gradle
74+
- "choco install gradle"
75+
76+
# setup make
77+
- "choco install make"
78+
79+
# Echo final Path
80+
- "echo %PATH%"
9181

9282
test_script:
93-
- "%PYTHON%\\python.exe -m pytest --cov aws_lambda_builders --cov-report term-missing tests/unit tests/functional"
94-
- "%PYTHON%\\python.exe -m pytest tests/integration"
83+
- "%PYTHON%\\python.exe -m pytest --cov aws_lambda_builders --cov-report term-missing tests/unit tests/functional"
84+
- "%PYTHON%\\python.exe -m pytest tests/integration"
9585

96-
-
97-
matrix:
86+
- matrix:
9887
only:
9988
- image: Ubuntu
10089

@@ -121,8 +110,8 @@ for:
121110
- sh: "go env -w GO111MODULE=auto"
122111

123112
build_script:
124-
- "python -c \"import sys; print(sys.executable)\""
125-
- "LAMBDA_BUILDERS_DEV=1 pip install -e \".[dev]\""
113+
- 'python -c "import sys; print(sys.executable)"'
114+
- 'LAMBDA_BUILDERS_DEV=1 pip install -e ".[dev]"'
126115

127116
test_script:
128117
- "LAMBDA_BUILDERS_DEV=1 pytest --cov aws_lambda_builders --cov-report term-missing --cov-fail-under ${LINE_COVERAGE} tests/unit tests/functional"

Makefile

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,3 @@ black-check:
2828

2929
# Verifications to run before sending a pull request
3030
pr: init dev black-check
31-
32-
# Verifications to run before sending a pull request, skipping black check because black requires Python 3.6+
33-
pr2.7: init dev

aws_lambda_builders/__init__.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
"""
22
AWS Lambda Builder Library
33
"""
4-
__version__ = "1.8.1"
4+
__version__ = "1.9.0"
55
RPC_PROTOCOL_VERSION = "0.3"

aws_lambda_builders/__main__.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -125,6 +125,9 @@ def main(): # pylint: disable=too-many-statements
125125
optimizations=params["optimizations"],
126126
options=params["options"],
127127
mode=params.get("mode", None),
128+
download_dependencies=params.get("download_dependencies", True),
129+
dependencies_dir=params.get("dependencies_dir", None),
130+
combine_dependencies=params.get("combine_dependencies", True),
128131
architecture=params.get("architecture", X86_64),
129132
)
130133

aws_lambda_builders/actions.py

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
"""
44

55
import logging
6+
import os
67
import shutil
78
import six
89

@@ -30,9 +31,18 @@ class Purpose(object):
3031
# Action is copying source code
3132
COPY_SOURCE = "COPY_SOURCE"
3233

34+
# Action is copying dependencies
35+
COPY_DEPENDENCIES = "COPY_DEPENDENCIES"
36+
37+
# Action is moving dependencies
38+
MOVE_DEPENDENCIES = "MOVE_DEPENDENCIES"
39+
3340
# Action is compiling source code
3441
COMPILE_SOURCE = "COMPILE_SOURCE"
3542

43+
# Action is cleaning up the target folder
44+
CLEAN_UP = "CLEAN_UP"
45+
3646
@staticmethod
3747
def has_value(item):
3848
return item in Purpose.__dict__.values()
@@ -99,3 +109,87 @@ def __init__(self, source_dir, dest_dir, excludes=None):
99109

100110
def execute(self):
101111
copytree(self.source_dir, self.dest_dir, ignore=shutil.ignore_patterns(*self.excludes))
112+
113+
114+
class CopyDependenciesAction(BaseAction):
115+
116+
NAME = "CopyDependencies"
117+
118+
DESCRIPTION = "Copying dependencies while skipping source file"
119+
120+
PURPOSE = Purpose.COPY_DEPENDENCIES
121+
122+
def __init__(self, source_dir, artifact_dir, destination_dir):
123+
self.source_dir = source_dir
124+
self.artifact_dir = artifact_dir
125+
self.dest_dir = destination_dir
126+
127+
def execute(self):
128+
source = set(os.listdir(self.source_dir))
129+
artifact = set(os.listdir(self.artifact_dir))
130+
dependencies = artifact - source
131+
132+
for name in dependencies:
133+
dependencies_source = os.path.join(self.artifact_dir, name)
134+
new_destination = os.path.join(self.dest_dir, name)
135+
136+
if os.path.isdir(dependencies_source):
137+
copytree(dependencies_source, new_destination)
138+
else:
139+
shutil.copy2(dependencies_source, new_destination)
140+
141+
142+
class MoveDependenciesAction(BaseAction):
143+
144+
NAME = "MoveDependencies"
145+
146+
DESCRIPTION = "Moving dependencies while skipping source file"
147+
148+
PURPOSE = Purpose.MOVE_DEPENDENCIES
149+
150+
def __init__(self, source_dir, artifact_dir, destination_dir):
151+
self.source_dir = source_dir
152+
self.artifact_dir = artifact_dir
153+
self.dest_dir = destination_dir
154+
155+
def execute(self):
156+
source = set(os.listdir(self.source_dir))
157+
artifact = set(os.listdir(self.artifact_dir))
158+
dependencies = artifact - source
159+
160+
for name in dependencies:
161+
dependencies_source = os.path.join(self.artifact_dir, name)
162+
new_destination = os.path.join(self.dest_dir, name)
163+
164+
shutil.move(dependencies_source, new_destination)
165+
166+
167+
class CleanUpAction(BaseAction):
168+
"""
169+
Class for cleaning the directory. It will clean all the files in the directory but doesn't delete the directory
170+
"""
171+
172+
NAME = "CleanUp"
173+
174+
DESCRIPTION = "Cleaning up the target folder"
175+
176+
PURPOSE = Purpose.CLEAN_UP
177+
178+
def __init__(self, target_dir):
179+
self.target_dir = target_dir
180+
181+
def execute(self):
182+
if not os.path.isdir(self.target_dir):
183+
LOG.info("Clean up action: %s does not exist and will be skipped.", str(self.target_dir))
184+
return
185+
targets = os.listdir(self.target_dir)
186+
LOG.info("Clean up action: folder %s will be cleaned", str(self.target_dir))
187+
188+
for name in targets:
189+
target_path = os.path.join(self.target_dir, name)
190+
LOG.debug("Clean up action: %s is deleted", str(target_path))
191+
192+
if os.path.isdir(target_path):
193+
shutil.rmtree(target_path)
194+
else:
195+
os.remove(target_path)

aws_lambda_builders/builder.py

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,9 @@ def build(
6565
options=None,
6666
executable_search_paths=None,
6767
mode=None,
68+
download_dependencies=True,
69+
dependencies_dir=None,
70+
combine_dependencies=True,
6871
architecture=X86_64,
6972
):
7073
"""
@@ -108,6 +111,19 @@ def build(
108111
:param mode:
109112
Optional, Mode the build should produce
110113
114+
:type download_dependencies: bool
115+
:param download_dependencies:
116+
Optional, Should download dependencies when building
117+
118+
:type dependencies_dir: str
119+
:param dependencies_dir:
120+
Optional, Path to folder the dependencies should be downloaded to
121+
122+
:type combine_dependencies: bool
123+
:param combine_dependencies:
124+
Optional, This flag will only be used if dependency_folder is specified. False will not copy dependencies
125+
from dependency_folder into build folder
126+
111127
:type architecture: str
112128
:param architecture:
113129
Type of architecture x86_64 and arm64 for Lambda Function
@@ -126,6 +142,9 @@ def build(
126142
options=options,
127143
executable_search_paths=executable_search_paths,
128144
mode=mode,
145+
download_dependencies=download_dependencies,
146+
dependencies_dir=dependencies_dir,
147+
combine_dependencies=combine_dependencies,
129148
architecture=architecture,
130149
)
131150

aws_lambda_builders/utils.py

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,10 @@ def copytree(source, destination, ignore=None):
3131
``ignore`` property of ``shutils.copytree`` method
3232
"""
3333

34+
if not os.path.exists(source):
35+
LOG.warning("Skipping copy operation since source %s does not exist", source)
36+
return
37+
3438
if not os.path.exists(destination):
3539
os.makedirs(destination)
3640

aws_lambda_builders/workflow.py

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -160,6 +160,9 @@ def __init__(
160160
optimizations=None,
161161
options=None,
162162
mode=BuildMode.RELEASE,
163+
download_dependencies=True,
164+
dependencies_dir=None,
165+
combine_dependencies=True,
163166
architecture=X86_64,
164167
):
165168
"""
@@ -178,7 +181,7 @@ def __init__(
178181
manifest_path : str
179182
Path to the dependency manifest
180183
runtime : str, optional
181-
name of the AWS Lambda runtime that you are building for. This is sent to the builder for
184+
Optional, name of the AWS Lambda runtime that you are building for. This is sent to the builder for
182185
informational purposes, by default None
183186
executable_search_paths : list, optional
184187
Additional list of paths to search for executables required by the workflow, by default None
@@ -188,6 +191,13 @@ def __init__(
188191
dictionary of options ot pass to build action. **Not supported**., by default None
189192
mode : str, optional
190193
Mode the build should produce, by default BuildMode.RELEASE
194+
download_dependencies: bool, optional
195+
Should download dependencies when building
196+
dependencies_dir : str, optional
197+
Path to folder the dependencies should be downloaded to
198+
combine_dependencies: bool, optional
199+
This flag will only be used if dependency_folder is specified. False will not copy dependencies
200+
from dependency_folder into build folder
191201
architecture : str, optional
192202
Architecture type either arm64 or x86_64 for which the build will be based on in AWS lambda, by default X86_64
193203
"""
@@ -201,6 +211,9 @@ def __init__(
201211
self.options = options
202212
self.executable_search_paths = executable_search_paths
203213
self.mode = mode
214+
self.download_dependencies = download_dependencies
215+
self.dependencies_dir = dependencies_dir
216+
self.combine_dependencies = combine_dependencies
204217
self.architecture = architecture
205218

206219
# Actions are registered by the subclasses as they seem fit

0 commit comments

Comments
 (0)