Skip to content

Commit 45fd62f

Browse files
authored
Fix deeploy by target nodes count (#288)
* fix: add etra logging to deeploy payment check * fix: signature check when no target nodes specified * fix: car image pull * fix: car image pull * fix: autoupdate * fix: login and image pull * fix: add extra logging & fix deeployment by target nodes count * chore: inc version
1 parent aa8fa9e commit 45fd62f

File tree

5 files changed

+485
-171
lines changed

5 files changed

+485
-171
lines changed

extensions/business/container_apps/container_app_runner.py

Lines changed: 266 additions & 71 deletions
Original file line numberDiff line numberDiff line change
@@ -1098,15 +1098,126 @@ def on_close(self):
10981098
super(ContainerAppRunnerPlugin, self).on_close()
10991099

11001100

1101+
def _get_local_image(self):
1102+
"""
1103+
Get the local Docker image if it exists.
1104+
1105+
Returns:
1106+
Image object or None if image doesn't exist locally
1107+
"""
1108+
if not self.cfg_image:
1109+
return None
1110+
1111+
try:
1112+
img = self.docker_client.images.get(self.cfg_image)
1113+
return img
1114+
except Exception:
1115+
return None
1116+
1117+
def _pull_image_from_registry(self):
1118+
"""
1119+
Pull image from registry (assumes authentication already done).
1120+
1121+
Returns:
1122+
Image object or None if pull failed
1123+
1124+
Raises:
1125+
RuntimeError: If authentication hasn't been performed
1126+
"""
1127+
if not self.cfg_image:
1128+
self.P("No Docker image configured", color='r')
1129+
return None
1130+
1131+
try:
1132+
self.P(f"Pulling image '{self.cfg_image}'...", color='b')
1133+
img = self.docker_client.images.pull(self.cfg_image)
1134+
1135+
# docker-py may return Image or list[Image]
1136+
if isinstance(img, list) and img:
1137+
img = img[-1]
1138+
1139+
self.P(f"Successfully pulled image '{self.cfg_image}'", color='g')
1140+
return img
1141+
1142+
except Exception as e:
1143+
self.P(f"Image pull failed: {e}", color='r')
1144+
return None
1145+
1146+
def _pull_image_with_fallback(self):
1147+
"""
1148+
Pull image from registry with fallback to local image.
1149+
1150+
This is the main image acquisition method that:
1151+
1. Authenticates with registry
1152+
2. Attempts to pull from registry
1153+
3. Falls back to local image if pull fails
1154+
4. Returns None only if both pull and local check fail
1155+
1156+
Returns:
1157+
Image object or None if no image is available
1158+
1159+
Raises:
1160+
RuntimeError: If authentication fails and no local image exists
1161+
"""
1162+
# Step 1: Authenticate with registry
1163+
if not self._login_to_registry():
1164+
self.P("Registry authentication failed", color='y')
1165+
# Try to use local image if authentication fails
1166+
local_img = self._get_local_image()
1167+
if local_img:
1168+
self.P(f"Using local image (registry login failed): {self.cfg_image}", color='y')
1169+
return local_img
1170+
raise RuntimeError("Failed to authenticate with registry and no local image available.")
1171+
1172+
# Step 2: Attempt to pull from registry
1173+
img = self._pull_image_from_registry()
1174+
if img:
1175+
return img
1176+
1177+
# Step 3: Fallback to local image
1178+
self.P(f"Pull failed, checking for local image: {self.cfg_image}", color='b')
1179+
local_img = self._get_local_image()
1180+
if local_img:
1181+
self.P(f"Using local image as fallback: {self.cfg_image}", color='y')
1182+
return local_img
1183+
1184+
# Step 4: No image available
1185+
self.P(f"No image available (pull failed and no local image): {self.cfg_image}", color='r')
1186+
return None
1187+
1188+
def _get_image_digest(self, img):
1189+
"""
1190+
Extract digest hash from image object.
1191+
1192+
Args:
1193+
img: Docker image object
1194+
1195+
Returns:
1196+
str or None: Digest hash (sha256:...) or None
1197+
"""
1198+
if not img:
1199+
return None
1200+
1201+
try:
1202+
img.reload()
1203+
except Exception as e:
1204+
self.Pd(f"Warning: Could not reload image attributes: {e}")
1205+
1206+
attrs = getattr(img, "attrs", {}) or {}
1207+
repo_digests = attrs.get("RepoDigests") or []
1208+
if repo_digests:
1209+
# 'repo@sha256:...'
1210+
digest = repo_digests[0].split("@")[-1]
1211+
return digest
1212+
# Fallback to image id (sha256:...)
1213+
return getattr(img, "id", None)
1214+
11011215
def _get_latest_image_hash(self):
11021216
"""
11031217
Get the latest identifier for the configured Docker image tag.
11041218
1105-
This method tries to resolve the remote content digest for ``self.cfg_image`` by
1106-
asking the Docker daemon to perform a metadata-only pull (if the image is
1107-
already up to date, no layers are re-downloaded). It returns the repo digest
1108-
(e.g., ``sha256:...``) when available; if not available, it falls back to the
1109-
local image ID.
1219+
This method pulls the image and extracts its digest for version tracking.
1220+
Used by AUTOUPDATE feature to detect image changes.
11101221
11111222
Returns
11121223
-------
@@ -1120,78 +1231,87 @@ def _get_latest_image_hash(self):
11201231
credentials configured.
11211232
- This call contacts the registry; tune ``poll_interval`` appropriately.
11221233
"""
1123-
if not self.cfg_image:
1124-
self.P("No Docker image configured", color='r')
1125-
return None
1234+
img = self._pull_image_with_fallback()
1235+
return self._get_image_digest(img)
11261236

1127-
# Ensure we're logged in to the registry before pulling
1128-
if not self._login_to_registry():
1129-
raise RuntimeError("Failed to login to container registry. Cannot proceed without authentication.")
1237+
def _has_image_hash_changed(self, latest_hash):
1238+
"""
1239+
Check if image hash has changed from current version.
1240+
1241+
Args:
1242+
latest_hash: Latest image hash from registry
1243+
1244+
Returns:
1245+
bool: True if hash changed and update needed, False otherwise
1246+
"""
1247+
if not latest_hash:
1248+
# Pull failed, can't determine if update needed
1249+
return False
1250+
1251+
if not self.current_image_hash:
1252+
# First time - establish baseline
1253+
self.P(f"Establishing baseline image hash: {latest_hash}", color='b')
1254+
self.current_image_hash = latest_hash
1255+
return False
1256+
1257+
# Compare hashes
1258+
return latest_hash != self.current_image_hash
1259+
1260+
def _handle_image_update(self, new_hash):
1261+
"""
1262+
Handle detected image update by updating hash and restarting container.
1263+
1264+
Args:
1265+
new_hash: New image hash detected
1266+
"""
1267+
self.P(f"New image version detected ({new_hash} != {self.current_image_hash}). Restarting container...", color='y')
1268+
1269+
# Update current_image_hash BEFORE restart
1270+
# This prevents infinite retry loops if restart fails
1271+
old_hash = self.current_image_hash
1272+
self.current_image_hash = new_hash
11301273

11311274
try:
1132-
self.P(f"Image check: pulling '{self.cfg_image}' for metadata...", color='b')
1133-
img = self.docker_client.images.pull(self.cfg_image)
1134-
# docker-py may return Image or list[Image]
1135-
if isinstance(img, list) and img:
1136-
img = img[-1]
1137-
# Ensure attributes loaded
1138-
try:
1139-
img.reload()
1140-
except Exception as e:
1141-
self.P(f"Warning: Could not reload image attributes: {e}", color='y')
1142-
# end try
1143-
1144-
attrs = getattr(img, "attrs", {}) or {}
1145-
repo_digests = attrs.get("RepoDigests") or []
1146-
if repo_digests:
1147-
# 'repo@sha256:...'
1148-
digest = repo_digests[0].split("@")[-1]
1149-
return digest
1150-
# Fallback to image id (sha256:...)
1151-
return getattr(img, "id", None)
1152-
1275+
self._restart_container()
11531276
except Exception as e:
1154-
self.P(f"Image pull failed: {e}", color='r')
1155-
# Fallback: check local image only
1156-
try:
1157-
self.P(f"Checking local image: {self.cfg_image}", color='b')
1158-
img = self.docker_client.images.get(self.cfg_image)
1159-
try:
1160-
img.reload()
1161-
except Exception as e:
1162-
self.P(f"Warning: Could not reload local image attributes: {e}", color='y')
1163-
# end try reload
1164-
attrs = getattr(img, "attrs", {}) or {}
1165-
repo_digests = attrs.get("RepoDigests") or []
1166-
if repo_digests:
1167-
digest = repo_digests[0].split("@")[-1]
1168-
return digest
1169-
return getattr(img, "id", None)
1170-
1171-
except Exception as e2:
1172-
self.P(f"Could not get local image: {e2}", color='r')
1173-
# end try check for local image
1174-
# end try
1175-
return None
1277+
self.P(f"Container restart failed after image update: {e}", color='r')
1278+
# Hash already updated, won't retry this version
1279+
self.P(f"Image hash updated from {old_hash} to {new_hash}, but container restart failed", color='y')
11761280

11771281
def _check_image_updates(self, current_time=None):
1178-
"""Check for a new version of the Docker image and restart container if found."""
1282+
"""
1283+
Periodic check for image updates when AUTOUPDATE is enabled.
1284+
1285+
This method:
1286+
1. Checks if update check is due (based on interval)
1287+
2. Pulls latest image and gets its hash
1288+
3. Compares with current hash
1289+
4. Triggers restart if changed
1290+
1291+
Args:
1292+
current_time: Current timestamp (for interval checking)
1293+
"""
11791294
if not self.cfg_autoupdate:
11801295
return
1181-
1182-
if current_time - self._last_image_check >= self.cfg_autoupdate_interval:
1183-
self._last_image_check = current_time
1184-
latest_image_hash = self._get_latest_image_hash()
1185-
if latest_image_hash and self.current_image_hash and latest_image_hash != self.current_image_hash:
1186-
self.P(f"New image version detected ({latest_image_hash} != {self.current_image_hash}). Restarting container...", color='y')
1187-
# Update current_image_hash to the new one
1188-
self.current_image_hash = latest_image_hash
1189-
# Restart container from scratch
1190-
self._restart_container()
1191-
elif latest_image_hash:
1192-
self.P(f"Current image hash: {self.current_image_hash} vs latest: {latest_image_hash}")
1193-
# end if new image hash
1194-
# end if time elapsed
1296+
1297+
# Check if update check is due
1298+
if current_time - self._last_image_check < self.cfg_autoupdate_interval:
1299+
return
1300+
1301+
self._last_image_check = current_time
1302+
1303+
# Get latest image hash
1304+
latest_hash = self._get_latest_image_hash()
1305+
if not latest_hash:
1306+
self.P("Failed to check for image updates (pull failed). Container continues running.", color='y')
1307+
return
1308+
1309+
# Check if update is needed
1310+
if self._has_image_hash_changed(latest_hash):
1311+
self._handle_image_update(latest_hash)
1312+
else:
1313+
self.Pd(f"Image up to date: {self.current_image_hash}")
1314+
11951315
return
11961316

11971317
def _restart_container(self):
@@ -1210,6 +1330,11 @@ def _restart_container(self):
12101330

12111331
self._validate_runner_config()
12121332

1333+
# Ensure image is available (respecting AUTOUPDATE and IMAGE_PULL_POLICY)
1334+
if not self._ensure_image_available():
1335+
self.P("Failed to ensure image availability during restart, cannot start container", color='r')
1336+
return
1337+
12131338
self.container = self.start_container()
12141339
if not self.container:
12151340
return
@@ -1219,12 +1344,82 @@ def _restart_container(self):
12191344
self._maybe_execute_build_and_run()
12201345
return
12211346

1347+
def _ensure_image_with_autoupdate(self):
1348+
"""
1349+
Ensure image is available with autoupdate enabled.
1350+
Always pulls and tracks hash for version comparison.
1351+
1352+
Returns:
1353+
bool: True if image available and hash tracked, False otherwise
1354+
"""
1355+
self.Pd("AUTOUPDATE enabled, pulling image and tracking hash")
1356+
self.current_image_hash = self._get_latest_image_hash()
1357+
return self.current_image_hash is not None
1358+
1359+
def _ensure_image_always_pull(self):
1360+
"""
1361+
Ensure image is available with 'always' pull policy.
1362+
Pulls image without tracking hash.
1363+
1364+
Returns:
1365+
bool: True if image pulled successfully, False otherwise
1366+
"""
1367+
self.Pd("IMAGE_PULL_POLICY is 'always', pulling image")
1368+
img = self._pull_image_with_fallback()
1369+
return img is not None
1370+
1371+
def _ensure_image_if_not_present(self):
1372+
"""
1373+
Ensure image is available with 'if-not-present' policy.
1374+
Only pulls if image doesn't exist locally.
1375+
1376+
Returns:
1377+
bool: True if image is available (locally or after pull), False otherwise
1378+
"""
1379+
# Check if image exists locally
1380+
local_img = self._get_local_image()
1381+
if local_img:
1382+
self.P(f"Image '{self.cfg_image}' found locally", color='g')
1383+
return True
1384+
1385+
# Image not found locally, pull it
1386+
self.P(f"Image not found locally, pulling '{self.cfg_image}'...", color='b')
1387+
img = self._pull_image_with_fallback()
1388+
return img is not None
1389+
1390+
def _ensure_image_available(self):
1391+
"""
1392+
Ensure the container image is available before starting container.
1393+
1394+
This method uses a strategy pattern based on configuration:
1395+
- AUTOUPDATE enabled: Always pull + track hash (update detection)
1396+
- IMAGE_PULL_POLICY='always': Always pull (no tracking)
1397+
- IMAGE_PULL_POLICY='if-not-present' or default: Pull only if missing locally
1398+
1399+
Returns:
1400+
bool: True if image is available, False otherwise
1401+
"""
1402+
# Strategy 1: AUTOUPDATE (takes precedence)
1403+
if self.cfg_autoupdate:
1404+
return self._ensure_image_with_autoupdate()
1405+
1406+
# Strategy 2: Always pull policy
1407+
if self.cfg_image_pull_policy == "always":
1408+
return self._ensure_image_always_pull()
1409+
1410+
# Strategy 3: If-not-present policy (default)
1411+
return self._ensure_image_if_not_present()
1412+
12221413
def _handle_initial_launch(self):
12231414
"""Handle the initial container launch."""
12241415
try:
12251416
self.P("Initial container launch...", color='b')
1226-
if self.cfg_autoupdate:
1227-
self.current_image_hash = self._get_latest_image_hash()
1417+
1418+
# Ensure image is available before starting container
1419+
if not self._ensure_image_available():
1420+
self.P("Failed to ensure image availability, cannot start container", color='r')
1421+
return
1422+
12281423
self.container = self.start_container()
12291424
if not self.container:
12301425
return

extensions/business/deeploy/deeploy_const.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -197,7 +197,7 @@ class DEEPLOY_PLUGIN_DATA:
197197

198198
CONTAINER_APP_RUNNER_SIGNATURE = 'CONTAINER_APP_RUNNER'
199199
WORKER_APP_RUNNER_SIGNATURE = 'WORKER_APP_RUNNER'
200-
200+
CONTAINERIZED_APPS_SIGNATURES = [CONTAINER_APP_RUNNER_SIGNATURE, WORKER_APP_RUNNER_SIGNATURE]
201201

202202
class JOB_APP_TYPES:
203203
GENERIC = "generic"

0 commit comments

Comments
 (0)