Skip to content

bulk create refactor #531

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ezk06eer wants to merge 3 commits into
base: master
Choose a base branch
from
Open

bulk create refactor #531

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
bd132bb
Front Updated
Oct 3, 2025
8c0522b
The `bulk_create.py` implementation was refactored to optimize memory…
ezk06eer Nov 4, 2025
926455c
Refactor on bulk create
ezk06eer Nov 4, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
276 changes: 217 additions & 59 deletions faraday/server/api/modules/bulk_create.py
View file
Open in desktop

Large diffs are not rendered by default.

68 changes: 68 additions & 0 deletions faraday/server/tasks.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,74 @@ def create_host_task(workspace_id, command: dict, host):
return created_objects


@celery.task(ignore_result=False, acks_late=True)
def create_vulnerabilities_task(workspace_id, command: dict, host_id: int, vulnerabilities: list):
"""Process a chunk of vulnerabilities for a specific host

This task processes a chunk of vulnerabilities (max 100) for a specific host.
It's designed to be memory efficient by processing vulnerabilities in smaller batches.

Args:
workspace_id: ID of the workspace
command: Command dictionary with id, tool, and user
host_id: ID of the host these vulnerabilities belong to
vulnerabilities: List of vulnerability data dictionaries (max 100)

Returns:
Dictionary with created and updated counts
"""
from faraday.server.api.modules.bulk_create import _create_hostvuln # pylint: disable=import-outside-toplevel

created_updated_count = {'created': 0, 'updated': 0}
db.engine.dispose()

workspace = Workspace.query.filter_by(id=workspace_id).first()
if not workspace:
logger.error("Workspace %s not found", workspace_id)
return created_updated_count

host = Host.query.filter_by(id=host_id, workspace_id=workspace_id).first()
if not host:
logger.error(f"Host with ID {host_id} not found in workspace {workspace_id}")
return created_updated_count

try:
logger.debug(f"Processing {len(vulnerabilities)} vulnerabilities for host {host.ip}")

processed_data = {}
for vuln_data in vulnerabilities:
host_vuln_dict, vuln_id = _create_hostvuln(workspace, host, vuln_data, command)

updated_processed_data = host_vuln_dict.get(vuln_id, None)
if not updated_processed_data:
logger.error(f"Vuln data for {vuln_id} not found")

processed_data.update(host_vuln_dict)

# Insert vulnerabilities in a single batch
from faraday.server.api.modules.bulk_create import insert_vulnerabilities # pylint: disable=import-outside-toplevel

host_vulns = []
for vuln_id, data in processed_data.items():
vuln_data = data.get('vuln_data')
if vuln_data:
host_vulns.append(vuln_data)

if host_vulns:
result = insert_vulnerabilities(host_vulns, processed_data, workspace_id=workspace_id)
created_updated_count['created'] += result.get('created', 0)
created_updated_count['updated'] += result.get('updated', 0)

# Clear memory
del processed_data
del host_vulns

except Exception as e:
logger.error(f"Could not process vulnerabilities for host {host.ip}: {e}")

return created_updated_count


@celery.task(ignore_result=False)
def pre_process_report_task(workspace_name: str, command_id: int, file_path: str,
plugin_id: Optional[int], user_id: Optional[int], ignore_info: bool,
Expand Down
1 change: 0 additions & 1 deletion faraday/server/www
View file
Open in desktop

This file was deleted.

179 changes: 179 additions & 0 deletions faraday/server/www/asset-manifest.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,179 @@
{
"files": {
"main.css": "/static/css/main.8a701783.chunk.css",
"main.js": "/static/js/main.13158219.chunk.js",
"main.js.map": "/static/js/main.13158219.chunk.js.map",
"runtime-main.js": "/static/js/runtime-main.ea9693b3.js",
"runtime-main.js.map": "/static/js/runtime-main.ea9693b3.js.map",
"static/css/2.5f87b264.chunk.css": "/static/css/2.5f87b264.chunk.css",
"static/js/2.1a7ec6dd.chunk.js": "/static/js/2.1a7ec6dd.chunk.js",
"static/js/2.1a7ec6dd.chunk.js.map": "/static/js/2.1a7ec6dd.chunk.js.map",
"index.html": "/index.html",
"precache-manifest.01b3e25f17dc8cb4865c6216c61478d3.js": "/precache-manifest.01b3e25f17dc8cb4865c6216c61478d3.js",
"service-worker.js": "/service-worker.js",
"static/css/2.5f87b264.chunk.css.map": "/static/css/2.5f87b264.chunk.css.map",
"static/css/main.8a701783.chunk.css.map": "/static/css/main.8a701783.chunk.css.map",
"static/js/2.1a7ec6dd.chunk.js.LICENSE.txt": "/static/js/2.1a7ec6dd.chunk.js.LICENSE.txt",
"static/js/main.13158219.chunk.js.LICENSE.txt": "/static/js/main.13158219.chunk.js.LICENSE.txt",
"static/media/index.scss": "/static/media/Sequel_Sans_Head_SemiBold.48873506.woff",
"static/media/Checkbox.b83098ac.svg": "/static/media/Checkbox.b83098ac.svg",
"static/media/Checkbox_empty.svg": "/static/media/Checkbox_empty.195b4c92.svg",
"static/media/Checkbox_ok.svg": "/static/media/Checkbox_ok.52c5a6cc.svg",
"static/media/activity-dashboard.1ce5f4e0.svg": "/static/media/activity-dashboard.1ce5f4e0.svg",
"static/media/agent_small.d1a2c9fe.svg": "/static/media/agent_small.d1a2c9fe.svg",
"static/media/analytics.6f16d31d.svg": "/static/media/analytics.6f16d31d.svg",
"static/media/api_link.dcbdf06d.svg": "/static/media/api_link.dcbdf06d.svg",
"static/media/archive.svg": "/static/media/archive.d055fa7c.svg",
"static/media/arrow-down.d8661d88.svg": "/static/media/arrow-down.d8661d88.svg",
"static/media/arrow-right.ec48cf40.svg": "/static/media/arrow-right.ec48cf40.svg",
"static/media/assets.581bf13b.svg": "/static/media/assets.581bf13b.svg",
"static/media/automation.e231d6be.svg": "/static/media/automation.e231d6be.svg",
"static/media/blank_reports_view.0e9ed0e5.svg": "/static/media/blank_reports_view.0e9ed0e5.svg",
"static/media/check2.1bcc7b2f.svg": "/static/media/check2.1bcc7b2f.svg",
"static/media/clear.26371e5f.svg": "/static/media/clear.26371e5f.svg",
"static/media/codeql_logo.svg": "/static/media/codeql_logo.a59d6f4b.svg",
"static/media/cog.6f2e860c.svg": "/static/media/cog.6f2e860c.svg",
"static/media/copy.7e1b1006.svg": "/static/media/copy.7e1b1006.svg",
"static/media/copy_link.852fb400.svg": "/static/media/copy_link.852fb400.svg",
"static/media/crack_map_exec_logo.svg": "/static/media/crack_map_exec_logo.e6c66877.svg",
"static/media/dependabot_logo.svg": "/static/media/dependabot_logo.9c45605e.svg",
"static/media/details.366b7797.svg": "/static/media/details.366b7797.svg",
"static/media/device.16a667bc.svg": "/static/media/device.16a667bc.svg",
"static/media/download-icon.eadbcf9c.svg": "/static/media/download-icon.eadbcf9c.svg",
"static/media/drag.svg": "/static/media/drag.b17c0ec9.svg",
"static/media/empty-feed.png": "/static/media/empty-feed.3ebce7f3.png",
"static/media/error-mark.364d0169.svg": "/static/media/error-mark.364d0169.svg",
"static/media/evidence-default.2dcc3b35.svg": "/static/media/evidence-default.2dcc3b35.svg",
"static/media/exclamation_error.73360ffd.svg": "/static/media/exclamation_error.73360ffd.svg",
"static/media/f_round.9a0b094d.svg": "/static/media/f_round.9a0b094d.svg",
"static/media/faraday-logo-dark.svg": "/static/media/faraday-logo-dark.ce97099f.svg",
"static/media/faraday-logo-nav.8385ead1.svg": "/static/media/faraday-logo-nav.8385ead1.svg",
"static/media/faraday-logo.svg": "/static/media/faraday-logo.6f6c122b.svg",
"static/media/faraday_loadingbar.gif": "/static/media/faraday_loadingbar.13bdbd54.gif",
"static/media/faraday_logo.svg": "/static/media/faraday_logo.3af44fcb.svg",
"static/media/faraday_logo_product.57beba14.svg": "/static/media/faraday_logo_product.57beba14.svg",
"static/media/ff_2fa.png": "/static/media/ff_2fa.efd0850b.png",
"static/media/ff_analytics.png": "/static/media/ff_analytics.018317e3.png",
"static/media/ff_cloud_agents.png": "/static/media/ff_cloud_agents.5083dd56.png",
"static/media/ff_duplicates.png": "/static/media/ff_duplicates.46352b01.png",
"static/media/ff_ldap.png": "/static/media/ff_ldap.e96f786d.png",
"static/media/ff_planner.png": "/static/media/ff_planner.58bfcb97.png",
"static/media/ff_reporting.png": "/static/media/ff_reporting.9d5a9bb8.png",
"static/media/ff_saml.png": "/static/media/ff_saml.119fe76f.png",
"static/media/ff_tags.png": "/static/media/ff_tags.ccc5427b.png",
"static/media/ff_users.png": "/static/media/ff_users.e5bdca7e.png",
"static/media/generic_file.svg": "/static/media/generic_file.f9988671.svg",
"static/media/github_logo.svg": "/static/media/github_logo.0bb6cf15.svg",
"static/media/grid_view.580f3d12.svg": "/static/media/grid_view.580f3d12.svg",
"static/media/help-sysreq.5680b8ff.svg": "/static/media/help-sysreq.5680b8ff.svg",
"static/media/help.svg": "/static/media/help.61ace590.svg",
"static/media/home.59d70ac8.svg": "/static/media/home.59d70ac8.svg",
"static/media/ico-vulnerability-templates.0d9a4fc0.svg": "/static/media/ico-vulnerability-templates.0d9a4fc0.svg",
"static/media/ico-web-shell.fc841a37.svg": "/static/media/ico-web-shell.fc841a37.svg",
"static/media/icon-action-bar-column.efe44a0e.svg": "/static/media/icon-action-bar-column.efe44a0e.svg",
"static/media/icon-action-bar-edit.a362c51d.svg": "/static/media/icon-action-bar-edit.a362c51d.svg",
"static/media/icon-action-bar-more.fd9db67e.svg": "/static/media/icon-action-bar-more.fd9db67e.svg",
"static/media/icon-action-bar-plus.8879f3f6.svg": "/static/media/icon-action-bar-plus.8879f3f6.svg",
"static/media/icon-action-bar-tags.c0f5d2b3.svg": "/static/media/icon-action-bar-tags.c0f5d2b3.svg",
"static/media/icon-action-bar-token.d7973c77.svg": "/static/media/icon-action-bar-token.d7973c77.svg",
"static/media/icon-action-bar-trash.b1589206.svg": "/static/media/icon-action-bar-trash.b1589206.svg",
"static/media/icon-action-bar-trigger-disabled.e0400bf1.svg": "/static/media/icon-action-bar-trigger-disabled.e0400bf1.svg",
"static/media/icon-action-bar-trigger.f9c88d81.svg": "/static/media/icon-action-bar-trigger.f9c88d81.svg",
"static/media/icon-clipboard.c53f2936.svg": "/static/media/icon-clipboard.c53f2936.svg",
"static/media/icon-close-without-background.9b3b0c89.svg": "/static/media/icon-close-without-background.9b3b0c89.svg",
"static/media/icon-close.9a5cad5c.svg": "/static/media/icon-close.9a5cad5c.svg",
"static/media/icon-customfields.8bd11a47.svg": "/static/media/icon-customfields.8bd11a47.svg",
"static/media/icon-edit-enabled.a68d0409.svg": "/static/media/icon-edit-enabled.a68d0409.svg",
"static/media/icon-evidence.fdee90dd.svg": "/static/media/icon-evidence.fdee90dd.svg",
"static/media/icon-goto.cb712c64.svg": "/static/media/icon-goto.cb712c64.svg",
"static/media/icon-pipeline-run.b606975b.svg": "/static/media/icon-pipeline-run.b606975b.svg",
"static/media/icon-pipeline-running.86dfed50.svg": "/static/media/icon-pipeline-running.86dfed50.svg",
"static/media/icon-show-duplicates.79a1569d.svg": "/static/media/icon-show-duplicates.79a1569d.svg",
"static/media/icon-toolbar-confirmed-on.6c7a2996.svg": "/static/media/icon-toolbar-confirmed-on.6c7a2996.svg",
"static/media/icon-toolbar-delete.b342d6a5.svg": "/static/media/icon-toolbar-delete.b342d6a5.svg",
"static/media/icon-toolbar-searchbar-loupe.dadec58e.svg": "/static/media/icon-toolbar-searchbar-loupe.dadec58e.svg",
"static/media/icon-trash-red.1ebe0df4.svg": "/static/media/icon-trash-red.1ebe0df4.svg",
"static/media/icon_clearsearch.e7b18936.svg": "/static/media/icon_clearsearch.e7b18936.svg",
"static/media/icon_close_error.459e6ef9.svg": "/static/media/icon_close_error.459e6ef9.svg",
"static/media/icon_drag.d52758c1.svg": "/static/media/icon_drag.d52758c1.svg",
"static/media/icon_edit.97c1ea04.svg": "/static/media/icon_edit.97c1ea04.svg",
"static/media/icon_filter_off.c7a0db30.svg": "/static/media/icon_filter_off.c7a0db30.svg",
"static/media/icon_filter_on.6de198fd.svg": "/static/media/icon_filter_on.6de198fd.svg",
"static/media/icon_help.1b34b217.svg": "/static/media/icon_help.1b34b217.svg",
"static/media/icon_loupe.663ddfdd.svg": "/static/media/icon_loupe.663ddfdd.svg",
"static/media/icon_modal_asset.f360b291.svg": "/static/media/icon_modal_asset.f360b291.svg",
"static/media/icon_severity.0e650ab4.svg": "/static/media/icon_severity.0e650ab4.svg",
"static/media/icon_upload.cd99484a.svg": "/static/media/icon_upload.cd99484a.svg",
"static/media/icon_users.6fe4d738.svg": "/static/media/icon_users.6fe4d738.svg",
"static/media/info.225e604d.svg": "/static/media/info.225e604d.svg",
"static/media/jobs_pipelines_empty.png": "/static/media/jobs_pipelines_empty.6d79bb91.png",
"static/media/laptop_icon.svg": "/static/media/laptop_icon.21863406.svg",
"static/media/laquo.d5f6e3f8.svg": "/static/media/laquo.d5f6e3f8.svg",
"static/media/list_view.cacc94c7.svg": "/static/media/list_view.cacc94c7.svg",
"static/media/manage.5b4dcff2.svg": "/static/media/manage.5b4dcff2.svg",
"static/media/mini-terminal.svg": "/static/media/mini-terminal.f5a8e1d0.svg",
"static/media/minibot.15584cdc.svg": "/static/media/minibot.15584cdc.svg",
"static/media/moon-stars-svgrepo-com.a543e5a3.svg": "/static/media/moon-stars-svgrepo-com.a543e5a3.svg",
"static/media/moveVuln.f8f1d51a.svg": "/static/media/moveVuln.f8f1d51a.svg",
"static/media/new_vuln_modal_icon.033159cd.svg": "/static/media/new_vuln_modal_icon.033159cd.svg",
"static/media/next.0e62ddad.svg": "/static/media/next.0e62ddad.svg",
"static/media/next_arrow.407b029f.svg": "/static/media/next_arrow.407b029f.svg",
"static/media/noun-help.svg": "/static/media/noun-help.a93b0458.svg",
"static/media/preferences_icons_Account.e0a4aa09.svg": "/static/media/preferences_icons_Account.e0a4aa09.svg",
"static/media/preferences_icons_Authentication.1b59fc92.svg": "/static/media/preferences_icons_Authentication.1b59fc92.svg",
"static/media/preferences_icons_Ticketing.a1064973.svg": "/static/media/preferences_icons_Ticketing.a1064973.svg",
"static/media/prev.ecd34f23.svg": "/static/media/prev.ecd34f23.svg",
"static/media/queue.b922c0be.svg": "/static/media/queue.b922c0be.svg",
"static/media/reload.0d7f0f5c.svg": "/static/media/reload.0d7f0f5c.svg",
"static/media/report_small.e8a3d444.svg": "/static/media/report_small.e8a3d444.svg",
"static/media/reports.f38dca7f.svg": "/static/media/reports.f38dca7f.svg",
"static/media/resize_bottom_right.eced93cf.svg": "/static/media/resize_bottom_right.eced93cf.svg",
"static/media/reveal.80d070fa.svg": "/static/media/reveal.80d070fa.svg",
"static/media/save_template.d74f944a.svg": "/static/media/save_template.d74f944a.svg",
"static/media/scheduler_calendar.c9744ba3.svg": "/static/media/scheduler_calendar.c9744ba3.svg",
"static/media/services.0f83c980.svg": "/static/media/services.0f83c980.svg",
"static/media/shape.9ea9d3d6.svg": "/static/media/shape.9ea9d3d6.svg",
"static/media/showHide.4fbfe919.svg": "/static/media/showHide.4fbfe919.svg",
"static/media/sort.062a68ce.svg": "/static/media/sort.062a68ce.svg",
"static/media/star.33d47a05.svg": "/static/media/star.33d47a05.svg",
"static/media/status.660ac4e6.svg": "/static/media/status.660ac4e6.svg",
"static/media/sun-dm.ea7e2497.svg": "/static/media/sun-dm.ea7e2497.svg",
"static/media/tasks_icon.35dc01bc.svg": "/static/media/tasks_icon.35dc01bc.svg",
"static/media/template.bc90870d.svg": "/static/media/template.bc90870d.svg",
"static/media/tenable_sc_logo.svg": "/static/media/tenable_sc_logo.56985821.svg",
"static/media/ticketing-license.png": "/static/media/ticketing-license.7db0ca98.png",
"static/media/tool_cisco_cybervision.png": "/static/media/tool_cisco_cybervision.dd65c743.png",
"static/media/tool_logo_appscan.png": "/static/media/tool_logo_appscan.9b037d8b.png",
"static/media/tool_logo_arachni.png": "/static/media/tool_logo_arachni.ac5b15cb.png",
"static/media/tool_logo_insightVM.png": "/static/media/tool_logo_insightVM.f0295560.png",
"static/media/tool_logo_nessus.png": "/static/media/tool_logo_nessus.8ed3098d.png",
"static/media/tool_logo_nikto.png": "/static/media/tool_logo_nikto.9acc1459.png",
"static/media/tool_logo_nmap.png": "/static/media/tool_logo_nmap.9a81da35.png",
"static/media/tool_logo_openvas.png": "/static/media/tool_logo_openvas.88898507.png",
"static/media/tool_logo_qualys.png": "/static/media/tool_logo_qualys.8f7b2a7f.png",
"static/media/tool_logo_sonarQube.jpeg": "/static/media/tool_logo_sonarQube.3af1625a.jpeg",
"static/media/tool_logo_tenable.png": "/static/media/tool_logo_tenable.05c390dc.png",
"static/media/tool_logo_w3af.png": "/static/media/tool_logo_w3af.221ef0f2.png",
"static/media/tool_logo_wpscan.png": "/static/media/tool_logo_wpscan.92adc6c5.png",
"static/media/tool_logo_zap.png": "/static/media/tool_logo_zap.62021c39.png",
"static/media/tool_trash.e6d04612.svg": "/static/media/tool_trash.e6d04612.svg",
"static/media/tool_wheel.1b4bc890.svg": "/static/media/tool_wheel.1b4bc890.svg",
"static/media/unarchive.6e388838.svg": "/static/media/unarchive.6e388838.svg",
"static/media/unknown_filetype.f6b0014d.svg": "/static/media/unknown_filetype.f6b0014d.svg",
"static/media/vuln-kb.6ac0b0d7.svg": "/static/media/vuln-kb.6ac0b0d7.svg",
"static/media/vulnerabilities.bc957e73.svg": "/static/media/vulnerabilities.bc957e73.svg",
"static/media/warning-delete.svg": "/static/media/warning-delete.33357364.svg",
"static/media/wf_arrow_collapsed.51829f9b.svg": "/static/media/wf_arrow_collapsed.51829f9b.svg",
"static/media/wf_arrow_expand.06e6e9bd.svg": "/static/media/wf_arrow_expand.06e6e9bd.svg",
"static/media/workspaces-license.png": "/static/media/workspaces-license.e869b8a0.png",
"static/media/ws-lock.d994c131.svg": "/static/media/ws-lock.d994c131.svg",
"static/media/ws-unlock.b7653513.svg": "/static/media/ws-unlock.b7653513.svg"
},
"entrypoints": [
"static/js/runtime-main.ea9693b3.js",
"static/css/2.5f87b264.chunk.css",
"static/js/2.1a7ec6dd.chunk.js",
"static/css/main.8a701783.chunk.css",
"static/js/main.13158219.chunk.js"
]
}
Binary file added faraday/server/www/favicon.ico
View file
Open in desktop
Binary file not shown.
1 change: 1 addition & 0 deletions faraday/server/www/index.html
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
<!doctype html><html><head><meta charset="utf-8"><title>Faraday</title><link href="/static/css/2.5f87b264.chunk.css" rel="stylesheet"><link href="/static/css/main.8a701783.chunk.css" rel="stylesheet"></head><body style="overflow:auto"><div id="root"></div><script>!function(e){function r(r){for(var n,a,f=r[0],l=r[1],i=r[2],p=0,s=[];p<f.length;p++)a=f[p],Object.prototype.hasOwnProperty.call(o,a)&&o[a]&&s.push(o[a][0]),o[a]=0;for(n in l)Object.prototype.hasOwnProperty.call(l,n)&&(e[n]=l[n]);for(c&&c(r);s.length;)s.shift()();return u.push.apply(u,i||[]),t()}function t(){for(var e,r=0;r<u.length;r++){for(var t=u[r],n=!0,f=1;f<t.length;f++){var l=t[f];0!==o[l]&&(n=!1)}n&&(u.splice(r--,1),e=a(a.s=t[0]))}return e}var n={},o={1:0},u=[];function a(r){if(n[r])return n[r].exports;var t=n[r]={i:r,l:!1,exports:{}};return e[r].call(t.exports,t,t.exports,a),t.l=!0,t.exports}a.m=e,a.c=n,a.d=function(e,r,t){a.o(e,r)||Object.defineProperty(e,r,{enumerable:!0,get:t})},a.r=function(e){"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})},a.t=function(e,r){if(1&r&&(e=a(e)),8&r)return e;if(4&r&&"object"==typeof e&&e&&e.__esModule)return e;var t=Object.create(null);if(a.r(t),Object.defineProperty(t,"default",{enumerable:!0,value:e}),2&r&&"string"!=typeof e)for(var n in e)a.d(t,n,function(r){return e[r]}.bind(null,n));return t},a.n=function(e){var r=e&&e.__esModule?function(){return e.default}:function(){return e};return a.d(r,"a",r),r},a.o=function(e,r){return Object.prototype.hasOwnProperty.call(e,r)},a.p="/";var f=this["webpackJsonpfaraday-react"]=this["webpackJsonpfaraday-react"]||[],l=f.push.bind(f);f.push=r,f=f.slice();for(var i=0;i<f.length;i++)r(f[i]);var c=l;t()}([])</script><script src="/static/js/2.1a7ec6dd.chunk.js"></script><script src="/static/js/main.13158219.chunk.js"></script></body></html>
Loading