diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index ca7dbc75f0..94340c327d 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,97 +1,97 @@ # ODC code owners, refer to https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners # These owners will be the default owners for everything in the repo. -* @yhilmare @yizhouxw +* @LioRoger @guowl3 @MarkPotato777 # libs -/libs/db-browser/ @yhilmare @PeachThinking -/libs/ob-sql-parser/ @yhilmare @PeachThinking +/libs/db-browser/ @PeachThinking @MarkPotato777 +/libs/ob-sql-parser/ @PeachThinking @MarkPotato777 # 3rd-party -/server/3rd-party/ @MarkPotato777 @yhilmare +/server/3rd-party/ @MarkPotato777 @LioRoger # common -/server/odc-common/ @yhilmare @yizhouxw +/server/odc-common/ @LioRoger @guowl3 # migrate -/server/odc-migrate/ @MarkPotato777 @yhilmare +/server/odc-migrate/ @MarkPotato777 @LioRoger # core -/server/odc-core/ @yhilmare @yizhouxw +/server/odc-core/ @LioRoger @guowl3 /server/odc-core/**/alarm/ @LuckyPickleZZ -/server/odc-core/**/authority/ @yhilmare +/server/odc-core/**/authority/ @MarkPotato777 /server/odc-core/**/datamasking/ @LuckyPickleZZ -/server/odc-core/**/datasource/ @yhilmare -/server/odc-core/**/flow/ @yhilmare -/server/odc-core/**/alarm/ @LuckyPickleZZ @yizhouxw -/server/odc-core/**/authority/ @yhilmare @yizhouxw -/server/odc-core/**/datamasking/ @LuckyPickleZZ @yhilmare -/server/odc-core/**/datasource/ @yhilmare @yizhouxw -/server/odc-core/**/flow/ @yhilmare @yizhouxw -/server/odc-core/**/migrate/ @yhilmare @yizhouxw -/server/odc-core/**/session/ @yhilmare @yizhouxw -/server/odc-core/**/shared/ @yhilmare @yizhouxw -/server/odc-core/**/sql/ @yhilmare @LuckyPickleZZ -/server/odc-core/**/task/ @yhilmare @yizhouxw @guowl3 +/server/odc-core/**/datasource/ @MarkPotato777 +/server/odc-core/**/flow/ @zijiacj @LioRoger @MarkPotato777 +/server/odc-core/**/alarm/ @LuckyPickleZZ @LioRoger +/server/odc-core/**/authority/ @MarkPotato777 @LioRoger +/server/odc-core/**/datamasking/ @LuckyPickleZZ @MarkPotato777 +/server/odc-core/**/datasource/ @MarkPotato777 @LioRoger +/server/odc-core/**/flow/ @zijiacj @LioRoger @MarkPotato777 +/server/odc-core/**/migrate/ @MarkPotato777 @LioRoger +/server/odc-core/**/session/ @LuckyPickleZZ @LioRoger +/server/odc-core/**/shared/ @LioRoger @guowl3 +/server/odc-core/**/sql/ @LuckyPickleZZ @LioRoger +/server/odc-core/**/task/ @LioRoger @guowl3 # service common -/server/odc-service/ @yhilmare @yizhouxw -/server/odc-service/**/config/ @yhilmare @yizhouxw -/server/odc-service/**/metadb/ @yhilmare @yizhouxw -/server/odc-service/**/service/common/ @yhilmare @MarkPotato777 +/server/odc-service/ @LioRoger @guowl3 +/server/odc-service/**/config/ @LioRoger @guowl3 +/server/odc-service/**/metadb/ @LioRoger @guowl3 +/server/odc-service/**/service/common/ @LioRoger @MarkPotato777 # service business -/server/odc-service/**/service/audit/ @MarkPotato777 @yizhouxw +/server/odc-service/**/service/audit/ @MarkPotato777 @LioRoger /server/odc-service/**/service/automation/ @LuckyPickleZZ @ungreat -/server/odc-service/**/service/captcha/ @MarkPotato777 @yizhouxw -/server/odc-service/**/service/collaboration/ @MarkPotato777 @yizhouxw -/server/odc-service/**/service/config/ @MarkPotato777 @yizhouxw -/server/odc-service/**/service/datasecurity/ @LuckyPickleZZ @yhilmare -/server/odc-service/**/service/datatransfer/ @LuckyPickleZZ @yhilmare -/server/odc-service/**/service/db/ @PeachThinking @yhilmare -/server/odc-service/**/service/diagnose/ @LuckyPickleZZ @yizhouxw -/server/odc-service/**/service/dispatch/ @yhilmare @yizhouxw +/server/odc-service/**/service/captcha/ @MarkPotato777 @LioRoger +/server/odc-service/**/service/collaboration/ @MarkPotato777 @LioRoger +/server/odc-service/**/service/config/ @MarkPotato777 @LioRoger +/server/odc-service/**/service/datasecurity/ @LuckyPickleZZ @MarkPotato777 +/server/odc-service/**/service/datatransfer/ @LuckyPickleZZ @LioRoger +/server/odc-service/**/service/db/ @PeachThinking @LioRoger +/server/odc-service/**/service/diagnose/ @LuckyPickleZZ @LioRoger +/server/odc-service/**/service/dispatch/ @LioRoger @guowl3 /server/odc-service/**/service/dlm/ @guowl3 @kiko-art /server/odc-service/**/service/dml/ @LuckyPickleZZ @PeachThinking -/server/odc-service/**/service/encryption/ @PeachThinking @yizhouxw -/server/odc-service/**/service/feature/ @MarkPotato777 @yizhouxw -/server/odc-service/**/service/flow/ @yhilmare @yizhouxw +/server/odc-service/**/service/encryption/ @PeachThinking @LioRoger +/server/odc-service/**/service/feature/ @MarkPotato777 @LioRoger +/server/odc-service/**/service/flow/ @zijiacj @LioRoger @MarkPotato777 /server/odc-service/**/service/i18n/ @LuckyPickleZZ -/server/odc-service/**/service/iam/ @MarkPotato777 @PeachThinking @yhilmare -/server/odc-service/**/service/info/ @yhilmare @yizhouxw -/server/odc-service/**/service/integration/ @yiminpeng @ungreat @yizhouxw +/server/odc-service/**/service/iam/ @MarkPotato777 @PeachThinking @LioRoger +/server/odc-service/**/service/info/ @MarkPotato777 @LioRoger +/server/odc-service/**/service/integration/ @ungreat @LioRoger @yiminpeng /server/odc-service/**/service/lab/ @LuckyPickleZZ @ungreat -/server/odc-service/**/service/monitor/ @ungreat @yizhouxw +/server/odc-service/**/service/monitor/ @ungreat @ysjemmm @LioRoger /server/odc-service/**/service/notification/ @LuckyPickleZZ @MarkPotato777 -/server/odc-service/**/service/objectstorage/ @MarkPotato777 @yizhouxw +/server/odc-service/**/service/objectstorage/ @CHLK @MarkPotato777 /server/odc-service/**/service/onlineschemachange/ @LioRoger @LuckyPickleZZ -/server/odc-service/**/service/partitionplan/ @guowl3 @yhilmare -/server/odc-service/**/service/permission/ @MarkPotato777 @yhilmare -/server/odc-service/**/service/pldebug/ @yhilmare @yizhouxw -/server/odc-service/**/service/plugin/ @yhilmare @LuckyPickleZZ -/server/odc-service/**/service/quartz/ @guowl3 @yhilmare -/server/odc-service/**/service/requlation/ @MarkPotato777 @yhilmare -/server/odc-service/**/service/resourcegroup/ @MarkPotato777 @yhilmare +/server/odc-service/**/service/partitionplan/ @guowl3 @LioRoger +/server/odc-service/**/service/permission/ @MarkPotato777 @LioRoger +/server/odc-service/**/service/pldebug/ @zijiacj @MarkPotato777 +/server/odc-service/**/service/plugin/ @LioRoger @LuckyPickleZZ +/server/odc-service/**/service/quartz/ @guowl3 @LioRoger +/server/odc-service/**/service/requlation/ @MarkPotato777 @zijiacj +/server/odc-service/**/service/resourcegroup/ @MarkPotato777 @zijiacj /server/odc-service/**/service/resultset/ @LuckyPickleZZ @PeachThinking /server/odc-service/**/service/rollbackplan/ @PeachThinking @MarkPotato777 -/server/odc-service/**/service/schedule/ @guowl3 @yhilmare -/server/odc-service/**/service/script/ @LuckyPickleZZ @yizhouxw -/server/odc-service/**/service/session/ @yhilmare @LuckyPickleZZ +/server/odc-service/**/service/schedule/ @guowl3 @LioRoger +/server/odc-service/**/service/script/ @LuckyPickleZZ @LioRoger +/server/odc-service/**/service/session/ @LuckyPickleZZ @LioRoger /server/odc-service/**/service/shadowtable/ @MarkPotato777 @PeachThinking -/server/odc-service/**/service/snippet/ @LuckyPickleZZ @yizhouxw -/server/odc-service/**/service/sqlcheck/ @yhilmare @PeachThinking -/server/odc-service/**/service/structurecompare/ @PeachThinking @yhilmare -/server/odc-service/**/service/systemconfig/ @MarkPotato777 @yizhouxw -/server/odc-service/**/service/task/ @yhilmare @guowl3 @yizhouxw -/server/odc-service/**/service/websocket/ @LuckyPickleZZ @yizhouxw +/server/odc-service/**/service/snippet/ @LuckyPickleZZ @LioRoger +/server/odc-service/**/service/sqlcheck/ @zijiacj @PeachThinking @MarkPotato777 +/server/odc-service/**/service/structurecompare/ @PeachThinking @MarkPotato777 +/server/odc-service/**/service/systemconfig/ @MarkPotato777 @LioRoger +/server/odc-service/**/service/task/ @guowl3 @LioRoger +/server/odc-service/**/service/websocket/ @LuckyPickleZZ @LioRoger # plugins -/server/plugins/ @yhilmare @yizhouxw -/server/plugins/connect-plugin-doris/ @yhilmare @yizhouxw -/server/plugins/connect-plugin-mysql/ @yhilmare @yizhouxw -/server/plugins/connect-plugin-ob-mysql/ @yhilmare @yizhouxw -/server/plugins/connect-plugin-ob-oracle/ @yhilmare @yizhouxw -/server/plugins/connect-plugin-oracle/ @yhilmare @yizhouxw +/server/plugins/ @LioRoger @guowl3 @MarkPotato777 +/server/plugins/connect-plugin-doris/ @LioRoger @guowl3 @MarkPotato777 +/server/plugins/connect-plugin-mysql/ @LioRoger @guowl3 @MarkPotato777 +/server/plugins/connect-plugin-ob-mysql/ @LioRoger @guowl3 @MarkPotato777 +/server/plugins/connect-plugin-ob-oracle/ @LioRoger @guowl3 @MarkPotato777 +/server/plugins/connect-plugin-oracle/ @LioRoger @guowl3 @MarkPotato777 /server/plugins/schema-plugin-api/ @PeachThinking @MarkPotato777 /server/plugins/schema-plugin-doris/ @PeachThinking @MarkPotato777 @@ -101,37 +101,37 @@ /server/plugins/schema-plugin-odp-sharding-ob-mysql/ @PeachThinking @MarkPotato777 /server/plugins/schema-plugin-oracle/ @PeachThinking @MarkPotato777 -/server/plugins/task-plugin-api/ @LuckyPickleZZ @yhilmare -/server/plugins/task-plugin-doris/ @LuckyPickleZZ @yhilmare -/server/plugins/task-plugin-mysql/ @LuckyPickleZZ @yhilmare -/server/plugins/task-plugin-ob-mysql/ @LuckyPickleZZ @yhilmare -/server/plugins/task-plugin-ob-oracle/ @LuckyPickleZZ @yhilmare -/server/plugins/task-plugin-oracle/ @LuckyPickleZZ @yhilmare +/server/plugins/task-plugin-api/ @LuckyPickleZZ @LioRoger +/server/plugins/task-plugin-doris/ @LuckyPickleZZ @LioRoger +/server/plugins/task-plugin-mysql/ @LuckyPickleZZ @LioRoger +/server/plugins/task-plugin-ob-mysql/ @LuckyPickleZZ @LioRoger +/server/plugins/task-plugin-ob-oracle/ @LuckyPickleZZ @LioRoger +/server/plugins/task-plugin-oracle/ @LuckyPickleZZ @LioRoger # starters -/server/starters/ @yhilmare @yizhouxw -/server/starters/desktop-starter/ @yhilmare @yizhouxw -/server/starters/web-starter/ @yhilmare @MarkPotato777 +/server/starters/ @LioRoger @guowl3 @MarkPotato777 +/server/starters/desktop-starter/ @LioRoger @guowl3 @MarkPotato777 +/server/starters/web-starter/ @LioRoger @guowl3 @MarkPotato777 # modules -/server/modules/ @yhilmare @yizhouxw +/server/modules/ @LioRoger @CHLK # CI/CD -/.github/ @MarkPotato777 @yhilmare @yizhouxw -/builds/ @MarkPotato777 @yhilmare @yizhouxw -/script/ @MarkPotato777 @yhilmare @yizhouxw -/distribution/ @MarkPotato777 @yhilmare @yizhouxw -/server/odc-test/ @MarkPotato777 @yhilmare @yizhouxw -/server/integration-test/ @MarkPotato777 @yhilmare @yizhouxw -/server/test-script/ @MarkPotato777 @yhilmare @yizhouxw +/.github/ @MarkPotato777 @LioRoger +/builds/ @MarkPotato777 @LioRoger +/script/ @MarkPotato777 @LioRoger +/distribution/ @MarkPotato777 @LioRoger +/server/odc-test/ @MarkPotato777 @LioRoger +/server/integration-test/ @MarkPotato777 @LioRoger +/server/test-script/ @MarkPotato777 @LioRoger # i18n -/server/odc-core/src/main/resources/i18n/ @Jane201510 @JessieWuJiexi @yizhouxw +/server/odc-core/src/main/resources/i18n/ @Jane201510 @JessieWuJiexi @LioRoger @MarkPotato777 @guowl3 # docs -/docs/ @Jane201510 @yhilmare @yizhouxw -CHANGELOG.md @Jane201510 @JessieWuJiexi @yhilmare @yizhouxw -CHANGELOG-zh-CN.md @Jane201510 @JessieWuJiexi @yhilmare @yizhouxw -README.md @Jane201510 @JessieWuJiexi @yizhouxw -README-zh.md @Jane201510 @JessieWuJiexi @yizhouxw +/docs/ @Jane201510 @MarkPotato777 @LioRoger @guowl3 +CHANGELOG.md @Jane201510 @JessieWuJiexi @MarkPotato777 @LioRoger @guowl3 +CHANGELOG-zh-CN.md @Jane201510 @JessieWuJiexi @MarkPotato777 @LioRoger @guowl3 +README.md @Jane201510 @JessieWuJiexi @MarkPotato777 @LioRoger @guowl3 +README-zh.md @Jane201510 @JessieWuJiexi @MarkPotato777 @LioRoger @guowl3 diff --git a/.gitmodules b/.gitmodules index bb0072910f..f52bde7184 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,7 +1,7 @@ [submodule "client"] path = client url = https://github.com/oceanbase/odc-client.git - branch = dev-4.3.2 + branch = dev-4.3.3 [submodule "build-resource"] path = build-resource url = https://github.com/oceanbase/odc-build-resource.git diff --git a/CHANGELOG-zh-CN.md b/CHANGELOG-zh-CN.md index 3dbf7d1847..6f5e3708c7 100644 --- a/CHANGELOG-zh-CN.md +++ b/CHANGELOG-zh-CN.md @@ -1,4 +1,98 @@ # OceanBase Developer Center (ODC) CHANGELOG +## 4.3.3 (2025-01-13) + +### 功能变化 + +数据生命周期管理 + +- 新增 Oracle 到对象存储的归档链路 +- 新增 MySQL 到对象存储的归档链路 +- 新增 OceanBase MySQL 到对象存储的归档链路 +- 新增 OceanBase Oracle 到对象存储的归档链路 +- 新增 PostgreSQL 到对象存储的归档链路 +- 支持回溯编辑历史，支持查看编辑内容前后对比 +- 支持定义动态目标表，解决按日、月等单独存放历史数据的诉求 +- 支持删除数据归档、清理任务，当任务已完成或已终止时支持对其进行删除操作 +- 优化回滚逻辑，仅回滚当次任务的归档数据 + +无锁结构变更 + +- 支持失败重试，为各环节可能导致失败的场景补充重试逻辑 +- 支持无锁结构变更状态展示，可以查看运行中任务进度 + +变更风险管控 + +- 增加全局项目角色，包括全局项目管理员、全局安全管理员以及全局 DBA +- 增加项目归档检测机制，归档前会检测项目中是否存在未结束的工单及周期任务 +- 支持删除项目，对于已归档的项目支持对其进行删除操作 +- 支持用户申请视图权限，对用户访问视图做了更细粒度的权限控制 +- SQL 窗口拓展了可执行的 SQL 类型，新支持 `call`、`comment`、`set session` 等类型 +- SQL 检查规范支持原生 Oracle 数据源 +- 支持原生 Oracle 数据源的变更走变更审批流程 +- 新增 2 条 SQL 检查规则，支持规范 `create like` 及 `create as` 建表语句 + +SQL 开发 + +- 支持 OceanBase 外表白屏化管理 +- 支持 OceanBase 分区表的二级分区展示 +- 支持编辑 OceanBase MySQL 模式的函数和存储过程 +- 支持通过 OBProxy 进行 PL 调试 + +其他 + +- 支持 SAML 的单点登录方式 +- 支持查杀原生 Oracle 数据源的会话 +- 适配 OceanBase 4.2.5、4.3.3 版本 +- 适配 OBKV SQL 模式 +- 启用 Secure Cookie 机制，加固数据传输安全 +- 平台表单（含工单列表、数据库列表）列宽支持拉伸 + +### 易用性改进 +- 支持固化项目搜索条件，避免频繁搜索高频操作项目 +- 支持用户登出再登入后仍旧可以定位在最近使用的项目下，简化用户操作路径 +- 风险识别规则中判断条件文案优化，统一采用运算符及英文表达，以避免歧义 +- 优化连接保活逻辑，每3分钟会主动发送一次数据库请求，保障连接的稳定性 +- 项目外工单模块增加项目列，方便用户快速识别工单所属项目 +- 除逻辑库变更, 分区计划, 影子表外，所有工单类型支持再次发起功能，再次发起后支持二次编辑工单参数 +- 工单可被管理及查看范围调整，管理员和 DBA 可管理项目内所有工单，其它角色仅可管理自己发起的工单。同时项目内所有成员均可查看项目内所有工单 + + +### 缺陷修复 + +数据源 + +- 堡垒机集成场景不会同步 `information_schema` 等内置数据库到项目内 +- 数据库同步异常挂起时无法恢复 + +工单 + +- 创建数据归档工单在个人空间仍产生审批流程 +- 数据归档/清理任务执行成功但执行记录状态异常 +- 非当前账号创建结构对比任务无法正常执行 +- Oracle 导出表结构存在虚拟列时导出会失败 +- OceanBase MySQL 源端库或目标库里若有一张表的 DDL 里指定全文索引的分词器，结构比对任务失败 +- 定时任务如果有太多的子任务，查看操作记录失败问题 +- 导出任务保留当前配置不生效 + +变更管控 + +- 没有导出权限也能导出视图 +- 分区计划无法禁用导致无法归档项目 + +SQL 开发 + +- SQL Check 特定场景下产生 NPE 异常 +- DROP PL 需要数据库变更的权限 +- 函数返回值类型为 Year 时无法正常显示 +- 当 PL 名称包含 @ 时 create 和 drop 语句将失败 +- 查看原生 Oracle 扩展了统计信息（`DBMS_STATS.CREATE_EXTENDED_STATS`）的表详情失败 +- 限制 SQL 影响的行数时，insert 语句不生效 +- 导出数组函数结果集时，空指针异常问题 +- 在 Chrome 118 版本的浏览器中，右键单击软件包子程序时没有运行按钮 +- 查看程序包包头中的子程序时报错 + +其他 +- 用户再次进入 ODC 时没有打开上次使用的项目 ## 4.3.2 (2024-09-27) diff --git a/CHANGELOG.md b/CHANGELOG.md index 21dcf26463..f369e1c8c1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,102 @@ +# OceanBase Developer Center (ODC) CHANGELOG + +## 4.3.3 (2025-01-13) + +### Feature Changes + +**Data Lifecycle Management** + +* Added archiving paths from Oracle to Object Storage. +* Added archiving paths from MySQL to Object Storage. +* Added archiving paths from OceanBase MySQL to Object Storage. +* Added archiving paths from OceanBase Oracle to Object Storage. +* Added archiving paths from PostgreSQL to Object Storage. +* Added support for editing history review with content comparison. +* Introduced dynamic target table definition to support storing historical data by day, month, or other time units. +* Added ability to delete archiving and cleanup tasks when completed or terminated. +* Optimized rollback logic to only revert data archived in the current task. + +**Online Schema Change** + +* Added retry mechanism with enhanced retry logic for various failure scenarios. +* Added status display for online schema changes, allowing progress tracking of running tasks. + +**Change Risk Management** + +* Introduced global project roles including Global Project Admin, Global Security Admin, and Global DBA. +* Added project archiving validation to check for unfinished tickets and periodic tasks before archiving. +* Added support for project deletion for archived projects. +* Implemented fine-grained view permission control with user-initiated permission requests. +* Expanded executable SQL types in SQL window to include `call`, `comment`, `set session`, and more. +* Extended SQL check support for native Oracle data sources. +* Added change approval workflow for native Oracle data sources. +* Added 2 new SQL check rules for standardizing `create like` and `create as` table creation statements. + +**SQL Development** + +* Added GUI support for OceanBase external tables. +* Added support for displaying secondary partitions in OceanBase partitioned tables. +* Added support for editing stored procedures in OceanBase MySQL mode. +* Added support for PL debugging via OBProxy. + +**Other Enhancements** + +* Added SAML single sign-on support. +* Added session kill capability for native Oracle data sources. +* Added compatibility for OceanBase V4.2.5 and V4.3.3. +* Added support for OBKV SQL mode. +* Enabled secure cookie mechanism for enhanced data transmission security. +* Added column width adjustment support for forms (ticket lists, database lists). + +### Usability Improvements + +* Added persistent project search criteria to reduce frequent searches. +* Maintained last used project context across user sessions. +* Standardized risk identification rule conditions using operators and English expressions to avoid ambiguity. +* Enhanced connection keep-alive logic with 3-minute database request intervals. +* Added project column in non-project ticket module for quick project identification. +* Extended "Create Again" functionality to all ticket types except logical database changes, partition plans, and shadow tables. +* Refined ticket management scope: admins and DBAs can manage all project tickets, while other roles can only manage self-initiated tickets. Also, all members can view tickets in their projects. + +### Bug Fixes + +**Data Sources** + +* Fixed synchronization of system databases like `information_schema` to projects in bastion host integration scenarios. +* Resolved database synchronization suspension issues. + +**Tickets** + +* Fixed approval workflow triggering in personal workspace for data archiving tickets. +* Fixed status inconsistency in data archiving/cleanup task execution records. +* Fixed structure comparison task execution issues for non-current account users. +* Fixed Oracle table structure export failures with virtual columns. +* Fixed structure comparison failures for OceanBase MySQL tables with full-text index tokenizers. +* Fixed operation record viewing failures for periodic tasks with numerous subtasks. +* Fixed non-working configuration retention in export tasks. + +**Change Management** + +* Fixed unauthorized view exports. +* Fixed partition plan disable issues affecting project archiving. + +**SQL Development** + +* Fixed NPE exceptions in specific SQL check scenarios. +* Fixed PL drop requiring database change permissions. +* Fixed display issues for functions with year return type. +* Fixed PL creation and drop failures with @ in names. +* Fixed table detail viewing failures for Oracle tables with extended statistics (`DBMS_STATS.CREATE_EXTENDED_STATS`). +* Fixed ineffective row limit for Insert statements. +* Fixed null pointer exceptions when exporting array function result sets. +* Fixed missing run button for package subprocedures in Chrome 118. +* Fixed error when viewing subprocedures in package headers. + +**Other** + +* Fixed last used project not opening on subsequent ODC access. + + ## 4.3.2 (2024-09-27) ### Feature Changes diff --git a/client b/client new file mode 160000 index 0000000000..5483141527 --- /dev/null +++ b/client @@ -0,0 +1 @@ +Subproject commit 5483141527674c54261206470deddcdf96aaead7 diff --git a/distribution/odc-server-VER.txt b/distribution/odc-server-VER.txt index cc2fbe89b6..e91d9be2a8 100644 --- a/distribution/odc-server-VER.txt +++ b/distribution/odc-server-VER.txt @@ -1 +1 @@ -4.3.2 +4.3.3 diff --git a/docs/en-US/DEVELOPER_GUIDE.md b/docs/en-US/DEVELOPER_GUIDE.md index dfad2e1808..c4c67059d7 100644 --- a/docs/en-US/DEVELOPER_GUIDE.md +++ b/docs/en-US/DEVELOPER_GUIDE.md @@ -407,6 +407,11 @@ The setup for starting OdcServer is shown below. ![image.png](../en-US/images/idea-run-configuration-start-odc-server-2.png) +The setup for environment variables is shown below. + +![image.png](../en-US/images/idea-run-configuration-start-odc-server-3.png) + + # 4. Frontend-backend integration ## 4.1 Front-end and back-end integration based on a static resource server diff --git a/docs/en-US/images/idea-run-configuration-start-odc-server-2.png b/docs/en-US/images/idea-run-configuration-start-odc-server-2.png index 45371ba394..2204690cff 100644 Binary files a/docs/en-US/images/idea-run-configuration-start-odc-server-2.png and b/docs/en-US/images/idea-run-configuration-start-odc-server-2.png differ diff --git a/docs/en-US/images/idea-run-configuration-start-odc-server-3.png b/docs/en-US/images/idea-run-configuration-start-odc-server-3.png new file mode 100644 index 0000000000..e6a88556c8 Binary files /dev/null and b/docs/en-US/images/idea-run-configuration-start-odc-server-3.png differ diff --git a/docs/zh-CN/DEVELOPER_GUIDE.md b/docs/zh-CN/DEVELOPER_GUIDE.md index c118e03a2f..752779a763 100644 --- a/docs/zh-CN/DEVELOPER_GUIDE.md +++ b/docs/zh-CN/DEVELOPER_GUIDE.md @@ -377,6 +377,11 @@ OdcServer 启动设置示意如下 ![image.png](../en-US/images/idea-run-configuration-start-odc-server-2.png) +环境变量（environment variables）配置示意如下 + +![image.png](../en-US/images/idea-run-configuration-start-odc-server-3.png) + + # 4. 前后端集成和联调 ## 4.1 基于静态资源服务器的前后端联调 diff --git a/libs/db-browser/pom.xml b/libs/db-browser/pom.xml index a40d4157de..a571e450d3 100644 --- a/libs/db-browser/pom.xml +++ b/libs/db-browser/pom.xml @@ -4,7 +4,7 @@ 4.0.0 com.oceanbase db-browser - 1.2.0 + 1.2.2 db-browser https://github.com/oceanbase/odc/tree/main/libs/db-browser diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/model/DBMySQLProcess.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/model/DBMySQLProcess.java index 82d0d92b5f..6bcf741cfc 100644 --- a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/model/DBMySQLProcess.java +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/model/DBMySQLProcess.java @@ -15,6 +15,8 @@ */ package com.oceanbase.tools.dbbrowser.model; +import com.oceanbase.tools.dbbrowser.util.StringUtils; + import lombok.Data; /** @@ -55,6 +57,16 @@ public class DBMySQLProcess { */ private String host; + /** + * observer ip, refer to the column `Ip` in the result set of `SHOW FULL PROCESSLIST` + */ + private String ip; + + /** + * observer SQL port, refer to the column `Port` in the result set of `SHOW FULL PROCESSLIST` + */ + private String port; + public DBSession toDBSession() { DBSession session = new DBSession(); session.setId(id); @@ -66,6 +78,7 @@ public DBSession toDBSession() { session.setHost(host); session.setProxyHost(host); session.setExecuteTime(Integer.parseInt(time)); + session.setSvrIp(StringUtils.join(ip, ":", port)); return session; } } diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/model/DBObjectType.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/model/DBObjectType.java index 7b2de36182..caeaa8f612 100644 --- a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/model/DBObjectType.java +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/model/DBObjectType.java @@ -24,6 +24,7 @@ public enum DBObjectType { */ SCHEMA("SCHEMA"), TABLE("TABLE"), + EXTERNAL_TABLE("EXTERNAL TABLE"), LOGICAL_TABLE("LOGICAL TABLE"), COLUMN("COLUMN"), INDEX("INDEX"), diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/model/DBTable.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/model/DBTable.java index 605363d4b0..debb2f6ddb 100644 --- a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/model/DBTable.java +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/model/DBTable.java @@ -58,6 +58,8 @@ public class DBTable implements DBObject, DBObjectWarningDescriptor { private String warning; + private DBObjectType type; + @Data public static class DBTableOptions { private String charsetName; diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/model/DBTableColumn.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/model/DBTableColumn.java index fe019fa42f..65865fdc1f 100644 --- a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/model/DBTableColumn.java +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/model/DBTableColumn.java @@ -122,7 +122,7 @@ public class DBTableColumn implements DBObject, DBObjectWarningDescriptor { private String collationName; /** - * MySQL special + * The generation column's expression */ private String genExpression; diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/model/DBTablePartitionDefinition.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/model/DBTablePartitionDefinition.java index 75851e560d..6879a7f437 100644 --- a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/model/DBTablePartitionDefinition.java +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/model/DBTablePartitionDefinition.java @@ -19,4 +19,5 @@ @Data public class DBTablePartitionDefinition extends DBTableAbstractPartitionDefinition { + private DBTablePartitionDefinition parentPartitionDefinition; } diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/parser/ParserUtil.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/parser/ParserUtil.java index 5611084cbf..118b4c4caa 100644 --- a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/parser/ParserUtil.java +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/parser/ParserUtil.java @@ -55,6 +55,7 @@ public static GeneralSqlType getGeneralSqlType(BasicResult result) { case DROP: case TRUNCATE: case ALTER: + case COMMENT_ON: return GeneralSqlType.DDL; case UNKNOWN: default: diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/parser/constant/SqlType.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/parser/constant/SqlType.java index f7148310fe..56e743f5e8 100644 --- a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/parser/constant/SqlType.java +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/parser/constant/SqlType.java @@ -19,25 +19,39 @@ * Created by mogao.zj */ public enum SqlType { - SELECT, - DELETE, - INSERT, - REPLACE, - UPDATE, - SET, - USE_DB, - EXPLAIN, - SHOW, - HELP, - START_TRANS, - COMMIT, - ROLLBACK, - SORT, - DESC, - DROP, - ALTER, - TRUNCATE, - CREATE, - OTHERS, - UNKNOWN + SELECT(null), + DELETE(null), + INSERT(null), + REPLACE(null), + UPDATE(null), + SET(null), + SET_SESSION(SET), + USE_DB(null), + EXPLAIN(null), + SHOW(null), + HELP(null), + START_TRANS(null), + COMMIT(null), + ROLLBACK(null), + SORT(null), + DESC(null), + DROP(null), + ALTER(null), + ALTER_SESSION(ALTER), + TRUNCATE(null), + CREATE(null), + CALL(null), + COMMENT_ON(null), + OTHERS(null), + UNKNOWN(null); + + private final SqlType parentType; + + SqlType(SqlType parentType) { + this.parentType = parentType; + } + + public SqlType getParentType() { + return parentType; + } } diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/parser/listener/MysqlModePLParserListener.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/parser/listener/MysqlModePLParserListener.java index 8c59c2ad68..5fd374b743 100644 --- a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/parser/listener/MysqlModePLParserListener.java +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/parser/listener/MysqlModePLParserListener.java @@ -159,4 +159,11 @@ public void enterSp_decl(Sp_declContext ctx) { varibale.setVarType(type); this.varibaleList.add(varibale); } + + @Override + public void enterCall_sp_stmt(PLParser.Call_sp_stmtContext ctx) { + this.sqlType = SqlType.CALL; + this.dbObjectType = DBObjectType.PROCEDURE; + } + } diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/parser/listener/MysqlModeSqlParserListener.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/parser/listener/MysqlModeSqlParserListener.java index 40f3745470..f5ab1727db 100644 --- a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/parser/listener/MysqlModeSqlParserListener.java +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/parser/listener/MysqlModeSqlParserListener.java @@ -228,6 +228,7 @@ public void enterScope_or_scope_alias(Scope_or_scope_aliasContext ctx) { if (scopeToken.getType() == OBLexer.GLOBAL || scopeToken.getType() == OBLexer.GLOBAL_ALIAS) { setDbObjectType(DBObjectType.GLOBAL_VARIABLE); } else if (scopeToken.getType() == OBLexer.SESSION || scopeToken.getType() == OBLexer.SESSION_ALIAS) { + this.sqlType = SqlType.SET_SESSION; setDbObjectType(DBObjectType.SESSION_VARIABLE); } } diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/parser/listener/OracleModePLParserListener.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/parser/listener/OracleModePLParserListener.java index 7568f3da48..77474ec04d 100644 --- a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/parser/listener/OracleModePLParserListener.java +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/parser/listener/OracleModePLParserListener.java @@ -555,6 +555,12 @@ public void enterAlter_package_stmt(Alter_package_stmtContext ctx) { } } + @Override + public void enterCall_spec(PLParser.Call_specContext ctx) { + this.sqlType = sqlType.CALL; + this.dbObjectType = DBObjectType.PROCEDURE; + } + private String getDdl(ParserRuleContext ctx) { Token start = ctx.getStart(); Token stop = ctx.getStop(); diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/parser/listener/OracleModeParserListener.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/parser/listener/OracleModeParserListener.java index d0dc419617..e20bb82060 100644 --- a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/parser/listener/OracleModeParserListener.java +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/parser/listener/OracleModeParserListener.java @@ -68,7 +68,6 @@ import com.oceanbase.tools.sqlparser.oracle.PlSqlParser.Procedure_bodyContext; import com.oceanbase.tools.sqlparser.oracle.PlSqlParser.Procedure_nameContext; import com.oceanbase.tools.sqlparser.oracle.PlSqlParser.Procedure_specContext; -import com.oceanbase.tools.sqlparser.oracle.PlSqlParser.Rollback_statementContext; import com.oceanbase.tools.sqlparser.oracle.PlSqlParser.Type_declarationContext; import com.oceanbase.tools.sqlparser.oracle.PlSqlParser.Variable_declarationContext; import com.oceanbase.tools.sqlparser.oracle.PlSqlParserBaseListener; @@ -418,6 +417,28 @@ public void enterRollback_statement(PlSqlParser.Rollback_statementContext ctx) { doRecord(SqlType.ROLLBACK, DBObjectType.OTHERS, null); } + @Override + public void enterComment_on_column(PlSqlParser.Comment_on_columnContext ctx) { + doRecord(SqlType.COMMENT_ON, DBObjectType.COLUMN, null); + } + + @Override + public void enterComment_on_table(PlSqlParser.Comment_on_tableContext ctx) { + doRecord(SqlType.COMMENT_ON, DBObjectType.TABLE, null); + } + + @Override + public void enterComment_on_materialized(PlSqlParser.Comment_on_materializedContext ctx) { + doRecord(SqlType.COMMENT_ON, DBObjectType.OTHERS, null); + } + + @Override + public void enterProcedure_call(PlSqlParser.Procedure_callContext ctx) { + if (ctx.CALL() != null) { + doRecord(SqlType.CALL, DBObjectType.PROCEDURE, null); + } + } + private void doRecord(SqlType sqlType, DBObjectType dbObjectType, String rawPlName) { if (Objects.nonNull(sqlType) && Objects.isNull(this.sqlType)) { this.sqlType = sqlType; @@ -431,6 +452,18 @@ private void doRecord(SqlType sqlType, DBObjectType dbObjectType, String rawPlNa } } + @Override + public void enterAlter_session(PlSqlParser.Alter_sessionContext ctx) { + doRecord(SqlType.ALTER_SESSION, DBObjectType.OTHERS, null); + } + + @Override + public void enterScope_or_scope_alias(PlSqlParser.Scope_or_scope_aliasContext ctx) { + if (ctx.SESSION() != null) { + doRecord(SqlType.SET_SESSION, DBObjectType.OTHERS, null); + } + } + private String getDdl(ParserRuleContext ctx) { Token start = ctx.getStart(); Token stop = ctx.getStop(); diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/parser/listener/OracleModeSqlParserListener.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/parser/listener/OracleModeSqlParserListener.java index 0ef7c736ac..e6274c7ea4 100644 --- a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/parser/listener/OracleModeSqlParserListener.java +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/parser/listener/OracleModeSqlParserListener.java @@ -284,6 +284,7 @@ public void enterScope_or_scope_alias(Scope_or_scope_aliasContext ctx) { if (scopeToken.getType() == OBLexer.GLOBAL || scopeToken.getType() == OBLexer.GLOBAL_ALIAS) { setDbObjectType(DBObjectType.GLOBAL_VARIABLE); } else if (scopeToken.getType() == OBLexer.SESSION || scopeToken.getType() == OBLexer.SESSION_ALIAS) { + sqlType = sqlType.SET_SESSION; setDbObjectType(DBObjectType.SESSION_VARIABLE); } } @@ -601,7 +602,7 @@ public void enterAlter_sequence_stmt(Alter_sequence_stmtContext ctx) { @Override public void enterAlter_session_stmt(Alter_session_stmtContext ctx) { - setSqlType(SqlType.ALTER); + setSqlType(SqlType.ALTER_SESSION); this.dbObjectType = DBObjectType.OTHERS; } @@ -700,4 +701,25 @@ public void enterRollback_stmt(Rollback_stmtContext ctx) { this.dbObjectType = DBObjectType.OTHERS; } + @Override + public void enterSet_comment_stmt(OBParser.Set_comment_stmtContext ctx) { + // comment on xxx + setSqlType(SqlType.COMMENT_ON); + if (ctx.TABLE() != null) { + this.dbObjectType = DBObjectType.TABLE; + return; + } + if (ctx.COLUMN() != null) { + this.dbObjectType = DBObjectType.COLUMN; + return; + } + this.dbObjectType = DBObjectType.OTHERS; + } + + @Override + public void enterCall_stmt(OBParser.Call_stmtContext ctx) { + setSqlType(SqlType.CALL); + this.dbObjectType = DBObjectType.PROCEDURE; + } + } diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/DBSchemaAccessor.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/DBSchemaAccessor.java index 21d68f020f..eb9bac6ab5 100644 --- a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/DBSchemaAccessor.java +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/DBSchemaAccessor.java @@ -81,6 +81,32 @@ default List showTables(String schemaName) { */ List listTables(String schemaName, String tableNameLike); + /** + * Show all external table names list in the specified schema + */ + default List showExternalTables(String schemaName) { + return showExternalTablesLike(schemaName, null); + } + + List showExternalTablesLike(String schemaName, String tableNameLike); + + /** + * List all external table as BObjectIdentity in the specified schema + */ + List listExternalTables(String schemaName, String tableNameLike); + + /** + * Judge whether the table is an external table. If the current data source does not support + * external table feature,return false + */ + boolean isExternalTable(String schemaName, String tableName); + + /** + * Synchronize the associated files of external table + */ + boolean syncExternalTableFiles(String schemaName, String tableName); + + /** * Show all view names list in the specified schema */ @@ -198,6 +224,16 @@ default List showTables(String schemaName) { */ List listBasicViewColumns(String schemaName, String viewName); + /** + * Get all external table columns(hold only basic info) in the specified schema + */ + Map> listBasicExternalTableColumns(String schemaName); + + /** + * Get all external table columns(hold only basic info) in the specified schema and view + */ + List listBasicExternalTableColumns(String schemaName, String externalTableName); + /** * Get all table and view columns info (hold only basic info: schema, table and column name) in the * specified schema diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/DBSchemaAccessorFactory.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/DBSchemaAccessorFactory.java index fd0bbbd6a0..616bf55b01 100644 --- a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/DBSchemaAccessorFactory.java +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/DBSchemaAccessorFactory.java @@ -30,10 +30,12 @@ import com.oceanbase.tools.dbbrowser.schema.mysql.OBMySQLBetween220And225XSchemaAccessor; import com.oceanbase.tools.dbbrowser.schema.mysql.OBMySQLBetween2260And2276SchemaAccessor; import com.oceanbase.tools.dbbrowser.schema.mysql.OBMySQLBetween2277And3XSchemaAccessor; +import com.oceanbase.tools.dbbrowser.schema.mysql.OBMySQLBetween400And432SchemaAccessor; import com.oceanbase.tools.dbbrowser.schema.mysql.OBMySQLNoGreaterThan1479SchemaAccessor; import com.oceanbase.tools.dbbrowser.schema.mysql.OBMySQLSchemaAccessor; import com.oceanbase.tools.dbbrowser.schema.mysql.ODPOBMySQLSchemaAccessor; import com.oceanbase.tools.dbbrowser.schema.oracle.OBOracleBetween4000And4100SchemaAccessor; +import com.oceanbase.tools.dbbrowser.schema.oracle.OBOracleBetween410And432SchemaAccessor; import com.oceanbase.tools.dbbrowser.schema.oracle.OBOracleLessThan2270SchemaAccessor; import com.oceanbase.tools.dbbrowser.schema.oracle.OBOracleLessThan400SchemaAccessor; import com.oceanbase.tools.dbbrowser.schema.oracle.OBOracleSchemaAccessor; @@ -72,9 +74,12 @@ public DBSchemaAccessor buildForMySQL() { @Override public DBSchemaAccessor buildForOBMySQL() { Validate.notNull(this.dbVersion, "DBVersion can not be null"); - if (VersionUtils.isGreaterThanOrEqualsTo(this.dbVersion, "4.0.0")) { - // OB version >= 4.0.0 + if (VersionUtils.isGreaterThanOrEqualsTo(this.dbVersion, "4.3.2")) { + // OB version >= 4.3.2 return new OBMySQLSchemaAccessor(getJdbcOperations()); + } else if (VersionUtils.isGreaterThan(this.dbVersion, "4.0.0")) { + // OB version between [4.0.0, 4.3.2) + return new OBMySQLBetween400And432SchemaAccessor(getJdbcOperations()); } else if (VersionUtils.isGreaterThan(this.dbVersion, "2.2.76")) { // OB version between [2.2.77, 4.0.0) return new OBMySQLBetween2277And3XSchemaAccessor(getJdbcOperations()); @@ -105,9 +110,12 @@ public DBSchemaAccessor buildForOBMySQL() { @Override public DBSchemaAccessor buildForOBOracle() { Validate.notNull(this.dbVersion, "DBVersion can not be null"); - if (VersionUtils.isGreaterThanOrEqualsTo(this.dbVersion, "4.1.0")) { - // OB version >= 4.1.0 + if (VersionUtils.isGreaterThanOrEqualsTo(this.dbVersion, "4.3.2")) { + // OB version >= 4.3.2 return new OBOracleSchemaAccessor(getJdbcOperations(), new ALLDataDictTableNames()); + } else if (VersionUtils.isGreaterThanOrEqualsTo(this.dbVersion, "4.1.0")) { + // OB version between [4.1.0, 4.3.2) + return new OBOracleBetween410And432SchemaAccessor(getJdbcOperations(), new ALLDataDictTableNames()); } else if (VersionUtils.isGreaterThanOrEqualsTo(this.dbVersion, "4.0.0")) { // OB version between [4.0.0, 4.1.0) return new OBOracleBetween4000And4100SchemaAccessor(getJdbcOperations(), new ALLDataDictTableNames()); diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/DBSchemaAccessorSqlMappers.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/DBSchemaAccessorSqlMappers.java index f2c9dc8473..eab8b54a3f 100644 --- a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/DBSchemaAccessorSqlMappers.java +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/DBSchemaAccessorSqlMappers.java @@ -46,7 +46,7 @@ public class DBSchemaAccessorSqlMappers { static { SQL_MAPPER_FILE_PATHS.addAll(Arrays.asList( - StatementsFiles.OBMYSQL_40X, + StatementsFiles.OBMYSQL_432x, StatementsFiles.OBMYSQL_3X, StatementsFiles.OBMYSQL_2276, StatementsFiles.OBMYSQL_225X, @@ -56,6 +56,7 @@ public class DBSchemaAccessorSqlMappers { StatementsFiles.OBORACLE_3_x, StatementsFiles.OBORACLE_4_0_x, StatementsFiles.OBORACLE_4_1_x, + StatementsFiles.OBORACLE_4_3_2_x, StatementsFiles.ORACLE_11_g)); for (String path : SQL_MAPPER_FILE_PATHS) { URL url = DBSchemaAccessorSqlMappers.class.getClassLoader().getResource(path); diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/constant/Statements.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/constant/Statements.java index edacfc2a80..61521e4982 100644 --- a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/constant/Statements.java +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/constant/Statements.java @@ -21,6 +21,8 @@ * @Description: [] */ public final class Statements { + public static final String LIST_BASIC_EXTERNAL_TABLE_COLUMNS = "list-basic-external-table-columns"; + public static final String LIST_BASIC_SCHEMA_EXTERNAL_TABLE_COLUMNS = "list-basic-schema-external-table-columns"; public static final String LIST_BASIC_TABLE_COLUMNS = "list-basic-table-columns"; public static final String LIST_BASIC_SCHEMA_TABLE_COLUMNS = "list-basic-schema-table-columns"; public static final String LIST_BASIC_VIEW_COLUMNS = "list-basic-view-columns"; diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/constant/StatementsFiles.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/constant/StatementsFiles.java index b85ca26295..d5b2c63895 100644 --- a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/constant/StatementsFiles.java +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/constant/StatementsFiles.java @@ -21,7 +21,7 @@ * @Description: [] */ public final class StatementsFiles { - public static final String OBMYSQL_40X = "schema/sql/obmysql/obmysql_4_0_x.yaml"; + public static final String OBMYSQL_432x = "schema/sql/obmysql/obmysql_4_3_2_x.yaml"; public static final String OBMYSQL_3X = "schema/sql/obmysql/obmysql_3_x.yaml"; @@ -35,6 +35,8 @@ public final class StatementsFiles { public static final String MYSQL_5_6_x = "schema/sql/mysql/mysql_5_6_x.yaml"; + public static final String OBORACLE_4_3_2_x = "schema/sql/oboracle/oboracle_4_3_2_x.yaml"; + public static final String OBORACLE_4_1_x = "schema/sql/oboracle/oboracle_4_1_x.yaml"; public static final String OBORACLE_4_0_x = "schema/sql/oboracle/oboracle_4_0_x.yaml"; diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/doris/DorisSchemaAccessor.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/doris/DorisSchemaAccessor.java index 90ce3b070e..6ac4355a26 100644 --- a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/doris/DorisSchemaAccessor.java +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/doris/DorisSchemaAccessor.java @@ -225,6 +225,26 @@ public List listTables(String schemaName, String tableNameLike return results; } + @Override + public List showExternalTablesLike(String schemaName, String tableNameLike) { + throw new UnsupportedOperationException("Not supported yet"); + } + + @Override + public List listExternalTables(String schemaName, String tableNameLike) { + throw new UnsupportedOperationException("Not supported yet"); + } + + @Override + public boolean isExternalTable(String schemaName, String tableName) { + return false; + } + + @Override + public boolean syncExternalTableFiles(String schemaName, String tableName) { + throw new UnsupportedOperationException("Not supported yet"); + } + protected List listBaseTables(String schemaName, String tableNameLike) throws DataAccessException { MySQLSqlBuilder sb = new MySQLSqlBuilder(); @@ -452,6 +472,16 @@ public List listBasicViewColumns(String schemaName, String viewNa return jdbcOperations.query(sql, new Object[] {schemaName, viewName}, listBasicTableColumnRowMapper()); } + @Override + public Map> listBasicExternalTableColumns(String schemaName) { + throw new UnsupportedOperationException("Not supported yet"); + } + + @Override + public List listBasicExternalTableColumns(String schemaName, String externalTableName) { + throw new UnsupportedOperationException("Not supported yet"); + } + @Override public Map> listBasicColumnsInfo(String schemaName) { String sql = sqlMapper.getSql(Statements.LIST_BASIC_SCHEMA_COLUMNS_INFO); diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/mysql/MySQLNoLessThan5700SchemaAccessor.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/mysql/MySQLNoLessThan5700SchemaAccessor.java index 7301ab76e9..5fc03fca41 100644 --- a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/mysql/MySQLNoLessThan5700SchemaAccessor.java +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/mysql/MySQLNoLessThan5700SchemaAccessor.java @@ -233,6 +233,32 @@ public List listTables(String schemaName, String tableNameLike return listBaseTables(schemaName, tableNameLike); } + @Override + public List showExternalTables(String schemaName) { + throw new UnsupportedOperationException("Not supported yet"); + } + + + @Override + public List showExternalTablesLike(String schemaName, String tableNameLike) { + throw new UnsupportedOperationException("Not supported yet"); + } + + @Override + public List listExternalTables(String schemaName, String tableNameLike) { + throw new UnsupportedOperationException("Not supported yet"); + } + + @Override + public boolean isExternalTable(String schemaName, String tableName) { + return false; + } + + @Override + public boolean syncExternalTableFiles(String schemaName, String tableName) { + throw new UnsupportedOperationException("Not supported yet"); + } + protected List listBaseTables(String schemaName, String tableNameLike) throws DataAccessException { MySQLSqlBuilder sb = new MySQLSqlBuilder(); @@ -467,6 +493,16 @@ public List listBasicViewColumns(String schemaName, String viewNa return jdbcOperations.query(sql, new Object[] {schemaName, viewName}, listBasicTableColumnRowMapper()); } + @Override + public Map> listBasicExternalTableColumns(String schemaName) { + throw new UnsupportedOperationException("not support yet"); + } + + @Override + public List listBasicExternalTableColumns(String schemaName, String externalTableName) { + throw new UnsupportedOperationException("not support yet"); + } + @Override public Map> listBasicColumnsInfo(String schemaName) { String sql = sqlMapper.getSql(Statements.LIST_BASIC_SCHEMA_COLUMNS_INFO); @@ -1297,31 +1333,24 @@ public DBProcedure getProcedure(String schemaName, String procedureName) { .value(schemaName) .append(" and ROUTINE_TYPE = 'PROCEDURE' and ROUTINE_NAME=") .value(procedureName); - - MySQLSqlBuilder queryForParameters = new MySQLSqlBuilder(); - queryForParameters.append( - "select PARAMETER_MODE, PARAMETER_NAME, DTD_IDENTIFIER from `information_schema`.`parameters` where SPECIFIC_SCHEMA=") - .value(schemaName) - .append(" and SPECIFIC_NAME=") - .value(procedureName) - .append(" and ROUTINE_TYPE='PROCEDURE'"); DBProcedure procedure = new DBProcedure(); procedure.setProName(procedureName); - MySQLSqlBuilder parameters = new MySQLSqlBuilder(); - - jdbcOperations.query(queryForParameters.toString(), (rs) -> { - parameters.append(rs.getString("PARAMETER_MODE")).space() - .identifier(rs.getString("PARAMETER_NAME")).space() - .append(rs.getString("DTD_IDENTIFIER")).append(","); + MySQLSqlBuilder getDDL = new MySQLSqlBuilder(); + getDDL.append("show create procedure "); + if (schemaName == null) { + getDDL.identifier(procedureName); + } else { + getDDL.identifier(schemaName); + getDDL.append("."); + getDDL.identifier(procedureName); + } + jdbcOperations.query(getDDL.toString(), (rs) -> { + procedure.setDdl(rs.getString("Create Procedure")); }); jdbcOperations.query(sql1.toString(), (rs) -> { procedure.setDefiner(rs.getString("DEFINER")); procedure.setCreateTime(Timestamp.valueOf(rs.getString("CREATED"))); procedure.setModifyTime(Timestamp.valueOf(rs.getString("LAST_ALTERED"))); - procedure.setDdl(String.format("create procedure %s (%s) %s;", - StringUtils.quoteMysqlIdentifier(procedure.getProName()), - StringUtils.substring(parameters.toString(), 0, parameters.length() - 1), - rs.getString("ROUTINE_DEFINITION"))); }); return parseProcedureDDL(procedure); } diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/mysql/OBMySQLBetween400And432SchemaAccessor.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/mysql/OBMySQLBetween400And432SchemaAccessor.java new file mode 100644 index 0000000000..650969ece9 --- /dev/null +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/mysql/OBMySQLBetween400And432SchemaAccessor.java @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.tools.dbbrowser.schema.mysql; + +import java.util.List; +import java.util.Map; + +import org.springframework.jdbc.core.JdbcOperations; + +import com.oceanbase.tools.dbbrowser.model.DBObjectIdentity; +import com.oceanbase.tools.dbbrowser.model.DBTableColumn; + +/** + * @description: applicable to OB [4.0.0,4.3.2) + * @author: zijia.cj + * @date: 2024/8/27 14:55 + * @since: 4.3.3 + */ +public class OBMySQLBetween400And432SchemaAccessor extends OBMySQLSchemaAccessor { + + + public OBMySQLBetween400And432SchemaAccessor(JdbcOperations jdbcOperations) { + super(jdbcOperations); + } + + @Override + public List showExternalTables(String schemaName) { + throw new UnsupportedOperationException( + "External table is supported by odc after the 432 version of oceanbase"); + } + + @Override + public List showExternalTablesLike(String schemaName, String tableNameLike) { + throw new UnsupportedOperationException( + "External table is supported by odc after the 432 version of oceanbase"); + } + + @Override + public List listExternalTables(String schemaName, String tableNameLike) { + throw new UnsupportedOperationException( + "External table is supported by odc after the 432 version of oceanbase"); + } + + @Override + public boolean isExternalTable(String schemaName, String tableName) { + return false; + } + + @Override + public Map> listBasicExternalTableColumns(String schemaName) { + throw new UnsupportedOperationException("not support yet"); + } + + @Override + public List listBasicExternalTableColumns(String schemaName, String externalTableName) { + throw new UnsupportedOperationException("not support yet"); + } + +} diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/mysql/OBMySQLSchemaAccessor.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/mysql/OBMySQLSchemaAccessor.java index 658d8f2dca..bc5720e892 100644 --- a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/mysql/OBMySQLSchemaAccessor.java +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/mysql/OBMySQLSchemaAccessor.java @@ -25,6 +25,7 @@ import java.util.stream.Collectors; import org.apache.commons.collections4.CollectionUtils; +import org.springframework.jdbc.core.BeanPropertyRowMapper; import org.springframework.jdbc.core.JdbcOperations; import com.oceanbase.tools.dbbrowser.model.DBColumnGroupElement; @@ -41,6 +42,7 @@ import com.oceanbase.tools.dbbrowser.parser.SqlParser; import com.oceanbase.tools.dbbrowser.parser.result.ParseSqlResult; import com.oceanbase.tools.dbbrowser.schema.DBSchemaAccessorSqlMappers; +import com.oceanbase.tools.dbbrowser.schema.constant.Statements; import com.oceanbase.tools.dbbrowser.schema.constant.StatementsFiles; import com.oceanbase.tools.dbbrowser.util.DBSchemaAccessorUtil; import com.oceanbase.tools.dbbrowser.util.MySQLSqlBuilder; @@ -52,14 +54,14 @@ import lombok.extern.slf4j.Slf4j; /** - * 适用 OB 版本：[4.0.0, ~) + * 适用 OB 版本：[4.3.2, ~) * * @author jingtian */ @Slf4j public class OBMySQLSchemaAccessor extends MySQLNoLessThan5700SchemaAccessor { - protected static final Set ESCAPE_SCHEMA_SET = new HashSet<>(3); + protected static final Set ESCAPE_SCHEMA_SET = new HashSet<>(4); static { ESCAPE_SCHEMA_SET.add("PUBLIC"); @@ -76,7 +78,7 @@ public List showDatabases() { public OBMySQLSchemaAccessor(JdbcOperations jdbcOperations) { super(jdbcOperations); - this.sqlMapper = DBSchemaAccessorSqlMappers.get(StatementsFiles.OBMYSQL_40X); + this.sqlMapper = DBSchemaAccessorSqlMappers.get(StatementsFiles.OBMYSQL_432x); } @Override @@ -287,16 +289,21 @@ private void parseDdlToSetIndexInfo(String ddl, List indexList) { fillWarning(indexList, DBObjectType.INDEX, "table ddl is blank, can not set index range by parse ddl"); return; } - ParseSqlResult result = SqlParser.parseMysql(ddl); - if (CollectionUtils.isEmpty(result.getIndexes())) { - fillWarning(indexList, DBObjectType.INDEX, "parse index DDL failed"); - } else { - indexList.forEach(index -> result.getIndexes().forEach(dbIndex -> { - if (StringUtils.equals(index.getName(), dbIndex.getName())) { - index.setGlobal("GLOBAL".equalsIgnoreCase(dbIndex.getRange().name())); - index.setColumnGroups(dbIndex.getColumnGroups()); - } - })); + try { + ParseSqlResult result = SqlParser.parseMysql(ddl); + if (CollectionUtils.isEmpty(result.getIndexes())) { + fillWarning(indexList, DBObjectType.INDEX, "parse index DDL failed"); + } else { + indexList.forEach(index -> result.getIndexes().forEach(dbIndex -> { + if (StringUtils.equals(index.getName(), dbIndex.getName())) { + index.setGlobal("GLOBAL".equalsIgnoreCase(dbIndex.getRange().name())); + index.setColumnGroups(dbIndex.getColumnGroups()); + } + })); + } + } catch (Exception e) { + fillWarning(indexList, DBObjectType.INDEX, "failed to set index info by parse ddl"); + log.warn("failed to set index info by parse ddl:{}", ddl, e); } } @@ -361,6 +368,90 @@ public Map getTables(@NonNull String schemaName, List t return returnVal; } + @Override + public List showExternalTables(String schemaName) { + return showExternalTablesLike(schemaName, null); + } + + + @Override + public List showExternalTablesLike(String schemaName, String tableNameLike) { + MySQLSqlBuilder sb = new MySQLSqlBuilder(); + sb.append("SELECT table_name FROM information_schema.tables WHERE TABLE_TYPE = 'EXTERNAL TABLE'"); + if (StringUtils.isNotBlank(schemaName)) { + sb.append(" AND table_schema="); + sb.value(schemaName); + } + if (StringUtils.isNotBlank(tableNameLike)) { + sb.append(" AND table_name LIKE "); + sb.value(tableNameLike); + } + sb.append(" ORDER BY table_name"); + return jdbcOperations.queryForList(sb.toString(), String.class); + } + + @Override + public List listExternalTables(String schemaName, String tableNameLike) { + MySQLSqlBuilder sb = new MySQLSqlBuilder(); + sb.append("select table_schema as schema_name, 'EXTERNAL_TABLE' as type, table_name as name "); + sb.append("from information_schema.tables where table_type = 'EXTERNAL TABLE'"); + if (StringUtils.isNotBlank(schemaName)) { + sb.append(" AND table_schema="); + sb.value(schemaName); + } + if (StringUtils.isNotBlank(tableNameLike)) { + sb.append(" AND table_name LIKE "); + sb.value(tableNameLike); + } + sb.append(" ORDER BY schema_name, table_name"); + return jdbcOperations.query(sb.toString(), new BeanPropertyRowMapper<>(DBObjectIdentity.class)); + } + + @Override + public boolean isExternalTable(String schemaName, String tableName) { + MySQLSqlBuilder sb = new MySQLSqlBuilder(); + sb.append("SELECT table_type FROM information_schema.tables"); + if (StringUtils.isNotBlank(schemaName)) { + sb.append(" Where table_schema="); + sb.value(schemaName); + } + if (StringUtils.isNotBlank(tableName)) { + sb.append(" AND table_name = "); + sb.value(tableName); + } + String tableType = jdbcOperations.queryForObject(sb.toString(), String.class); + if (tableType == null) { + throw new IllegalArgumentException("table name: " + tableName + " is not exist"); + } + if (StringUtils.equalsIgnoreCase(tableType, "EXTERNAL TABLE")) { + return true; + } else { + return false; + } + } + + @Override + public boolean syncExternalTableFiles(String schemaName, String tableName) { + MySQLSqlBuilder sb = new MySQLSqlBuilder(); + sb.append("ALTER EXTERNAL TABLE ").identifier(schemaName, tableName).append(" REFRESH"); + jdbcOperations.execute(sb.toString()); + return true; + } + + @Override + public Map> listBasicExternalTableColumns(String schemaName) { + String sql = sqlMapper.getSql(Statements.LIST_BASIC_SCHEMA_EXTERNAL_TABLE_COLUMNS); + List tableColumns = jdbcOperations.query(sql, new Object[] {schemaName, schemaName}, + listBasicTableColumnRowMapper()); + return tableColumns.stream().collect(Collectors.groupingBy(DBTableColumn::getTableName)); + } + + @Override + public List listBasicExternalTableColumns(String schemaName, String externalTableName) { + String sql = sqlMapper.getSql(Statements.LIST_BASIC_EXTERNAL_TABLE_COLUMNS); + return jdbcOperations.query(sql, new Object[] {schemaName, externalTableName}, listBasicTableColumnRowMapper()); + } + @Override protected void correctColumnPrecisionIfNeed(List tableColumns) {} } diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/mysql/ODPOBMySQLSchemaAccessor.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/mysql/ODPOBMySQLSchemaAccessor.java index 5b341587c0..8ad6f47f37 100644 --- a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/mysql/ODPOBMySQLSchemaAccessor.java +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/mysql/ODPOBMySQLSchemaAccessor.java @@ -162,4 +162,19 @@ public DBProcedure getProcedure(String schemaName, String procedureName) { throw new UnsupportedOperationException("Not supported yet"); } + @Override + public List showExternalTables(String schemaName) { + throw new UnsupportedOperationException("Not supported yet"); + } + + @Override + public List showExternalTablesLike(String schemaName, String tableNameLike) { + throw new UnsupportedOperationException("Not supported yet"); + } + + @Override + public List listExternalTables(String schemaName, String tableNameLike) { + throw new UnsupportedOperationException("Not supported yet"); + } + } diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/oracle/OBOracleBetween4000And4100SchemaAccessor.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/oracle/OBOracleBetween4000And4100SchemaAccessor.java index f88aec2d4c..0a9e5a4894 100644 --- a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/oracle/OBOracleBetween4000And4100SchemaAccessor.java +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/oracle/OBOracleBetween4000And4100SchemaAccessor.java @@ -27,7 +27,7 @@ * @author jingtian * @date 2024/4/15 */ -public class OBOracleBetween4000And4100SchemaAccessor extends OBOracleSchemaAccessor { +public class OBOracleBetween4000And4100SchemaAccessor extends OBOracleBetween410And432SchemaAccessor { public OBOracleBetween4000And4100SchemaAccessor(JdbcOperations jdbcOperations, OracleDataDictTableNames dataDictTableNames) { super(jdbcOperations, dataDictTableNames); diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/oracle/OBOracleBetween410And432SchemaAccessor.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/oracle/OBOracleBetween410And432SchemaAccessor.java new file mode 100644 index 0000000000..c5467643d5 --- /dev/null +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/oracle/OBOracleBetween410And432SchemaAccessor.java @@ -0,0 +1,116 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.tools.dbbrowser.schema.oracle; + +import java.util.List; +import java.util.Map; + +import org.springframework.jdbc.core.BeanPropertyRowMapper; +import org.springframework.jdbc.core.JdbcOperations; + +import com.oceanbase.tools.dbbrowser.model.DBObjectIdentity; +import com.oceanbase.tools.dbbrowser.model.DBTableColumn; +import com.oceanbase.tools.dbbrowser.schema.DBSchemaAccessorSqlMappers; +import com.oceanbase.tools.dbbrowser.schema.constant.StatementsFiles; +import com.oceanbase.tools.dbbrowser.util.OracleDataDictTableNames; +import com.oceanbase.tools.dbbrowser.util.OracleSqlBuilder; +import com.oceanbase.tools.dbbrowser.util.StringUtils; + +/** + * @description: applicable to OB [4.1.0,4.3.2) + * @author: zijia.cj + * @date: 2024/8/27 15:13 + * @since: 4.3.3 + */ +public class OBOracleBetween410And432SchemaAccessor extends OBOracleSchemaAccessor { + + public OBOracleBetween410And432SchemaAccessor(JdbcOperations jdbcOperations, + OracleDataDictTableNames dataDictTableNames) { + super(jdbcOperations, dataDictTableNames); + this.sqlMapper = DBSchemaAccessorSqlMappers.get(StatementsFiles.OBORACLE_4_1_x); + } + + @Override + public List showExternalTables(String schemaName) { + throw new UnsupportedOperationException( + "External table is supported by odc after the 432 version of oceanbase"); + } + + + @Override + public List showExternalTablesLike(String schemaName, String tableNameLike) { + throw new UnsupportedOperationException( + "External table is supported by odc after the 432 version of oceanbase"); + } + + @Override + public List listExternalTables(String schemaName, String tableNameLike) { + throw new UnsupportedOperationException( + "External table is supported by odc after the 432 version of oceanbase"); + } + + @Override + public List showTablesLike(String schemaName, String tableNameLike) { + OracleSqlBuilder sb = new OracleSqlBuilder(); + sb.append("SELECT TABLE_NAME FROM "); + sb.append(dataDictTableNames.TABLES()); + sb.append(" WHERE OWNER="); + sb.value(schemaName); + if (StringUtils.isNotBlank(tableNameLike)) { + sb.append(" AND TABLE_NAME LIKE "); + sb.value(tableNameLike); + } + sb.append(" ORDER BY TABLE_NAME ASC"); + return jdbcOperations.queryForList(sb.toString(), String.class); + } + + @Override + public boolean syncExternalTableFiles(String schemaName, String tableName) { + throw new UnsupportedOperationException("Not supported yet"); + } + + @Override + public List listTables(String schemaName, String tableNameLike) { + OracleSqlBuilder sb = new OracleSqlBuilder(); + sb.append("select OWNER as schema_name, 'TABLE' as type,TABLE_NAME as name"); + sb.append(" from "); + sb.append(dataDictTableNames.TABLES()); + sb.append(" where 1=1 "); + + if (StringUtils.isNotBlank(schemaName)) { + sb.append(" AND OWNER="); + sb.value(schemaName); + } + if (StringUtils.isNotBlank(tableNameLike)) { + sb.append(" AND TABLE_NAME LIKE "); + sb.value(tableNameLike); + } + sb.append(" ORDER BY schema_name, type, name"); + return jdbcOperations.query(sb.toString(), new BeanPropertyRowMapper<>(DBObjectIdentity.class)); + } + + @Override + public Map> listBasicExternalTableColumns(String schemaName) { + throw new UnsupportedOperationException("not support yet"); + } + + @Override + public List listBasicExternalTableColumns(String schemaName, String externalTableName) { + throw new UnsupportedOperationException("not support yet"); + } + + +} diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/oracle/OBOracleSchemaAccessor.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/oracle/OBOracleSchemaAccessor.java index 8d57ebd5f0..4ca56d6f6e 100644 --- a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/oracle/OBOracleSchemaAccessor.java +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/oracle/OBOracleSchemaAccessor.java @@ -71,6 +71,7 @@ import com.oceanbase.tools.dbbrowser.parser.result.ParseOraclePLResult; import com.oceanbase.tools.dbbrowser.parser.result.ParseSqlResult; import com.oceanbase.tools.dbbrowser.schema.DBSchemaAccessorSqlMappers; +import com.oceanbase.tools.dbbrowser.schema.constant.Statements; import com.oceanbase.tools.dbbrowser.schema.constant.StatementsFiles; import com.oceanbase.tools.dbbrowser.util.DBSchemaAccessorUtil; import com.oceanbase.tools.dbbrowser.util.OracleDataDictTableNames; @@ -85,7 +86,7 @@ import lombok.extern.slf4j.Slf4j; /** - * 适用于的 DB 版本：[4.1.0, ~) + * 适用于的 DB 版本：[4.3.2, ~) * * @author jingtian */ @@ -103,7 +104,7 @@ public class OBOracleSchemaAccessor extends OracleSchemaAccessor { public OBOracleSchemaAccessor(JdbcOperations jdbcOperations, OracleDataDictTableNames dataDictTableNames) { super(jdbcOperations, dataDictTableNames); - this.sqlMapper = DBSchemaAccessorSqlMappers.get(StatementsFiles.OBORACLE_4_1_x); + this.sqlMapper = DBSchemaAccessorSqlMappers.get(StatementsFiles.OBORACLE_4_3_2_x); } @Override @@ -1035,4 +1036,113 @@ public Map getTables(@NonNull String schemaName, List t } return returnVal; } + + @Override + public List showExternalTables(String schemaName) { + return showExternalTablesLike(schemaName, null); + } + + + @Override + public List showExternalTablesLike(String schemaName, String tableNameLike) { + return commonShowTablesLike(schemaName, tableNameLike, DBObjectType.EXTERNAL_TABLE); + } + + + + @Override + public List listExternalTables(String schemaName, String tableNameLike) { + OracleSqlBuilder sb = new OracleSqlBuilder(); + sb.append("select OWNER as schema_name, 'EXTERNAL_TABLE' as type,TABLE_NAME as name"); + sb.append(" from "); + sb.append(dataDictTableNames.TABLES()); + sb.append(" where EXTERNAL = 'YES'"); + + if (StringUtils.isNotBlank(schemaName)) { + sb.append(" AND OWNER="); + sb.value(schemaName); + } + if (StringUtils.isNotBlank(tableNameLike)) { + sb.append(" AND TABLE_NAME LIKE "); + sb.value(tableNameLike); + } + sb.append(" ORDER BY schema_name, type, name"); + return jdbcOperations.query(sb.toString(), new BeanPropertyRowMapper<>(DBObjectIdentity.class)); + } + + @Override + public boolean syncExternalTableFiles(String schemaName, String tableName) { + OracleSqlBuilder sb = new OracleSqlBuilder(); + sb.append("ALTER EXTERNAL TABLE ").identifier(schemaName, tableName).append(" REFRESH"); + jdbcOperations.execute(sb.toString()); + return true; + } + + // After ob version 4.3.2, oracle model displaying table list needs to exclude external tables + @Override + public List showTablesLike(String schemaName, String tableNameLike) { + return commonShowTablesLike(schemaName, tableNameLike, DBObjectType.TABLE); + } + + // After ob version 4.3.2, oracle model displaying table list needs to exclude external tables + @Override + public List listTables(String schemaName, String tableNameLike) { + OracleSqlBuilder sb = new OracleSqlBuilder(); + sb.append("select OWNER as schema_name, 'TABLE' as type,TABLE_NAME as name"); + sb.append(" from "); + sb.append(dataDictTableNames.TABLES()); + sb.append(" where EXTERNAL = 'NO'"); + + if (StringUtils.isNotBlank(schemaName)) { + sb.append(" AND OWNER="); + sb.value(schemaName); + } + if (StringUtils.isNotBlank(tableNameLike)) { + sb.append(" AND TABLE_NAME LIKE "); + sb.value(tableNameLike); + } + sb.append(" ORDER BY schema_name, type, name"); + return jdbcOperations.query(sb.toString(), new BeanPropertyRowMapper<>(DBObjectIdentity.class)); + } + + @Override + public Map> listBasicExternalTableColumns(String schemaName) { + String sql = sqlMapper.getSql(Statements.LIST_BASIC_SCHEMA_EXTERNAL_TABLE_COLUMNS); + List tableColumns = + jdbcOperations.query(sql, new Object[] {schemaName, schemaName}, listBasicColumnsRowMapper()); + return tableColumns.stream().collect(Collectors.groupingBy(DBTableColumn::getTableName)); + } + + @Override + public List listBasicExternalTableColumns(String schemaName, String externalTableName) { + String sql = sqlMapper.getSql(Statements.LIST_BASIC_EXTERNAL_TABLE_COLUMNS); + return jdbcOperations.query(sql, new Object[] {schemaName, externalTableName}, listBasicColumnsRowMapper()); + } + + private List commonShowTablesLike(String schemaName, String tableNameLike, + @NonNull DBObjectType tableType) { + OracleSqlBuilder sb = new OracleSqlBuilder(); + sb.append("SELECT TABLE_NAME FROM "); + sb.append(dataDictTableNames.TABLES()); + switch (tableType) { + case TABLE: + sb.append(" WHERE EXTERNAL = 'NO'"); + break; + case EXTERNAL_TABLE: + sb.append(" WHERE EXTERNAL = 'YES'"); + break; + default: + throw new UnsupportedOperationException("Not supported table type"); + } + if (StringUtils.isNotBlank(schemaName)) { + sb.append(" AND OWNER="); + sb.value(schemaName); + } + if (StringUtils.isNotBlank(tableNameLike)) { + sb.append(" AND TABLE_NAME LIKE "); + sb.value(tableNameLike); + } + sb.append(" ORDER BY TABLE_NAME ASC"); + return jdbcOperations.queryForList(sb.toString(), String.class); + } } diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/oracle/OracleSchemaAccessor.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/oracle/OracleSchemaAccessor.java index 35bba2e82d..d3c2de4b8d 100644 --- a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/oracle/OracleSchemaAccessor.java +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/oracle/OracleSchemaAccessor.java @@ -247,6 +247,26 @@ public List listTables(String schemaName, String tableNameLike return jdbcOperations.query(sb.toString(), new BeanPropertyRowMapper<>(DBObjectIdentity.class)); } + @Override + public List showExternalTablesLike(String schemaName, String tableNameLike) { + throw new UnsupportedOperationException("Not support yet"); + } + + @Override + public List listExternalTables(String schemaName, String tableNameLike) { + throw new UnsupportedOperationException("Not support yet"); + } + + @Override + public boolean isExternalTable(String schemaName, String tableName) { + return false; + } + + @Override + public boolean syncExternalTableFiles(String schemaName, String tableName) { + throw new UnsupportedOperationException("Not supported yet"); + } + @Override public List listViews(String schemaName) { OracleSqlBuilder sb = new OracleSqlBuilder(); @@ -624,6 +644,16 @@ public List listBasicViewColumns(String schemaName, String viewNa return jdbcOperations.query(sql, new Object[] {schemaName, viewName}, listBasicColumnsRowMapper()); } + @Override + public Map> listBasicExternalTableColumns(String schemaName) { + throw new UnsupportedOperationException("not support yet"); + } + + @Override + public List listBasicExternalTableColumns(String schemaName, String externalTableName) { + throw new UnsupportedOperationException("not support yet"); + } + @Override public Map> listBasicColumnsInfo(String schemaName) { String sql = sqlMapper.getSql(Statements.LIST_BASIC_SCHEMA_COLUMNS_INFO); @@ -839,7 +869,7 @@ protected RowMapper listColumnsRowMapper() { tableColumn.setVirtual("YES".equalsIgnoreCase(rs.getString(OracleConstants.COL_VIRTUAL_COLUMN))); tableColumn.setDefaultValue("NULL".equals(defaultValue) ? null : defaultValue); if (tableColumn.getVirtual()) { - tableColumn.setGenExpression(rs.getString(OracleConstants.COL_DATA_DEFAULT)); + tableColumn.setGenExpression(defaultValue); } return tableColumn; }; @@ -1835,5 +1865,4 @@ protected String getSynonymOwnerSymbol(DBSynonymType synonymType, String schemaN throw new UnsupportedOperationException("Not supported Synonym type"); } } - } diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/postgre/PostgresSchemaAccessor.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/postgre/PostgresSchemaAccessor.java index a4aa3be576..c76721fd57 100644 --- a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/postgre/PostgresSchemaAccessor.java +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/schema/postgre/PostgresSchemaAccessor.java @@ -137,6 +137,26 @@ public List listTables(String schemaName, String tableNameLike throw new UnsupportedOperationException("Not supported yet"); } + @Override + public List showExternalTablesLike(String schemaName, String tableNameLike) { + throw new UnsupportedOperationException("Not supported yet"); + } + + @Override + public List listExternalTables(String schemaName, String tableNameLike) { + throw new UnsupportedOperationException("Not supported yet"); + } + + @Override + public boolean isExternalTable(String schemaName, String tableName) { + return false; + } + + @Override + public boolean syncExternalTableFiles(String schemaName, String tableName) { + throw new UnsupportedOperationException("Not supported yet"); + } + @Override public List listViews(String schemaName) { throw new UnsupportedOperationException("Not supported yet"); @@ -261,6 +281,16 @@ public List listBasicViewColumns(String schemaName, String viewNa throw new UnsupportedOperationException("Not supported yet"); } + @Override + public Map> listBasicExternalTableColumns(String schemaName) { + throw new UnsupportedOperationException("not support yet"); + } + + @Override + public List listBasicExternalTableColumns(String schemaName, String externalTableName) { + throw new UnsupportedOperationException("not support yet"); + } + @Override public Map> listBasicColumnsInfo(String schemaName) { throw new UnsupportedOperationException("Not supported yet"); diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/stats/mysql/OBMySQLNoLessThan400StatsAccessor.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/stats/mysql/OBMySQLNoLessThan400StatsAccessor.java index a219659da4..35fe77a72b 100644 --- a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/stats/mysql/OBMySQLNoLessThan400StatsAccessor.java +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/stats/mysql/OBMySQLNoLessThan400StatsAccessor.java @@ -41,7 +41,7 @@ public class OBMySQLNoLessThan400StatsAccessor extends OBMySQLStatsAccessor { + " STATE, " + " `USER_CLIENT_IP` as HOST, " + " HOST as PROXY_HOST, " - + " CONCAT(SVR_IP, ':', SVR_PORT) AS SVR_IP, " + + " CONCAT(SVR_IP, ':', SQL_PORT) AS SVR_IP, " + " TIME as EXECUTE_TIME, " + " CASE " + " WHEN `TRANS_STATE` IS NULL OR `TRANS_STATE` IN ('', 'IDLE', 'IN_TERMINATE', 'ABORTED', " diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/stats/mysql/OBMySQLStatsAccessor.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/stats/mysql/OBMySQLStatsAccessor.java index 4483190185..ec105f1fe4 100644 --- a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/stats/mysql/OBMySQLStatsAccessor.java +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/stats/mysql/OBMySQLStatsAccessor.java @@ -15,8 +15,15 @@ */ package com.oceanbase.tools.dbbrowser.stats.mysql; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + import org.springframework.jdbc.core.JdbcOperations; +import com.oceanbase.tools.dbbrowser.model.DBSession; +import com.oceanbase.tools.dbbrowser.util.StringUtils; + import lombok.NonNull; /** @@ -28,8 +35,25 @@ */ public class OBMySQLStatsAccessor extends MySQLNoLessThan5700StatsAccessor { + private static final String LIST_SESSIONS_BY_SHOW_PROCESSLIST = "SHOW FULL PROCESSLIST"; + public OBMySQLStatsAccessor(@NonNull JdbcOperations jdbcOperations) { super(jdbcOperations); } + @Override + public List listAllSessions() { + List sessions = super.listAllSessions(); + Map sessionId2SvrIp = new HashMap<>(); + jdbcOperations.query(LIST_SESSIONS_BY_SHOW_PROCESSLIST, rs -> { + if (rs.getMetaData().getColumnCount() == 11) { + String id = rs.getString("Id"); + String svrIp = StringUtils.join(rs.getString("Ip"), ":", rs.getString("Port")); + sessionId2SvrIp.put(id, svrIp); + } + }); + sessions.forEach(session -> session.setSvrIp(sessionId2SvrIp.get(session.getId()))); + return sessions; + } + } diff --git a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/stats/oracle/OBOracleNoLessThan400StatsAccessor.java b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/stats/oracle/OBOracleNoLessThan400StatsAccessor.java index cb727753a4..6ce9432dcf 100644 --- a/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/stats/oracle/OBOracleNoLessThan400StatsAccessor.java +++ b/libs/db-browser/src/main/java/com/oceanbase/tools/dbbrowser/stats/oracle/OBOracleNoLessThan400StatsAccessor.java @@ -38,7 +38,7 @@ public class OBOracleNoLessThan400StatsAccessor extends BaseOBOracleStatsAccesso + " STATE, " + " USER_CLIENT_IP as HOST, " + " HOST as PROXY_HOST, " - + " SVR_IP || ':' || TO_CHAR(SVR_PORT) AS SVR_IP, " + + " SVR_IP || ':' || TO_CHAR(SQL_PORT) AS SVR_IP, " + " TIME as EXECUTE_TIME, " + " CASE " + " WHEN TRANS_STATE IS NULL OR TRANS_STATE IN ('', 'IDLE', 'IN_TERMINATE', 'ABORTED', " diff --git a/libs/db-browser/src/main/resources/schema/sql/obmysql/obmysql_4_0_x.yaml b/libs/db-browser/src/main/resources/schema/sql/obmysql/obmysql_4_3_2_x.yaml similarity index 86% rename from libs/db-browser/src/main/resources/schema/sql/obmysql/obmysql_4_0_x.yaml rename to libs/db-browser/src/main/resources/schema/sql/obmysql/obmysql_4_3_2_x.yaml index bb8fe23387..601d4e6d92 100644 --- a/libs/db-browser/src/main/resources/schema/sql/obmysql/obmysql_4_0_x.yaml +++ b/libs/db-browser/src/main/resources/schema/sql/obmysql/obmysql_4_3_2_x.yaml @@ -34,6 +34,41 @@ sqls: ORDER BY TABLE_NAME ASC, ORDINAL_POSITION ASC + list-basic-external-table-columns: |- + SELECT + TABLE_SCHEMA, + TABLE_NAME, + COLUMN_NAME, + DATA_TYPE, + COLUMN_COMMENT + FROM + information_schema.columns + WHERE + TABLE_SCHEMA = ? AND TABLE_NAME = ? + ORDER BY + ORDINAL_POSITION ASC + list-basic-schema-external-table-columns: |- + SELECT + TABLE_SCHEMA, + TABLE_NAME, + COLUMN_NAME, + DATA_TYPE, + COLUMN_COMMENT + FROM + information_schema.columns + WHERE + TABLE_SCHEMA = ? + AND TABLE_NAME IN ( + SELECT + TABLE_NAME + FROM + information_schema.tables + WHERE + TABLE_SCHEMA = ? AND TABLE_TYPE = 'EXTERNAL TABLE' + ) + ORDER BY + TABLE_NAME ASC, + ORDINAL_POSITION ASC list-basic-view-columns: |- SELECT TABLE_SCHEMA, diff --git a/libs/db-browser/src/main/resources/schema/sql/oboracle/oboracle_4_3_2_x.yaml b/libs/db-browser/src/main/resources/schema/sql/oboracle/oboracle_4_3_2_x.yaml new file mode 100644 index 0000000000..98dfe4076a --- /dev/null +++ b/libs/db-browser/src/main/resources/schema/sql/oboracle/oboracle_4_3_2_x.yaml @@ -0,0 +1,324 @@ +sqls: + list-basic-table-columns: |- + SELECT + OWNER, + TABLE_NAME, + COLUMN_NAME, + DATA_TYPE, + COMMENTS + FROM + SYS.ALL_TAB_COLS NATURAL JOIN SYS.ALL_COL_COMMENTS + WHERE + OWNER = ? AND TABLE_NAME = ? AND USER_GENERATED='YES' + ORDER BY + COLUMN_ID ASC + list-basic-schema-table-columns: |- + SELECT + OWNER, + TABLE_NAME, + COLUMN_NAME, + DATA_TYPE, + COMMENTS + FROM + SYS.ALL_TAB_COLS NATURAL JOIN SYS.ALL_COL_COMMENTS + WHERE + OWNER = ? + AND TABLE_NAME IN ( + SELECT + TABLE_NAME + FROM + SYS.ALL_TABLES + WHERE + OWNER = ? AND EXTERNAL='NO' + ) AND USER_GENERATED='YES' + ORDER BY + TABLE_NAME ASC, + COLUMN_ID ASC + list-basic-external-table-columns: |- + SELECT + OWNER, + TABLE_NAME, + COLUMN_NAME, + DATA_TYPE, + COMMENTS + FROM + SYS.ALL_TAB_COLS NATURAL JOIN SYS.ALL_COL_COMMENTS + WHERE + OWNER = ? AND TABLE_NAME = ? AND USER_GENERATED='YES' + ORDER BY + COLUMN_ID ASC + list-basic-schema-external-table-columns: |- + SELECT + OWNER, + TABLE_NAME, + COLUMN_NAME, + DATA_TYPE, + COMMENTS + FROM + SYS.ALL_TAB_COLS NATURAL JOIN SYS.ALL_COL_COMMENTS + WHERE + OWNER = ? + AND TABLE_NAME IN ( + SELECT + TABLE_NAME + FROM + SYS.ALL_TABLES + WHERE + OWNER = ? AND EXTERNAL='YES' + ) AND USER_GENERATED='YES' + ORDER BY + TABLE_NAME ASC, + COLUMN_ID ASC + list-basic-view-columns: |- + SELECT + OWNER, + TABLE_NAME, + COLUMN_NAME, + DATA_TYPE, + COMMENTS + FROM + SYS.ALL_TAB_COLS NATURAL JOIN SYS.ALL_COL_COMMENTS + WHERE + OWNER = ? AND TABLE_NAME = ? AND USER_GENERATED='YES' + ORDER BY + COLUMN_ID ASC + list-basic-schema-view-columns: |- + SELECT + OWNER, + TABLE_NAME, + COLUMN_NAME, + DATA_TYPE, + COMMENTS + FROM + SYS.ALL_TAB_COLS NATURAL JOIN SYS.ALL_COL_COMMENTS + WHERE + OWNER = ? + AND TABLE_NAME IN ( + SELECT + VIEW_NAME + FROM + SYS.ALL_VIEWS + WHERE + OWNER = ? + ) AND USER_GENERATED='YES' + ORDER BY + TABLE_NAME ASC, + COLUMN_ID ASC + list-basic-schema-columns-info: |- + SELECT + OWNER, + TABLE_NAME, + COLUMN_NAME + FROM + SYS.ALL_TAB_COLS + WHERE + OWNER = ? + ORDER BY + TABLE_NAME ASC, + COLUMN_ID ASC + list-table-columns: |- + SELECT + OWNER, + TABLE_NAME, + COLUMN_ID, + COLUMN_NAME, + DATA_TYPE, + DATA_SCALE, + DATA_PRECISION, + DATA_LENGTH, + CHAR_LENGTH, + DATA_TYPE_MOD, + CHAR_USED, + NULLABLE, + DATA_DEFAULT, + HIDDEN_COLUMN, + VIRTUAL_COLUMN + FROM + SYS.ALL_TAB_COLS + WHERE + OWNER = ? AND TABLE_NAME = ? and USER_GENERATED='YES' + ORDER BY + COLUMN_ID ASC + list-schema-columns: |- + SELECT + OWNER, + TABLE_NAME, + COLUMN_ID, + COLUMN_NAME, + DATA_TYPE, + DATA_SCALE, + DATA_PRECISION, + DATA_LENGTH, + CHAR_LENGTH, + DATA_TYPE_MOD, + CHAR_USED, + NULLABLE, + DATA_DEFAULT, + HIDDEN_COLUMN, + VIRTUAL_COLUMN + FROM + SYS.ALL_TAB_COLS + WHERE + OWNER = ? and USER_GENERATED='YES' + ORDER BY + COLUMN_ID ASC + list-table-indexes: |- + SELECT + IDX.OWNER, + IDX.TABLE_OWNER, + IDX.TABLE_NAME, + IDX.INDEX_NAME, + IDX_COL.COLUMN_NAME, + IDX_COL.COLUMN_POSITION, + IDX.INDEX_TYPE, + IDX.UNIQUENESS, + IDX.COMPRESSION, + IDX.VISIBILITY, + IDX.STATUS + FROM + SYS.ALL_IND_COLUMNS IDX_COL + LEFT JOIN + SYS.ALL_INDEXES IDX ON IDX_COL.TABLE_OWNER = IDX.TABLE_OWNER + AND IDX_COL.TABLE_NAME = IDX.TABLE_NAME + AND IDX_COL.INDEX_NAME = IDX.INDEX_NAME + WHERE + IDX.TABLE_OWNER = ? AND IDX.TABLE_NAME = ? + ORDER BY + IDX.INDEX_NAME, + IDX_COL.COLUMN_POSITION ASC + list-schema-index: |- + SELECT + IDX.OWNER, + IDX.TABLE_OWNER, + IDX.TABLE_NAME, + IDX.INDEX_NAME, + IDX_COL.COLUMN_NAME, + IDX_COL.COLUMN_POSITION, + IDX.INDEX_TYPE, + IDX.UNIQUENESS, + IDX.COMPRESSION, + IDX.VISIBILITY, + IDX.STATUS + FROM + SYS.ALL_IND_COLUMNS IDX_COL + LEFT JOIN + SYS.ALL_INDEXES IDX ON IDX_COL.TABLE_OWNER = IDX.TABLE_OWNER + AND IDX_COL.TABLE_NAME = IDX.TABLE_NAME + AND IDX_COL.INDEX_NAME = IDX.INDEX_NAME + WHERE + IDX.TABLE_OWNER = ? + ORDER BY + IDX.TABLE_NAME, + IDX.INDEX_NAME, + IDX_COL.COLUMN_POSITION ASC + list-table-constraints: |- + SELECT + t1.OWNER, + t1.CONSTRAINT_NAME, + t1.CONSTRAINT_TYPE, + t1.TABLE_NAME, + t1.SEARCH_CONDITION, + t1.R_OWNER, + t1.R_CONSTRAINT_NAME, + t1.DELETE_RULE, + t1.STATUS, + t1.DEFERRABLE, + t1.DEFERRED, + t1.VALIDATED, + t2.TABLE_NAME as R_TABLE_NAME, + t2.COLUMN_NAME as R_COLUMN_NAME, + t3.position as POSITION, + t3.COLUMN_NAME as COLUMN_NAME + FROM + SYS.ALL_CONSTRAINTS t1 + JOIN SYS.ALL_CONS_COLUMNS t3 on t1.CONSTRAINT_NAME = t3.CONSTRAINT_NAME and t1.OWNER = t3.OWNER + LEFT JOIN SYS.ALL_CONS_COLUMNS t2 on t2.CONSTRAINT_NAME = t1.R_CONSTRAINT_NAME + AND t2.OWNER = t1.R_OWNER + WHERE + t1.OWNER = ? and t1.TABLE_NAME = ? + ORDER BY + t1.CONSTRAINT_NAME, + t3.position ASC + list-schema-constraints: |- + SELECT + t1.OWNER, + t1.CONSTRAINT_NAME, + t1.CONSTRAINT_TYPE, + t1.TABLE_NAME, + t1.SEARCH_CONDITION, + t1.R_OWNER, + t1.R_CONSTRAINT_NAME, + t1.DELETE_RULE, + t1.STATUS, + t1.DEFERRABLE, + t1.DEFERRED, + t1.VALIDATED, + t2.TABLE_NAME as R_TABLE_NAME, + t2.COLUMN_NAME as R_COLUMN_NAME, + t3.position as POSITION, + t3.COLUMN_NAME as COLUMN_NAME + FROM + SYS.ALL_CONSTRAINTS t1 + JOIN SYS.ALL_CONS_COLUMNS t3 on t1.CONSTRAINT_NAME = t3.CONSTRAINT_NAME and t1.OWNER = t3.OWNER + LEFT JOIN SYS.ALL_CONS_COLUMNS t2 on t2.CONSTRAINT_NAME = t1.R_CONSTRAINT_NAME + AND t2.OWNER = t1.R_OWNER + WHERE + t1.OWNER = ? + ORDER BY + t1.TABLE_NAME, + t1.CONSTRAINT_NAME, + t3.position ASC + get-partition-option: |- + SELECT + PARTITIONING_TYPE + FROM + SYS.ALL_PART_TABLES + WHERE + OWNER = ? + AND TABLE_NAME = ? + list-partitions-options: |- + SELECT + TABLE_NAME, + PARTITIONING_TYPE + FROM + SYS.ALL_PART_TABLES + WHERE + OWNER = ? + list-partition-definitions: |- + SELECT + PARTITION_NAME, + PARTITION_POSITION, + HIGH_VALUE + FROM + SYS.ALL_TAB_PARTITIONS + WHERE + TABLE_OWNER = ? + AND TABLE_NAME = ? + ORDER BY + PARTITION_POSITION ASC + list-partitions-definitions: |- + SELECT + TABLE_NAME, + PARTITION_NAME, + PARTITION_POSITION, + HIGH_VALUE + FROM + SYS.ALL_TAB_PARTITIONS + WHERE + TABLE_OWNER = ? + ORDER BY + PARTITION_POSITION ASC + list-database: |- + select + USERNAME, + USERID + from + ALL_USERS + get-database: |- + SELECT + USERNAME, + USERID + from + SYS.ALL_USERS + WHERE + USERNAME = ? \ No newline at end of file diff --git a/libs/db-browser/src/test/java/com/oceanbase/tools/dbbrowser/parser/PLParserTest.java b/libs/db-browser/src/test/java/com/oceanbase/tools/dbbrowser/parser/PLParserTest.java index 5aded51d8a..63390a1a23 100644 --- a/libs/db-browser/src/test/java/com/oceanbase/tools/dbbrowser/parser/PLParserTest.java +++ b/libs/db-browser/src/test/java/com/oceanbase/tools/dbbrowser/parser/PLParserTest.java @@ -57,6 +57,172 @@ public void testParseMysqlProcedure() { Assert.assertEquals(1, result.getParamList().size()); } + @Test + public void testParseMysqlProcedureContainsDefiner() { + String pl = "create DEFINER = `root`@`%` PROCEDURE testProduce (out p1 int) \n" + "BEGIN \n" + + "DECLARE Eno INT DEFAULT 10000;\n" + + "DECLARE En VARCHAR(20);\n" + "DECLARE J VARCHAR(20);\n" + "DECLARE M INT DEFAULT 80000;\n" + + "DECLARE H YEAR;\n" + "DECLARE Dno INT;\n" + "DECLARE i INT DEFAULT 1; \n" + "RETURN;\n" + "END;"; + ParseMysqlPLResult result = PLParser.parseObMysql(pl); + Assert.assertEquals(7, result.getVaribaleList().size()); + Assert.assertEquals("testProduce", result.getPlName()); + Assert.assertEquals("PROCEDURE", result.getPlType()); + Assert.assertEquals(1, result.getParamList().size()); + } + + @Test + public void testParseMysqlProcedureContainsDefinerWithoutParams() { + String pl = "CREATE DEFINER = `root`@`%` PROCEDURE simple_procedure()\n" + + "BEGIN\n" + + " SELECT 'Hello, World!';\n" + + "END"; + ParseMysqlPLResult result = PLParser.parseObMysql(pl); + Assert.assertEquals(0, result.getVaribaleList().size()); + Assert.assertEquals("simple_procedure", result.getPlName()); + Assert.assertEquals("PROCEDURE", result.getPlType()); + Assert.assertEquals(0, result.getParamList().size()); + } + + @Test + public void testParseMysqlProcedureContainsDefinerWithInput() { + String pl = "CREATE DEFINER = `root`@`%` PROCEDURE greet_user(IN user_name VARCHAR(50))\n" + + "BEGIN\n" + + " SELECT CONCAT('Hello, ', user_name, '!');\n" + + "END"; + ParseMysqlPLResult result = PLParser.parseObMysql(pl); + Assert.assertEquals(0, result.getVaribaleList().size()); + Assert.assertEquals("greet_user", result.getPlName()); + Assert.assertEquals("PROCEDURE", result.getPlType()); + Assert.assertEquals(1, result.getParamList().size()); + } + + @Test + public void testParseMysqlProcedureContainsDefinerWithInputAndOutput() { + String pl = "CREATE DEFINER = `root`@`%` PROCEDURE get_user_name(IN user_id INT, OUT user_name VARCHAR(50))\n" + + "BEGIN\n" + + " SELECT name INTO user_name FROM users WHERE id = user_id;\n" + + "END"; + ParseMysqlPLResult result = PLParser.parseObMysql(pl); + Assert.assertEquals(0, result.getVaribaleList().size()); + Assert.assertEquals("get_user_name", result.getPlName()); + Assert.assertEquals("PROCEDURE", result.getPlType()); + Assert.assertEquals(2, result.getParamList().size()); + } + + @Test + public void testParseMysqlProcedureContainsDefinerAndCondition() { + String pl = + "CREATE DEFINER = `root`@`%` PROCEDURE check_user_status(IN user_id INT, OUT user_status VARCHAR(20))\n" + + + "BEGIN\n" + + " DECLARE user_count INT;\n" + + "\n" + + " SELECT COUNT(*) INTO user_count FROM users WHERE id = user_id;\n" + + "\n" + + " IF user_count > 0 THEN\n" + + " SELECT status INTO user_status FROM users WHERE id = user_id;\n" + + " ELSE\n" + + " SET user_status = 'User not found';\n" + + " END IF;\n" + + "END"; + ParseMysqlPLResult result = PLParser.parseObMysql(pl); + Assert.assertEquals(1, result.getVaribaleList().size()); + Assert.assertEquals("check_user_status", result.getPlName()); + Assert.assertEquals("PROCEDURE", result.getPlType()); + Assert.assertEquals(2, result.getParamList().size()); + } + + @Test + public void testParseMysqlProcedureContainsDefinerAndFunction() { + String pl = "CREATE DEFINER = `root`@`%` PROCEDURE calculate_user_average_age(OUT average_age FLOAT)\n" + + "BEGIN\n" + + " SELECT AVG(age) INTO average_age FROM users;\n" + + "END"; + ParseMysqlPLResult result = PLParser.parseObMysql(pl); + Assert.assertEquals(0, result.getVaribaleList().size()); + Assert.assertEquals("calculate_user_average_age", result.getPlName()); + Assert.assertEquals("PROCEDURE", result.getPlType()); + Assert.assertEquals(1, result.getParamList().size()); + } + + @Test + public void testParseMysqlProcedureContainsDefinerAndCursor() { + String pl = "CREATE DEFINER = `root`@`%` PROCEDURE list_all_users()\n" + + "BEGIN\n" + + " DECLARE done INT DEFAULT 0;\n" + + " DECLARE user_id INT;\n" + + " DECLARE user_name VARCHAR(50);\n" + + "\n" + + " DECLARE user_cursor CURSOR FOR SELECT id, name FROM users;\n" + + " DECLARE CONTINUE HANDLER FOR NOT FOUND SET done = 1;\n" + + "\n" + + " OPEN user_cursor;\n" + + "\n" + + " read_loop: LOOP\n" + + " FETCH user_cursor INTO user_id, user_name;\n" + + " IF done THEN\n" + + " LEAVE read_loop;\n" + + " END IF;\n" + + " SELECT CONCAT('User ID: ', user_id, ', User Name: ', user_name);\n" + + " END LOOP;\n" + + "\n" + + " CLOSE user_cursor;\n" + + "END"; + ParseMysqlPLResult result = PLParser.parseObMysql(pl); + Assert.assertEquals(5, result.getVaribaleList().size()); + Assert.assertEquals("list_all_users", result.getPlName()); + Assert.assertEquals("PROCEDURE", result.getPlType()); + Assert.assertEquals(0, result.getParamList().size()); + } + + @Test + public void testParseMysqlProcedureContainsDefinerAndException() { + String pl = + "CREATE DEFINER = `root`@`%` PROCEDURE safe_update_user_email(IN user_id INT, IN new_email VARCHAR(100), OUT result_message VARCHAR(100))\n" + + + "BEGIN\n" + + " DECLARE CONTINUE HANDLER FOR SQLEXCEPTION\n" + + " BEGIN\n" + + " SET result_message = 'An error occurred while updating the email.';\n" + + " END;\n" + + "\n" + + " UPDATE users SET email = new_email WHERE id = user_id;\n" + + " IF ROW_COUNT() = 0 THEN\n" + + " SET result_message = 'No user found with the provided ID.';\n" + + " ELSE\n" + + " SET result_message = 'Email updated successfully.';\n" + + " END IF;\n" + + "END"; + ParseMysqlPLResult result = PLParser.parseObMysql(pl); + Assert.assertEquals(1, result.getVaribaleList().size()); + Assert.assertEquals("safe_update_user_email", result.getPlName()); + Assert.assertEquals("PROCEDURE", result.getPlType()); + Assert.assertEquals(3, result.getParamList().size()); + } + + @Test + public void testParseMysqlProcedureContainsDefinerAndMultipleTables() { + String pl = + "CREATE DEFINER = `root`@`%` PROCEDURE activate_user_and_log(IN user_id INT, OUT result_message VARCHAR(100))\n" + + + "BEGIN\n" + + " UPDATE users SET status = 'active' WHERE id = user_id;\n" + + " \n" + + " IF ROW_COUNT() > 0 THEN\n" + + " INSERT INTO logs (user_id, action, created_at) VALUES (user_id, 'Activated user', NOW());\n" + + + " SET result_message = 'User activated and logged.';\n" + + " ELSE\n" + + " SET result_message = 'User not found for activation.';\n" + + " END IF;\n" + + "END"; + ParseMysqlPLResult result = PLParser.parseObMysql(pl); + Assert.assertEquals(0, result.getVaribaleList().size()); + Assert.assertEquals("activate_user_and_log", result.getPlName()); + Assert.assertEquals("PROCEDURE", result.getPlType()); + Assert.assertEquals(2, result.getParamList().size()); + } + @Test public void testParseOracleProcedure() { String pl = "create procedure pl_test(p1 in int default 1, p2 in varchar2) \n" + "is \n" + "v1 number;\n" @@ -1183,4 +1349,54 @@ public void parseOracle_rollbackStmt_getSqlTypeSucceed() { Assert.assertEquals(SqlType.ROLLBACK, actual.getSqlType()); } + @Test + public void parseOracle_commentOnTable_getSqlTypeSucceed() { + ParseOraclePLResult actual = PLParser.parseOracle("comment on table a is 'xxx'"); + Assert.assertEquals(DBObjectType.TABLE, actual.getDbObjectType()); + Assert.assertEquals(SqlType.COMMENT_ON, actual.getSqlType()); + } + + @Test + public void parseOracle_commentOnColumn_getSqlTypeSucceed() { + ParseOraclePLResult actual = PLParser.parseOracle("comment on column a is 'xxx'"); + Assert.assertEquals(DBObjectType.COLUMN, actual.getDbObjectType()); + Assert.assertEquals(SqlType.COMMENT_ON, actual.getSqlType()); + } + + @Test + public void parseOracle_commentOnMaterialized_getSqlTypeSucceed() { + ParseOraclePLResult actual = PLParser.parseOracle("comment on materialized a is 'xxx'"); + Assert.assertEquals(DBObjectType.OTHERS, actual.getDbObjectType()); + Assert.assertEquals(SqlType.COMMENT_ON, actual.getSqlType()); + } + + @Test + public void parseOracle_call_getSqlTypeSucceed() { + ParseOraclePLResult actual = PLParser.parseOracle("call proc()"); + Assert.assertEquals(DBObjectType.PROCEDURE, actual.getDbObjectType()); + Assert.assertEquals(SqlType.CALL, actual.getSqlType()); + } + + @Test + public void parseOBMysql_call_getSqlTypeSucceed() { + ParseMysqlPLResult actual = PLParser.parseObMysql("call proc()"); + Assert.assertEquals(DBObjectType.PROCEDURE, actual.getDbObjectType()); + Assert.assertEquals(SqlType.CALL, actual.getSqlType()); + } + + @Test + public void test_oracle_alter_session() { + String sql = "alter SESSION set ob_query_timeout=6000000000;"; + ParseOraclePLResult result = PLParser.parseOracle(sql); + Assert.assertEquals(SqlType.ALTER_SESSION, result.getSqlType()); + Assert.assertEquals(DBObjectType.OTHERS, result.getDbObjectType()); + } + + @Test + public void test_oracle_set_session() { + String sql = "SET SESSION ob_query_timeout=6000000000;"; + ParseOraclePLResult result = PLParser.parseOracle(sql); + Assert.assertEquals(SqlType.SET_SESSION, result.getSqlType()); + Assert.assertEquals(DBObjectType.OTHERS, result.getDbObjectType()); + } } diff --git a/libs/db-browser/src/test/java/com/oceanbase/tools/dbbrowser/parser/ParserUtilTest.java b/libs/db-browser/src/test/java/com/oceanbase/tools/dbbrowser/parser/ParserUtilTest.java index 06d00c1d56..4a51bb61c2 100644 --- a/libs/db-browser/src/test/java/com/oceanbase/tools/dbbrowser/parser/ParserUtilTest.java +++ b/libs/db-browser/src/test/java/com/oceanbase/tools/dbbrowser/parser/ParserUtilTest.java @@ -288,4 +288,96 @@ public void test_mysql_pl_syntax_error() { BasicResult result = ParserUtil.parseOracleType(sql); Assert.assertTrue(result.getSyntaxError()); } + + @Test + public void test_mysql_set_session() { + String sql = "SET SESSION time_zone = '+00:00';"; + BasicResult result = ParserUtil.parseMysqlType(sql); + Assert.assertEquals(SqlType.SET_SESSION, result.getSqlType()); + Assert.assertEquals(SqlType.SET, result.getSqlType().getParentType()); + Assert.assertEquals(DBObjectType.SESSION_VARIABLE, result.getDbObjectType()); + } + + @Test + public void test_mysql_set_global() { + String sql = "SET GLOBAL time_zone = '+00:00';"; + BasicResult result = ParserUtil.parseMysqlType(sql); + Assert.assertEquals(SqlType.SET, result.getSqlType()); + Assert.assertEquals(DBObjectType.GLOBAL_VARIABLE, result.getDbObjectType()); + } + + @Test + public void test_oracle_set_global() { + String sql = "SET GLOBAL time_zone = '+00:00';"; + BasicResult result = ParserUtil.parseMysqlType(sql); + Assert.assertEquals(SqlType.SET, result.getSqlType()); + Assert.assertEquals(DBObjectType.GLOBAL_VARIABLE, result.getDbObjectType()); + } + + @Test + public void test_oracle_alter_system() { + String sql = "ALTER SYSTEM SET time_zone = '+00:00';"; + BasicResult result = ParserUtil.parseOracleType(sql); + Assert.assertEquals(SqlType.ALTER, result.getSqlType()); + Assert.assertEquals(DBObjectType.OTHERS, result.getDbObjectType()); + } + + @Test + public void test_mysql_set() { + String sql = "SET time_zone = '+00:00';"; + BasicResult result = ParserUtil.parseMysqlType(sql); + Assert.assertEquals(SqlType.SET, result.getSqlType()); + Assert.assertEquals(DBObjectType.SYSTEM_VARIABLE, result.getDbObjectType()); + } + + @Test + public void test_mysql_call() { + String sql = "call proc()"; + BasicResult result = ParserUtil.parseMysqlType(sql); + Assert.assertEquals(SqlType.CALL, result.getSqlType()); + Assert.assertEquals(DBObjectType.PROCEDURE, result.getDbObjectType()); + } + + @Test + public void test_oracle_call() { + String sql = "call proc()"; + BasicResult result = ParserUtil.parseOracleType(sql); + Assert.assertEquals(SqlType.CALL, result.getSqlType()); + Assert.assertEquals(DBObjectType.PROCEDURE, result.getDbObjectType()); + } + + @Test + public void test_oracle_alterSession() { + String sql = "alter SESSION set ob_query_timeout=6000000000;"; + BasicResult result = ParserUtil.parseOracleType(sql); + Assert.assertEquals(SqlType.ALTER_SESSION, result.getSqlType()); + Assert.assertEquals(SqlType.ALTER, result.getSqlType().getParentType()); + Assert.assertEquals(DBObjectType.OTHERS, result.getDbObjectType()); + } + + @Test + public void test_oracle_setSession() { + String sql = "set SESSION ob_query_timeout=6000000000;"; + BasicResult result = ParserUtil.parseOracleType(sql); + Assert.assertEquals(SqlType.SET_SESSION, result.getSqlType()); + Assert.assertEquals(SqlType.SET, result.getSqlType().getParentType()); + Assert.assertEquals(DBObjectType.SESSION_VARIABLE, result.getDbObjectType()); + } + + @Test + public void test_oracle_comment_on_table() { + String sql = "comment on table t is 'abc'"; + BasicResult result = ParserUtil.parseOracleType(sql); + Assert.assertEquals(SqlType.COMMENT_ON, result.getSqlType()); + Assert.assertEquals(DBObjectType.TABLE, result.getDbObjectType()); + } + + @Test + public void test_oracle_comment_on_column() { + String sql = "comment on column t is 'abc'"; + BasicResult result = ParserUtil.parseOracleType(sql); + Assert.assertEquals(SqlType.COMMENT_ON, result.getSqlType()); + Assert.assertEquals(DBObjectType.COLUMN, result.getDbObjectType()); + } + } diff --git a/libs/db-browser/src/test/java/com/oceanbase/tools/dbbrowser/parser/SqlParserTest.java b/libs/db-browser/src/test/java/com/oceanbase/tools/dbbrowser/parser/SqlParserTest.java index 006996da61..07057790cc 100644 --- a/libs/db-browser/src/test/java/com/oceanbase/tools/dbbrowser/parser/SqlParserTest.java +++ b/libs/db-browser/src/test/java/com/oceanbase/tools/dbbrowser/parser/SqlParserTest.java @@ -224,7 +224,7 @@ public void test_parse_mysql_fulltext_index_range() { + " `title` varchar(200) DEFAULT NULL,\n" + " `content` text DEFAULT NULL,\n" + " PRIMARY KEY (`id`), \n" - + " FULLTEXT KEY `title` (`title`, `content`) CTXCAT(`title`, `content`) WITH PARSER 'TAOBAO_CHN' BLOCK_SIZE 16384\n" + + " FULLTEXT KEY `title` (`title`, `content`) CTXCAT(`title`, `content`) WITH PARSER TAOBAO_CHN BLOCK_SIZE 16384\n" + ") "; ParseSqlResult result = SqlParser.parseMysql(sql); Assert.assertEquals(1, result.getIndexes().size()); @@ -284,4 +284,79 @@ public void parseOracle_rollbackStmt_getSqlTypeSucceed() { Assert.assertEquals(SqlType.ROLLBACK, actual.getSqlType()); } + @Test + public void parseOracle_commentOnTable_getSqlTypeSucceed() { + ParseSqlResult actual = SqlParser.parseOracle("comment on table a is 'xxx'"); + Assert.assertEquals(DBObjectType.TABLE, actual.getDbObjectType()); + Assert.assertEquals(SqlType.COMMENT_ON, actual.getSqlType()); + } + + @Test + public void parseOracle_commentOnColumn_getSqlTypeSucceed() { + ParseSqlResult actual = SqlParser.parseOracle("comment on column a is 'xxx'"); + Assert.assertEquals(DBObjectType.COLUMN, actual.getDbObjectType()); + Assert.assertEquals(SqlType.COMMENT_ON, actual.getSqlType()); + } + + @Test + public void parseOracle_call_getSqlTypeSucceed() { + ParseSqlResult actual = SqlParser.parseOracle("call proc()"); + Assert.assertEquals(DBObjectType.PROCEDURE, actual.getDbObjectType()); + Assert.assertEquals(SqlType.CALL, actual.getSqlType()); + } + + + @Test + public void parseMysql_setSession_getSqlTypeSucceed() { + ParseSqlResult actual = SqlParser.parseMysql("SET SESSION time_zone = '+00:00';"); + Assert.assertEquals(DBObjectType.SESSION_VARIABLE, actual.getDbObjectType()); + Assert.assertEquals(SqlType.SET_SESSION, actual.getSqlType()); + Assert.assertEquals(SqlType.SET, actual.getSqlType().getParentType()); + } + + @Test + public void parseMysql_setGlobal_getSqlTypeSucceed() { + ParseSqlResult actual = SqlParser.parseMysql("SET GLOBAL time_zone = '+00:00';"); + Assert.assertEquals(DBObjectType.GLOBAL_VARIABLE, actual.getDbObjectType()); + Assert.assertEquals(SqlType.SET, actual.getSqlType()); + } + + @Test + public void parseMysql_set_getSqlTypeSucceed() { + ParseSqlResult actual = SqlParser.parseMysql("SET time_zone = '+00:00';"); + Assert.assertEquals(DBObjectType.SYSTEM_VARIABLE, actual.getDbObjectType()); + Assert.assertEquals(SqlType.SET, actual.getSqlType()); + } + + @Test + public void parseOracle_alterSession_getSqlTypeSucceed() { + ParseSqlResult actual = SqlParser.parseOracle("alter SESSION set ob_query_timeout=6000000000;"); + Assert.assertEquals(SqlType.ALTER_SESSION, actual.getSqlType()); + Assert.assertEquals(SqlType.ALTER, actual.getSqlType().getParentType()); + Assert.assertEquals(DBObjectType.OTHERS, actual.getDbObjectType()); + } + + @Test + public void parseOracle_setSession_getSqlTypeSucceed() { + ParseSqlResult actual = SqlParser.parseOracle("set SESSION ob_query_timeout=6000000000;"); + Assert.assertEquals(SqlType.SET_SESSION, actual.getSqlType()); + Assert.assertEquals(SqlType.SET, actual.getSqlType().getParentType()); + Assert.assertEquals(DBObjectType.SESSION_VARIABLE, actual.getDbObjectType()); + } + + @Test + public void parseOracle_setGlobal_getSqlTypeSucceed() { + ParseSqlResult actual = SqlParser.parseOracle("SET GLOBAL time_zone = '+00:00';"); + Assert.assertEquals(DBObjectType.GLOBAL_VARIABLE, actual.getDbObjectType()); + Assert.assertEquals(SqlType.SET, actual.getSqlType()); + } + + @Test + public void parseOracle_alterSystem_getSqlTypeSucceed() { + ParseSqlResult actual = SqlParser.parseOracle("alter system set time_zone = '+00:00';"); + Assert.assertEquals(DBObjectType.OTHERS, actual.getDbObjectType()); + Assert.assertEquals(SqlType.ALTER, actual.getSqlType()); + } + + } diff --git a/libs/db-browser/src/test/java/com/oceanbase/tools/dbbrowser/schema/OracleSchemaAccessorTest.java b/libs/db-browser/src/test/java/com/oceanbase/tools/dbbrowser/schema/OracleSchemaAccessorTest.java index 87f98a3e1c..4e867b4f1c 100644 --- a/libs/db-browser/src/test/java/com/oceanbase/tools/dbbrowser/schema/OracleSchemaAccessorTest.java +++ b/libs/db-browser/src/test/java/com/oceanbase/tools/dbbrowser/schema/OracleSchemaAccessorTest.java @@ -137,6 +137,16 @@ public void listTableColumns_Success() { Assert.assertEquals(13, columns.size()); } + @Test + public void listTableColumns_TestCreateExtendedStats_Success() { + String sql = "SELECT DBMS_STATS.CREATE_EXTENDED_STATS('" + getOracleSchema() + + "', 'TEST_EXTENDED_STATS_COL', '(X, Y)') FROM DUAL"; + jdbcTemplate.execute(sql); + List columns = + accessor.listTableColumns(getOracleSchema(), "TEST_EXTENDED_STATS_COL"); + Assert.assertEquals(3, columns.size()); + } + @Test public void listTableConstraint_TestPrimaryKey_Success() { List constraintListList = diff --git a/libs/db-browser/src/test/java/com/oceanbase/tools/dbbrowser/stats/OBMySQLStatsAccessorTest.java b/libs/db-browser/src/test/java/com/oceanbase/tools/dbbrowser/stats/OBMySQLStatsAccessorTest.java index 3a52ee29f8..1ea11da4fd 100644 --- a/libs/db-browser/src/test/java/com/oceanbase/tools/dbbrowser/stats/OBMySQLStatsAccessorTest.java +++ b/libs/db-browser/src/test/java/com/oceanbase/tools/dbbrowser/stats/OBMySQLStatsAccessorTest.java @@ -45,8 +45,9 @@ public void currentSession() { @Test public void listAllSessions() { DBStatsAccessor accessor = new OBMySQLStatsAccessor(new JdbcTemplate(getOBMySQLDataSource())); - List session = accessor.listAllSessions(); - Assert.assertTrue(session.size() > 0); + List sessions = accessor.listAllSessions(); + Assert.assertTrue(sessions.size() > 0); + sessions.stream().forEach(s -> Assert.assertNotNull(s.getSvrIp())); } } diff --git a/libs/db-browser/src/test/java/com/oceanbase/tools/dbbrowser/stats/OBOracleNoLessThan2270StatsAccessorTest.java b/libs/db-browser/src/test/java/com/oceanbase/tools/dbbrowser/stats/OBOracleNoLessThan2270StatsAccessorTest.java index 82feec931a..740110649e 100644 --- a/libs/db-browser/src/test/java/com/oceanbase/tools/dbbrowser/stats/OBOracleNoLessThan2270StatsAccessorTest.java +++ b/libs/db-browser/src/test/java/com/oceanbase/tools/dbbrowser/stats/OBOracleNoLessThan2270StatsAccessorTest.java @@ -50,5 +50,6 @@ public void listAllSessions_Succeed() { DBStatsAccessor accessor = new OBOracleNoLessThan2270StatsAccessor(new JdbcTemplate(getOBOracleDataSource())); List sessions = accessor.listAllSessions(); Assert.assertTrue(sessions.size() > 0); + sessions.stream().forEach(s -> Assert.assertNotNull(s.getSvrIp())); } } diff --git a/libs/db-browser/src/test/resources/table/oracle/drop.sql b/libs/db-browser/src/test/resources/table/oracle/drop.sql index a492143ad2..abc60f8e79 100644 --- a/libs/db-browser/src/test/resources/table/oracle/drop.sql +++ b/libs/db-browser/src/test/resources/table/oracle/drop.sql @@ -26,6 +26,8 @@ call DROPIFEXISTS_TABLE('TEST_INDEX_TYPE') call DROPIFEXISTS_TABLE('TEST_VIEW_TABLE') / call DROPIFEXISTS_TABLE('PART_HASH_TEST') +/ +call DROPIFEXISTS_TABLE('TEST_EXTENDED_STATS_COL') diff --git a/libs/db-browser/src/test/resources/table/oracle/testTableColumnDDL.sql b/libs/db-browser/src/test/resources/table/oracle/testTableColumnDDL.sql index 228f8c78b9..5fa93ab5ef 100644 --- a/libs/db-browser/src/test/resources/table/oracle/testTableColumnDDL.sql +++ b/libs/db-browser/src/test/resources/table/oracle/testTableColumnDDL.sql @@ -13,4 +13,10 @@ create table TEST_COL_DATA_TYPE( col12 interval year to month, col13 interval day to second ) +/ + +CREATE TABLE TEST_EXTENDED_STATS_COL( + "X" NUMBER(*,0), + "Y" NUMBER(*,0) +) / \ No newline at end of file diff --git a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLAlterTableActionFactory.java b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLAlterTableActionFactory.java index 96502b8838..6526049e79 100644 --- a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLAlterTableActionFactory.java +++ b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLAlterTableActionFactory.java @@ -15,18 +15,18 @@ */ package com.oceanbase.tools.sqlparser.adapter.mysql; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; +import java.util.*; import java.util.stream.Collectors; import org.antlr.v4.runtime.ParserRuleContext; import com.oceanbase.tools.sqlparser.adapter.StatementFactory; +import com.oceanbase.tools.sqlparser.obmysql.OBParser.Add_external_table_partition_actionsContext; import com.oceanbase.tools.sqlparser.obmysql.OBParser.Alter_column_behaviorContext; -import com.oceanbase.tools.sqlparser.obmysql.OBParser.Alter_column_group_optionContext; +import com.oceanbase.tools.sqlparser.obmysql.OBParser.Alter_column_group_actionContext; import com.oceanbase.tools.sqlparser.obmysql.OBParser.Alter_column_optionContext; import com.oceanbase.tools.sqlparser.obmysql.OBParser.Alter_constraint_optionContext; +import com.oceanbase.tools.sqlparser.obmysql.OBParser.Alter_external_table_actionContext; import com.oceanbase.tools.sqlparser.obmysql.OBParser.Alter_index_optionContext; import com.oceanbase.tools.sqlparser.obmysql.OBParser.Alter_partition_optionContext; import com.oceanbase.tools.sqlparser.obmysql.OBParser.Alter_table_actionContext; @@ -35,6 +35,7 @@ import com.oceanbase.tools.sqlparser.obmysql.OBParser.Name_listContext; import com.oceanbase.tools.sqlparser.obmysql.OBParser.Opt_partition_range_or_listContext; import com.oceanbase.tools.sqlparser.obmysql.OBParserBaseVisitor; +import com.oceanbase.tools.sqlparser.statement.Expression; import com.oceanbase.tools.sqlparser.statement.alter.table.AlterTableAction; import com.oceanbase.tools.sqlparser.statement.alter.table.AlterTableAction.AlterColumnBehavior; import com.oceanbase.tools.sqlparser.statement.common.ColumnGroupElement; @@ -63,10 +64,14 @@ public MySQLAlterTableActionFactory(@NonNull Alter_table_actionContext alterTabl this.parserRuleContext = alterTableActionContext; } - public MySQLAlterTableActionFactory(@NonNull Alter_column_group_optionContext alterTableActionContext) { + public MySQLAlterTableActionFactory(@NonNull Alter_column_group_actionContext alterTableActionContext) { this.parserRuleContext = alterTableActionContext; } + public MySQLAlterTableActionFactory(@NonNull Alter_external_table_actionContext alterExternalTableActionContext) { + this.parserRuleContext = alterExternalTableActionContext; + } + @Override public AlterTableAction generate() { return visit(this.parserRuleContext); @@ -95,6 +100,20 @@ public AlterTableAction visitAlter_table_action(Alter_table_actionContext ctx) { return visitChildren(ctx); } + @Override + public AlterTableAction visitAlter_external_table_action(Alter_external_table_actionContext ctx) { + AlterTableAction action = new AlterTableAction(ctx); + action.setExternalTableLocation(ctx.STRING_VALUE().getText()); + if (ctx.DROP() != null && ctx.PARTITION() != null) { + action.setDropExternalTablePartition(true); + } else if (ctx.ADD() != null && ctx.PARTITION() != null) { + Map externalTablePartition = new HashMap<>(); + visitAddExternalTablePartitionActions(externalTablePartition, ctx.add_external_table_partition_actions()); + action.setAddExternalTablePartition(externalTablePartition); + } + return action; + } + @Override public AlterTableAction visitAlter_column_option(Alter_column_optionContext ctx) { AlterTableAction alterTableAction = new AlterTableAction(ctx); @@ -108,6 +127,10 @@ public AlterTableAction visitAlter_column_option(Alter_column_optionContext ctx) .collect(Collectors.toList()); } alterTableAction.setAddColumns(addColumns); + if (ctx.lob_storage_clause() != null) { + alterTableAction.setLobStorageOption( + MySQLTableOptionsFactory.getLobStorageOption(ctx.lob_storage_clause())); + } } else if (ctx.DROP() != null) { String option = null; if (ctx.CASCADE() != null) { @@ -131,6 +154,8 @@ public AlterTableAction visitAlter_column_option(Alter_column_optionContext ctx) AlterColumnBehavior behavior = new AlterColumnBehavior(aCtx); if (aCtx.signed_literal() != null) { behavior.setDefaultValue(MySQLTableElementFactory.getSignedLiteral(aCtx.signed_literal())); + } else if (aCtx.expr() != null) { + behavior.setDefaultValue(new MySQLExpressionFactory(aCtx.expr()).generate()); } alterTableAction.alterColumnBehavior(colRef, behavior); } else if (ctx.RENAME() != null) { @@ -211,6 +236,9 @@ public AlterTableAction visitAlter_partition_option(Alter_partition_optionContex getPartitionElements(ctx.opt_partition_range_or_list())); } else if (ctx.REMOVE() != null && ctx.PARTITIONING() != null) { alterTableAction.setRemovePartitioning(true); + } else if (ctx.EXCHANGE() != null && ctx.PARTITION() != null) { + alterTableAction.setExchangePartition(ctx.relation_name().getText(), + MySQLFromReferenceFactory.getRelationFactor(ctx.relation_factor())); } return alterTableAction; } @@ -241,7 +269,7 @@ public AlterTableAction visitAlter_constraint_option(Alter_constraint_optionCont } @Override - public AlterTableAction visitAlter_column_group_option(Alter_column_group_optionContext ctx) { + public AlterTableAction visitAlter_column_group_action(Alter_column_group_actionContext ctx) { AlterTableAction action = new AlterTableAction(ctx); List columnGroupElements = ctx.column_group_list().column_group_element() .stream().map(c -> new MySQLColumnGroupElementFactory(c).generate()).collect(Collectors.toList()); @@ -273,4 +301,15 @@ private List getPartitionElements(Opt_partition_range_or_listC .map(c -> new MySQLPartitionElementFactory(c).generate()).collect(Collectors.toList()); } + private void visitAddExternalTablePartitionActions(Map externalTablePartition, + Add_external_table_partition_actionsContext context) { + if (context == null) { + return; + } + Expression value = new MySQLExpressionFactory() + .visit(context.add_external_table_partition_action().expr_const()); + externalTablePartition.put(context.add_external_table_partition_action().column_name().getText(), value); + visitAddExternalTablePartitionActions(externalTablePartition, context.add_external_table_partition_actions()); + } + } diff --git a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLAlterTableFactory.java b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLAlterTableFactory.java index 415eede3e2..1f0d6cd107 100644 --- a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLAlterTableFactory.java +++ b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLAlterTableFactory.java @@ -22,7 +22,8 @@ import org.antlr.v4.runtime.ParserRuleContext; import com.oceanbase.tools.sqlparser.adapter.StatementFactory; -import com.oceanbase.tools.sqlparser.obmysql.OBParser.Alter_column_group_optionContext; +import com.oceanbase.tools.sqlparser.obmysql.OBParser.Alter_column_group_actionContext; +import com.oceanbase.tools.sqlparser.obmysql.OBParser.Alter_external_table_actionContext; import com.oceanbase.tools.sqlparser.obmysql.OBParser.Alter_table_actionsContext; import com.oceanbase.tools.sqlparser.obmysql.OBParser.Alter_table_stmtContext; import com.oceanbase.tools.sqlparser.obmysql.OBParserBaseVisitor; @@ -58,8 +59,10 @@ public AlterTable visitAlter_table_stmt(Alter_table_stmtContext ctx) { List actions = null; if (ctx.alter_table_actions() != null) { actions = getAlterTableActions(ctx.alter_table_actions()); - } else if (ctx.alter_column_group_option() != null) { - actions = getAlterTableActions(ctx.alter_column_group_option()); + } else if (ctx.alter_column_group_action() != null) { + actions = getAlterTableActions(ctx.alter_column_group_action()); + } else if (ctx.alter_external_table_action() != null) { + actions = getAlterTableActions(ctx.alter_external_table_action()); } AlterTable alterTable = new AlterTable(ctx, factor, actions); if (ctx.EXTERNAL() != null) { @@ -83,7 +86,11 @@ private List getAlterTableActions(Alter_table_actionsContext c return actions; } - private List getAlterTableActions(Alter_column_group_optionContext context) { + private List getAlterTableActions(Alter_column_group_actionContext context) { + return Collections.singletonList(new MySQLAlterTableActionFactory(context).generate()); + } + + private List getAlterTableActions(Alter_external_table_actionContext context) { return Collections.singletonList(new MySQLAlterTableActionFactory(context).generate()); } diff --git a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLCreateTableFactory.java b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLCreateTableFactory.java index 35bb5c5e52..4f23429d73 100644 --- a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLCreateTableFactory.java +++ b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLCreateTableFactory.java @@ -69,8 +69,7 @@ public CreateTable visitCreate_table_like_stmt(Create_table_like_stmtContext ctx createTable.setIfNotExists(true); } createTable.setUserVariable(factor.getUserVariable()); - RelationFactor likeFactor = MySQLFromReferenceFactory.getRelationFactor(ctx.relation_factor(1)); - createTable.setLikeTable(likeFactor); + createTable.setLikeTable(MySQLFromReferenceFactory.getRelationFactor(ctx.relation_factor(1))); return createTable; } @@ -86,7 +85,6 @@ public CreateTable visitCreate_table_stmt(Create_table_stmtContext ctx) { createTable.setTemporary(true); } } - if (ctx.IF() != null && ctx.not() != null && ctx.EXISTS() != null) { createTable.setIfNotExists(true); } @@ -112,6 +110,10 @@ public CreateTable visitCreate_table_stmt(Create_table_stmtContext ctx) { .map(c -> new MySQLColumnGroupElementFactory(c).generate()).collect(Collectors.toList()); createTable.setColumnGroupElements(columnGroupElements); } + if (ctx.ignore_or_replace() != null) { + createTable.setIgnore(ctx.ignore_or_replace().IGNORE() != null); + createTable.setReplace(ctx.ignore_or_replace().REPLACE() != null); + } return createTable; } diff --git a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLDataTypeFactory.java b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLDataTypeFactory.java index f46fb7cb16..8ece4121bd 100644 --- a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLDataTypeFactory.java +++ b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLDataTypeFactory.java @@ -90,7 +90,7 @@ public DataType visitData_type(Data_typeContext ctx) { if (ctx.STRING_VALUE() != null) { return new GeneralDataType(ctx, ctx.STRING_VALUE().getText(), null); } else if (ctx.data_type() != null) { - return new ArrayType(visitData_type(ctx.data_type())); + return new ArrayType(ctx, visitData_type(ctx.data_type())); } return visitChildren(ctx); } diff --git a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLExpressionFactory.java b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLExpressionFactory.java index 0f072f42ea..d9d1d60b1b 100644 --- a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLExpressionFactory.java +++ b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLExpressionFactory.java @@ -45,6 +45,7 @@ import com.oceanbase.tools.sqlparser.obmysql.OBParser.Expr_listContext; import com.oceanbase.tools.sqlparser.obmysql.OBParser.In_exprContext; import com.oceanbase.tools.sqlparser.obmysql.OBParser.Json_on_responseContext; +import com.oceanbase.tools.sqlparser.obmysql.OBParser.Json_query_exprContext; import com.oceanbase.tools.sqlparser.obmysql.OBParser.Json_table_column_defContext; import com.oceanbase.tools.sqlparser.obmysql.OBParser.Json_table_exists_column_defContext; import com.oceanbase.tools.sqlparser.obmysql.OBParser.Json_table_exprContext; @@ -56,7 +57,11 @@ import com.oceanbase.tools.sqlparser.obmysql.OBParser.Mock_jt_on_error_on_emptyContext; import com.oceanbase.tools.sqlparser.obmysql.OBParser.Mvt_paramContext; import com.oceanbase.tools.sqlparser.obmysql.OBParser.On_emptyContext; +import com.oceanbase.tools.sqlparser.obmysql.OBParser.On_empty_queryContext; import com.oceanbase.tools.sqlparser.obmysql.OBParser.On_errorContext; +import com.oceanbase.tools.sqlparser.obmysql.OBParser.On_error_queryContext; +import com.oceanbase.tools.sqlparser.obmysql.OBParser.On_mismatch_queryContext; +import com.oceanbase.tools.sqlparser.obmysql.OBParser.Opt_response_queryContext; import com.oceanbase.tools.sqlparser.obmysql.OBParser.Opt_value_on_empty_or_error_or_mismatchContext; import com.oceanbase.tools.sqlparser.obmysql.OBParser.Parameterized_trimContext; import com.oceanbase.tools.sqlparser.obmysql.OBParser.PredicateContext; @@ -68,8 +73,10 @@ import com.oceanbase.tools.sqlparser.obmysql.OBParser.Ttl_exprContext; import com.oceanbase.tools.sqlparser.obmysql.OBParser.Utc_time_funcContext; import com.oceanbase.tools.sqlparser.obmysql.OBParser.Utc_timestamp_funcContext; +import com.oceanbase.tools.sqlparser.obmysql.OBParser.Vector_distance_exprContext; import com.oceanbase.tools.sqlparser.obmysql.OBParser.Win_fun_first_last_paramsContext; import com.oceanbase.tools.sqlparser.obmysql.OBParser.Window_functionContext; +import com.oceanbase.tools.sqlparser.obmysql.OBParser.Wrapper_optsContext; import com.oceanbase.tools.sqlparser.obmysql.OBParser.Ws_nweightsContext; import com.oceanbase.tools.sqlparser.obmysql.OBParserBaseVisitor; import com.oceanbase.tools.sqlparser.statement.Expression; @@ -79,24 +86,7 @@ import com.oceanbase.tools.sqlparser.statement.common.CharacterType; import com.oceanbase.tools.sqlparser.statement.common.GeneralDataType; import com.oceanbase.tools.sqlparser.statement.common.WindowSpec; -import com.oceanbase.tools.sqlparser.statement.expression.ArrayExpression; -import com.oceanbase.tools.sqlparser.statement.expression.BoolValue; -import com.oceanbase.tools.sqlparser.statement.expression.CaseWhen; -import com.oceanbase.tools.sqlparser.statement.expression.CollectionExpression; -import com.oceanbase.tools.sqlparser.statement.expression.ColumnReference; -import com.oceanbase.tools.sqlparser.statement.expression.CompoundExpression; -import com.oceanbase.tools.sqlparser.statement.expression.ConstExpression; -import com.oceanbase.tools.sqlparser.statement.expression.DefaultExpression; -import com.oceanbase.tools.sqlparser.statement.expression.ExpressionParam; -import com.oceanbase.tools.sqlparser.statement.expression.FullTextSearch; -import com.oceanbase.tools.sqlparser.statement.expression.FunctionCall; -import com.oceanbase.tools.sqlparser.statement.expression.FunctionParam; -import com.oceanbase.tools.sqlparser.statement.expression.GroupConcat; -import com.oceanbase.tools.sqlparser.statement.expression.IntervalExpression; -import com.oceanbase.tools.sqlparser.statement.expression.JsonOnOption; -import com.oceanbase.tools.sqlparser.statement.expression.NullExpression; -import com.oceanbase.tools.sqlparser.statement.expression.TextSearchMode; -import com.oceanbase.tools.sqlparser.statement.expression.WhenClause; +import com.oceanbase.tools.sqlparser.statement.expression.*; import lombok.NonNull; @@ -223,6 +213,9 @@ public Expression visitBool_pri(Bool_priContext ctx) { } else if (ctx.bool_pri() != null && ctx.any_expr() != null) { left = visit(ctx.bool_pri()); right = visit(ctx.any_expr()); + } else if (ctx.bool_pri() != null && ctx.select_with_parens() != null) { + left = visit(ctx.bool_pri()); + right = new MySQLSelectBodyFactory(ctx.select_with_parens()).generate(); } if (left == null || right == null || operator == null) { throw new IllegalStateException("Unable to build expression, some syntax modules are missing"); @@ -389,6 +382,9 @@ public Expression visitSimple_expr(Simple_exprContext ctx) { } FullTextSearch f = new FullTextSearch(ctx, params, ctx.STRING_VALUE().getText()); f.setSearchMode(searchMode); + if (ctx.WITH() != null && ctx.QUERY() != null && ctx.EXPANSION() != null) { + f.setWithQueryExpansion(true); + } return f; } else if (ctx.func_expr() != null) { return visit(ctx.func_expr()); @@ -477,6 +473,8 @@ public Expression visitSimple_func_expr(Simple_func_exprContext ctx) { p.addOption(new ConstExpression(e.STRING_VALUE())); return p; }).collect(Collectors.toList())); + } else if (ctx.INTNUM() != null) { + params.add(new ExpressionParam(new ConstExpression(ctx.INTNUM()))); } else if (ctx.column_ref() != null) { params.add(new ExpressionParam(new MySQLColumnRefFactory(ctx.column_ref()).generate())); if (CollectionUtils.isNotEmpty(ctx.mvt_param())) { @@ -533,6 +531,9 @@ public Expression visitComplex_func_expr(Complex_func_exprContext ctx) { } else if (ctx.CAST() != null) { FunctionParam p = wrap(ctx.expr()); p.addOption(new MySQLDataTypeFactory(ctx.cast_data_type()).generate()); + if (ctx.ARRAY() != null) { + p.addOption(new ConstExpression(ctx.ARRAY())); + } return new FunctionCall(ctx, ctx.CAST().getText(), Collections.singletonList(p)); } else if (ctx.CONVERT() != null) { FunctionParam p = wrap(ctx.expr()); @@ -635,36 +636,11 @@ public Expression visitComplex_func_expr(Complex_func_exprContext ctx) { } return fCall; } else if (ctx.json_value_expr() != null) { - Json_value_exprContext jsonValue = ctx.json_value_expr(); - List params = new ArrayList<>(); - params.add(new ExpressionParam(visit(jsonValue.simple_expr()))); - params.add(new ExpressionParam(visit(jsonValue.complex_string_literal()))); - FunctionCall fCall = new FunctionCall(ctx, jsonValue.JSON_VALUE().getText(), params); - if (jsonValue.cast_data_type() != null) { - fCall.addOption(new MySQLDataTypeFactory(jsonValue.cast_data_type()).generate()); - } - if (jsonValue.TRUNCATE() != null) { - fCall.addOption(new ConstExpression(jsonValue.TRUNCATE())); - } - if (jsonValue.ASCII() != null) { - fCall.addOption(new ConstExpression(jsonValue.ASCII())); - } - JsonOnOption jsonOnOption; - if (jsonValue.on_empty() != null && jsonValue.on_error() != null) { - jsonOnOption = new JsonOnOption(jsonValue.on_empty(), jsonValue.on_error()); - setOnError(jsonOnOption, jsonValue.on_error()); - setOnEmpty(jsonOnOption, jsonValue.on_empty()); - fCall.addOption(jsonOnOption); - } else if (jsonValue.on_error() != null) { - jsonOnOption = new JsonOnOption(jsonValue.on_error()); - setOnError(jsonOnOption, jsonValue.on_error()); - fCall.addOption(jsonOnOption); - } else if (jsonValue.on_empty() != null) { - jsonOnOption = new JsonOnOption(jsonValue.on_empty()); - setOnEmpty(jsonOnOption, jsonValue.on_empty()); - fCall.addOption(jsonOnOption); - } - return fCall; + return visit(ctx.json_value_expr()); + } else if (ctx.json_query_expr() != null) { + return visit(ctx.json_query_expr()); + } else if (ctx.vector_distance_expr() != null) { + return visit(ctx.vector_distance_expr()); } String funcName = null; List params = new ArrayList<>(); @@ -711,10 +687,6 @@ public Expression visitComplex_func_expr(Complex_func_exprContext ctx) { } else if (ctx.sys_interval_func().CHECK() != null) { funcName = ctx.sys_interval_func().CHECK().getText(); } - } else if (ctx.vector_distance_expr() != null) { - params = ctx.vector_distance_expr().expr() - .stream().map(this::wrap).collect(Collectors.toList()); - funcName = ctx.vector_distance_expr().VECTOR_DISTANCE().getText(); } if (funcName == null) { throw new IllegalStateException("Missing function name"); @@ -722,11 +694,93 @@ public Expression visitComplex_func_expr(Complex_func_exprContext ctx) { return new FunctionCall(ctx, funcName, params); } + @Override + public Expression visitJson_query_expr(Json_query_exprContext ctx) { + List params = new ArrayList<>(); + params.add(new ExpressionParam(visit(ctx.simple_expr()))); + params.add(new ExpressionParam(visit(ctx.complex_string_literal()))); + FunctionCall fCall = new FunctionCall(ctx, ctx.JSON_QUERY().getText(), params); + if (ctx.cast_data_type() != null) { + fCall.addOption(new MySQLDataTypeFactory(ctx.cast_data_type()).generate()); + } + if (ctx.json_query_opt() != null) { + JsonOption jsonOpt = new JsonOption(ctx.json_query_opt()); + fCall.addOption(jsonOpt); + if (ctx.json_query_opt().TRUNCATE() != null) { + jsonOpt.setTruncate(true); + } + if (ctx.json_query_opt().scalars_opt() != null) { + if (ctx.json_query_opt().scalars_opt().ALLOW() != null) { + jsonOpt.setScalarsMode(JsonOption.ScalarsMode.ALLOW_SCALARS); + } else { + jsonOpt.setScalarsMode(JsonOption.ScalarsMode.DISALLOW_SCALARS); + } + } + if (ctx.json_query_opt().PRETTY() != null) { + jsonOpt.setPretty(true); + } + if (ctx.json_query_opt().ASCII() != null) { + jsonOpt.setAscii(true); + } + setWrapperMode(jsonOpt, ctx.json_query_opt().wrapper_opts()); + if (ctx.json_query_opt().ASIS() != null) { + jsonOpt.setAsis(true); + } + if (ctx.json_query_opt().json_query_on_opt() != null) { + JsonOnOption jsonOnOption = new JsonOnOption(ctx.json_query_opt().json_query_on_opt()); + jsonOpt.setOnOption(jsonOnOption); + setOnError(jsonOnOption, ctx.json_query_opt().json_query_on_opt().on_error_query()); + setOnEmpty(jsonOnOption, ctx.json_query_opt().json_query_on_opt().on_empty_query()); + setOnMismatch(jsonOnOption, ctx.json_query_opt().json_query_on_opt().on_mismatch_query()); + } + if (ctx.json_query_opt().MULTIVALUE() != null) { + jsonOpt.setMultiValue(true); + } + } + return fCall; + } + + @Override + public Expression visitVector_distance_expr(Vector_distance_exprContext ctx) { + List params = ctx.expr().stream().map(this::wrap).collect(Collectors.toList()); + if (ctx.vector_distance_metric() != null) { + params.add(new ExpressionParam(new ConstExpression(ctx.vector_distance_metric()))); + } + return new FunctionCall(ctx, ctx.VECTOR_DISTANCE().getText(), params); + } + + @Override + public Expression visitJson_value_expr(Json_value_exprContext ctx) { + List params = new ArrayList<>(); + params.add(new ExpressionParam(visit(ctx.simple_expr()))); + params.add(new ExpressionParam(visit(ctx.complex_string_literal()))); + FunctionCall fCall = new FunctionCall(ctx, ctx.JSON_VALUE().getText(), params); + if (ctx.cast_data_type() != null) { + fCall.addOption(new MySQLDataTypeFactory(ctx.cast_data_type()).generate()); + } + if (ctx.json_value_opt() != null) { + JsonOption jsonOpt = new JsonOption(ctx.json_value_opt()); + fCall.addOption(jsonOpt); + if (ctx.json_value_opt().TRUNCATE() != null) { + jsonOpt.setTruncate(true); + } + if (ctx.json_value_opt().ASCII() != null) { + jsonOpt.setAscii(true); + } + if (ctx.json_value_opt().json_value_on_opt() != null) { + JsonOnOption jsonOnOption = new JsonOnOption(ctx.json_value_opt().json_value_on_opt()); + jsonOpt.setOnOption(jsonOnOption); + setOnError(jsonOnOption, ctx.json_value_opt().json_value_on_opt().on_error()); + setOnEmpty(jsonOnOption, ctx.json_value_opt().json_value_on_opt().on_empty()); + } + } + return fCall; + } + @Override public Expression visitTtl_expr(Ttl_exprContext ctx) { Expression right = new IntervalExpression(ctx.INTERVAL(), ctx.ttl_unit(), new ConstExpression(ctx.INTNUM()), ctx.ttl_unit().getText()); - return new CompoundExpression(ctx, new MySQLColumnRefFactory(ctx.column_definition_ref()).generate(), right, Operator.ADD); } @@ -881,7 +935,7 @@ public Expression visitJson_table_expr(Json_table_exprContext ctx) { FunctionCall fCall = new FunctionCall(ctx, ctx.JSON_TABLE().getText(), Arrays.asList(new ExpressionParam(visit(ctx.simple_expr())), new ExpressionParam(visit(ctx.literal())))); - fCall.addOption(getJsonOnOption(ctx.mock_jt_on_error_on_empty())); + fCall.addOption(getJsonOption(ctx.mock_jt_on_error_on_empty())); ctx.jt_column_list().json_table_column_def().forEach(c -> fCall.addOption(visitJsonTableColumnDef(c))); return fCall; } @@ -894,7 +948,12 @@ public Expression visitAny_expr(Any_exprContext ctx) { return visit(ctx.expr_list()); } - private JsonOnOption getJsonOnOption(Mock_jt_on_error_on_emptyContext ctx) { + @Override + public Expression visitOpt_response_query(Opt_response_queryContext ctx) { + return ctx.NULLX() != null ? new ConstExpression(ctx.NULLX()) : new ConstExpression(ctx.ERROR_P()); + } + + private JsonOnOption getJsonOption(Mock_jt_on_error_on_emptyContext ctx) { return null; } @@ -924,6 +983,53 @@ private void setOnError(JsonOnOption jsonOnOption, On_errorContext ctx) { jsonOnOption.setOnError(visit(ctx.json_on_response())); } + private void setOnEmpty(JsonOnOption jsonOnOption, On_empty_queryContext ctx) { + if (ctx == null) { + return; + } + if (ctx.opt_response_query() != null) { + jsonOnOption.setOnEmpty(visit(ctx.opt_response_query())); + } else if (ctx.EMPTY().size() == 2) { + TerminalNode endNode = ctx.EMPTY(0); + if (ctx.ARRAY() != null) { + endNode = ctx.ARRAY(); + } else if (ctx.OBJECT() != null) { + endNode = ctx.OBJECT(); + } + jsonOnOption.setOnEmpty(new ConstExpression(ctx.EMPTY(0), endNode)); + } + } + + private void setOnError(JsonOnOption jsonOnOption, On_error_queryContext ctx) { + if (ctx == null) { + return; + } + if (ctx.opt_response_query() != null) { + jsonOnOption.setOnError(visit(ctx.opt_response_query())); + } else if (ctx.EMPTY() != null) { + TerminalNode endNode = ctx.EMPTY(); + if (ctx.ARRAY() != null) { + endNode = ctx.ARRAY(); + } else if (ctx.OBJECT() != null) { + endNode = ctx.OBJECT(); + } + jsonOnOption.setOnError(new ConstExpression(ctx.EMPTY(), endNode)); + } + } + + private void setOnMismatch(JsonOnOption jsonOnOption, On_mismatch_queryContext ctx) { + if (ctx == null) { + return; + } + Expression opt; + if (ctx.DOT() != null) { + opt = new ConstExpression(ctx.DOT()); + } else { + opt = visit(ctx.opt_response_query()); + } + jsonOnOption.setOnMismatches(Collections.singletonList(new JsonOnOption.OnMismatch(ctx, opt, null))); + } + private FunctionParam visitJsonTableColumnDef(Json_table_column_defContext ctx) { if (ctx.json_table_ordinality_column_def() != null) { return visitJsonTableOrdinalityColumnDef(ctx.json_table_ordinality_column_def()); @@ -951,7 +1057,7 @@ private FunctionParam visitJsonTableExistsColumnDef(Json_table_exists_column_def } param.addOption(new ConstExpression(ctx.EXISTS())); param.addOption(visit(ctx.literal())); - param.addOption(getJsonOnOption(ctx.mock_jt_on_error_on_empty())); + param.addOption(getJsonOption(ctx.mock_jt_on_error_on_empty())); return param; } @@ -963,7 +1069,11 @@ private FunctionParam visitJsonTableValueColumnDef(Json_table_value_column_defCo param.addOption(new ConstExpression(ctx.collation().collation_name())); } param.addOption(visit(ctx.literal())); - param.addOption(getJsonOnOption(ctx.opt_value_on_empty_or_error_or_mismatch())); + if (ctx.opt_value_on_empty_or_error_or_mismatch() != null) { + JsonOption jsonOpt = new JsonOption(ctx.opt_value_on_empty_or_error_or_mismatch()); + param.addOption(jsonOpt); + jsonOpt.setOnOption(getJsonOnOption(ctx.opt_value_on_empty_or_error_or_mismatch())); + } return param; } @@ -1004,4 +1114,35 @@ private ConstExpression getAggregator(Complex_func_exprContext ctx) { return null; } + private void setWrapperMode(JsonOption c, Wrapper_optsContext ctx) { + if (ctx == null) { + return; + } + if (ctx.WITH() != null) { + if (ctx.ARRAY() != null) { + if (ctx.CONDITIONAL() != null) { + c.setWrapperMode(JsonOption.WrapperMode.WITH_CONDITIONAL_ARRAY_WRAPPER); + } else if (ctx.UNCONDITIONAL() != null) { + c.setWrapperMode(JsonOption.WrapperMode.WITH_UNCONDITIONAL_ARRAY_WRAPPER); + } else { + c.setWrapperMode(JsonOption.WrapperMode.WITH_ARRAY_WRAPPER); + } + } else { + if (ctx.CONDITIONAL() != null) { + c.setWrapperMode(JsonOption.WrapperMode.WITH_CONDITIONAL_WRAPPER); + } else if (ctx.UNCONDITIONAL() != null) { + c.setWrapperMode(JsonOption.WrapperMode.WITH_UNCONDITIONAL_WRAPPER); + } else { + c.setWrapperMode(JsonOption.WrapperMode.WITH_WRAPPER); + } + } + } else { + if (ctx.ARRAY() != null) { + c.setWrapperMode(JsonOption.WrapperMode.WITHOUT_ARRAY_WRAPPER); + } else { + c.setWrapperMode(JsonOption.WrapperMode.WITHOUT_WRAPPER); + } + } + } + } diff --git a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLFromReferenceFactory.java b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLFromReferenceFactory.java index 75b330b7ca..2701e02480 100644 --- a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLFromReferenceFactory.java +++ b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLFromReferenceFactory.java @@ -15,9 +15,7 @@ */ package com.oceanbase.tools.sqlparser.adapter.mysql; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; +import java.util.*; import java.util.stream.Collectors; import org.antlr.v4.runtime.ParserRuleContext; @@ -108,7 +106,11 @@ public FromReference visitTable_factor(Table_factorContext ctx) { if (ctx.tbl_name() != null) { return visit(ctx.tbl_name()); } else if (ctx.table_subquery() != null) { - return visit(ctx.table_subquery()); + ExpressionReference reference = (ExpressionReference) visit(ctx.table_subquery()); + if (ctx.LATERAL() != null) { + reference.setLateral(true); + } + return reference; } else if (ctx.table_reference() != null) { FromReference from = visit(ctx.table_reference()); if (ctx.LeftBrace() == null || ctx.OJ() == null || ctx.RightBrace() == null) { @@ -123,6 +125,9 @@ public FromReference visitTable_factor(Table_factorContext ctx) { } StatementFactory factory = new MySQLSelectBodyFactory(ctx.select_with_parens()); ExpressionReference reference = new ExpressionReference(ctx, factory.generate(), null); + if (ctx.LATERAL() != null) { + reference.setLateral(true); + } if (ctx.use_flashback() != null) { reference.setFlashbackUsage(visitFlashbackUsage(ctx.use_flashback())); } @@ -193,7 +198,7 @@ public FromReference visitTbl_name(Tbl_nameContext ctx) { } NameReference nameReference = new NameReference(ctx, factor.getSchema(), factor.getRelation(), alias); if (ctx.use_partition() != null) { - nameReference.setPartitionUsage(visitPartitonUsage(ctx.use_partition())); + nameReference.setPartitionUsage(visitPartitionUsage(ctx.use_partition())); } if (ctx.use_flashback() != null) { nameReference.setFlashbackUsage(visitFlashbackUsage(ctx.use_flashback())); @@ -296,10 +301,17 @@ private FlashbackUsage visitFlashbackUsage(Use_flashbackContext ctx) { return new FlashbackUsage(ctx, FlashBackType.AS_OF_SNAPSHOT, factory.generate()); } - public static PartitionUsage visitPartitonUsage(Use_partitionContext usePartition) { - List nameList = new ArrayList<>(); - visitNameList(usePartition.name_list(), nameList); - return new PartitionUsage(usePartition, PartitionType.PARTITION, nameList); + public static PartitionUsage visitPartitionUsage(Use_partitionContext usePartition) { + if (usePartition.name_list() != null) { + List nameList = new ArrayList<>(); + visitNameList(usePartition.name_list(), nameList); + return new PartitionUsage(usePartition, PartitionType.PARTITION, nameList); + } + Map externalTablePartition = new HashMap<>(); + usePartition.external_table_partitions().external_table_partition() + .forEach(ctx -> externalTablePartition.put(ctx.relation_name().getText(), + new MySQLExpressionFactory().visit(ctx.expr_const()))); + return new PartitionUsage(usePartition, PartitionType.PARTITION, externalTablePartition); } private static void visitNameList(Name_listContext ctx, List nameList) { diff --git a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLIndexOptionsFactory.java b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLIndexOptionsFactory.java index bcc5217006..1ba44cf805 100644 --- a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLIndexOptionsFactory.java +++ b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLIndexOptionsFactory.java @@ -15,7 +15,9 @@ */ package com.oceanbase.tools.sqlparser.adapter.mysql; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; import com.oceanbase.tools.sqlparser.adapter.StatementFactory; @@ -61,6 +63,8 @@ public IndexOptions visitOpt_index_options(Opt_index_optionsContext ctx) { indexOptions.setGlobal(false); } else if (option.BLOCK_SIZE() != null) { indexOptions.setBlockSize(getInteger(option)); + } else if (option.KEY_BLOCK_SIZE() != null) { + indexOptions.setKeyBlockSize(getInteger(option)); } else if (option.DATA_TABLE_ID() != null) { indexOptions.setDataTableId(getInteger(option)); } else if (option.INDEX_TABLE_ID() != null) { @@ -76,9 +80,14 @@ public IndexOptions visitOpt_index_options(Opt_index_optionsContext ctx) { } else if (option.CTXCAT() != null) { indexOptions.setCtxcat(getReference(option)); } else if (option.WITH() != null && option.PARSER() != null) { - indexOptions.setWithParser(option.STRING_VALUE().getText()); + indexOptions.setWithParser(option.relation_name().getText()); } else if (option.WITH() != null && option.ROWID() != null) { indexOptions.setWithRowId(true); + } else if (option.WITH() != null && option.vec_index_params() != null) { + Map params = new HashMap<>(); + option.vec_index_params().vec_index_param() + .forEach(i -> params.put(i.relation_name().getText(), i.vec_index_param_value().getText())); + indexOptions.setVectorIndexParams(params); } else if (option.index_using_algorithm() != null) { indexOptions.merge(visit(option.index_using_algorithm())); } else if (option.visibility_option() != null) { diff --git a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLInsertFactory.java b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLInsertFactory.java index 597e97f07d..4c5b491fe4 100644 --- a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLInsertFactory.java +++ b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLInsertFactory.java @@ -67,6 +67,14 @@ public Insert visitInsert_stmt(Insert_stmtContext ctx) { if (ctx.IGNORE() != null) { insert.setIgnore(true); } + if (ctx.HIGH_PRIORITY() != null) { + insert.setHighPriority(true); + } else if (ctx.LOW_PRIORITY() != null) { + insert.setLowPriority(true); + } + if (ctx.OVERWRITE() != null) { + insert.setOverwrite(true); + } if (ctx.update_asgn_list() != null) { insert.setOnDuplicateKeyUpdateColumns(getSetColumns(ctx.update_asgn_list())); } @@ -79,7 +87,7 @@ public Insert visitSingle_table_insert(Single_table_insertContext ctx) { .getRelationFactor(ctx.dml_table_name().relation_factor())); if (ctx.dml_table_name().use_partition() != null) { insertTable.setPartitionUsage(MySQLFromReferenceFactory - .visitPartitonUsage(ctx.dml_table_name().use_partition())); + .visitPartitionUsage(ctx.dml_table_name().use_partition())); } if (ctx.column_list() != null) { insertTable.setColumns(ctx.column_list().column_definition_ref().stream() diff --git a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLSelectBodyFactory.java b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLSelectBodyFactory.java index 9b4f351fe0..865ad6c677 100644 --- a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLSelectBodyFactory.java +++ b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLSelectBodyFactory.java @@ -231,12 +231,13 @@ private RelationType getRelationType(Set_typeContext setType) { public SelectBody visitSimple_select_with_order_and_limit(Simple_select_with_order_and_limitContext ctx) { SelectBody select = new SelectBody(ctx, visit(ctx.simple_select())); if (ctx.order_by() != null) { - StatementFactory factory = new MySQLOrderByFactory(ctx.order_by()); - select.setOrderBy(factory.generate()); + select.setOrderBy(new MySQLOrderByFactory(ctx.order_by()).generate()); + } + if (ctx.opt_approx() != null) { + select.setApproximate(true); } if (ctx.limit_clause() != null) { - StatementFactory factory = new MySQLLimitFactory(ctx.limit_clause()); - select.setLimit(factory.generate()); + select.setLimit(new MySQLLimitFactory(ctx.limit_clause()).generate()); } return select; } diff --git a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLTableElementFactory.java b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLTableElementFactory.java index 17f63470df..3460a75e3f 100644 --- a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLTableElementFactory.java +++ b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLTableElementFactory.java @@ -243,6 +243,12 @@ public TableElement visitColumn_definition(Column_definitionContext ctx) { ColumnAttributes attributes = visitGeneratedColumnAttributeList(ctx.opt_generated_column_attribute_list()); definition.setColumnAttributes(attributes); } + if (ctx.references_clause() != null) { + definition.setForeignReference(visitForeignReference(ctx.references_clause())); + } + if (ctx.SERIAL() != null) { + definition.setSerial(true); + } if (ctx.FIRST() != null) { definition.setLocation(new Location(ctx.FIRST().getText(), null)); } else if (ctx.BEFORE() != null) { @@ -422,8 +428,14 @@ private ColumnAttributes visitColumnAttribute(Column_attributeContext ctx) { attribute = new InLineCheckConstraint(ctx, constraintName, state, new MySQLExpressionFactory(ctx.expr()).generate()); attributes.setConstraints(Collections.singletonList(attribute)); - } else if (ctx.DEFAULT() != null || ctx.ORIG_DEFAULT() != null) { - Expression expr = visitNowOrSignedLiteral(ctx.now_or_signed_literal()); + } else if ((ctx.DEFAULT() != null || ctx.ORIG_DEFAULT() != null) + && (ctx.now_or_signed_literal() != null || ctx.expr() != null)) { + Expression expr = null; + if (ctx.now_or_signed_literal() != null) { + expr = visitNowOrSignedLiteral(ctx.now_or_signed_literal()); + } else if (ctx.expr() != null) { + expr = new MySQLExpressionFactory(ctx.expr()).generate(); + } if (ctx.DEFAULT() != null) { attributes.setDefaultValue(expr); } else { @@ -450,6 +462,16 @@ private ColumnAttributes visitColumnAttribute(Column_attributeContext ctx) { skipIndexTypes.add(ctx.skip_index_type().getText()); } attributes.setSkipIndexTypes(skipIndexTypes); + } else if (ctx.lob_chunk_size() != null) { + if (ctx.lob_chunk_size().STRING_VALUE() != null) { + attributes.setLobChunkSize(ctx.lob_chunk_size().STRING_VALUE().getText()); + } else if (ctx.lob_chunk_size().INTNUM() != null) { + attributes.setLobChunkSize(ctx.lob_chunk_size().INTNUM().getText()); + } + } else if (ctx.COLUMN_FORMAT() != null) { + attributes.setColumnFormat(ctx.col_attri_value.getText()); + } else if (ctx.STORAGE() != null) { + attributes.setStorage(ctx.col_attri_value.getText()); } return attributes; } diff --git a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLTableOptionsFactory.java b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLTableOptionsFactory.java index 325b9da697..863599ddeb 100644 --- a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLTableOptionsFactory.java +++ b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/mysql/MySQLTableOptionsFactory.java @@ -16,6 +16,7 @@ package com.oceanbase.tools.sqlparser.adapter.mysql; import java.math.BigDecimal; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -24,12 +25,14 @@ import org.antlr.v4.runtime.ParserRuleContext; import com.oceanbase.tools.sqlparser.adapter.StatementFactory; +import com.oceanbase.tools.sqlparser.obmysql.OBParser.Lob_storage_clauseContext; import com.oceanbase.tools.sqlparser.obmysql.OBParser.Parallel_optionContext; import com.oceanbase.tools.sqlparser.obmysql.OBParser.Table_optionContext; import com.oceanbase.tools.sqlparser.obmysql.OBParser.Table_option_listContext; import com.oceanbase.tools.sqlparser.obmysql.OBParser.Table_option_list_space_seperatedContext; import com.oceanbase.tools.sqlparser.obmysql.OBParserBaseVisitor; import com.oceanbase.tools.sqlparser.statement.Expression; +import com.oceanbase.tools.sqlparser.statement.common.mysql.LobStorageOption; import com.oceanbase.tools.sqlparser.statement.createtable.TableOptions; import com.oceanbase.tools.sqlparser.statement.expression.BoolValue; import com.oceanbase.tools.sqlparser.statement.expression.CollectionExpression; @@ -172,6 +175,8 @@ public TableOptions visitTable_option(Table_optionContext ctx) { .map(ex -> new MySQLExpressionFactory(ex).generate()) .collect(Collectors.toList()); value = new CollectionExpression(e.expr_list(), exprs); + } else if (e.compression_name() != null) { + value = new ConstExpression(e.compression_name()); } formatMap.put(e.format_key.getText().toUpperCase(), value); }); @@ -187,6 +192,60 @@ public TableOptions visitTable_option(Table_optionContext ctx) { target.setDefaultLobInRowThreshold(Integer.valueOf(ctx.INTNUM().getText())); } else if (ctx.LOB_INROW_THRESHOLD() != null) { target.setLobInRowThreshold(Integer.valueOf(ctx.INTNUM().getText())); + } else if (ctx.KEY_BLOCK_SIZE() != null) { + target.setKeyBlockSize(Integer.valueOf(ctx.INTNUM().getText())); + } else if (ctx.AUTO_INCREMENT_CACHE_SIZE() != null) { + target.setAutoIncrementCacheSize(Integer.valueOf(ctx.INTNUM().getText())); + } else if (ctx.PARTITION_TYPE() != null) { + target.setPartitionType(ctx.USER_SPECIFIED().getText()); + } else if (ctx.PROPERTIES() != null) { + Map externalProperties = new HashMap<>(); + ctx.external_properties_list().external_properties().forEach(e -> { + externalProperties.put(e.external_properties_key().getText(), e.STRING_VALUE().getText()); + }); + target.setExternalProperties(externalProperties); + } else if (ctx.lob_storage_clause() != null) { + target.setLobStorageOption(getLobStorageOption(ctx.lob_storage_clause())); + } else if (ctx.MICRO_INDEX_CLUSTERED() != null) { + target.setMicroIndexClustered(Boolean.valueOf(ctx.BOOL_VALUE().getText())); + } else if (ctx.AUTO_REFRESH() != null) { + if (ctx.OFF() != null) { + target.setAutoRefresh(ctx.OFF().getText()); + } else if (ctx.IMMEDIATE() != null) { + target.setAutoRefresh(ctx.IMMEDIATE().getText()); + } else if (ctx.INTERVAL() != null) { + target.setAutoRefresh(ctx.INTERVAL().getText()); + } + } else if (ctx.MIN_ROWS() != null) { + target.setMinRows(Integer.valueOf(ctx.INTNUM().getText())); + } else if (ctx.MAX_ROWS() != null) { + target.setMaxRows(Integer.valueOf(ctx.INTNUM().getText())); + } else if (ctx.PASSWORD() != null) { + target.setPassword(ctx.STRING_VALUE().getText()); + } else if (ctx.PACK_KEYS() != null) { + target.setPackKeys(ctx.INTNUM() != null ? ctx.INTNUM().getText() : ctx.DEFAULT().getText()); + } else if (ctx.CONNECTION() != null) { + target.setConnection(ctx.STRING_VALUE().getText()); + } else if (ctx.DATA() != null && ctx.DIRECTORY() != null) { + target.setDataDirectory(ctx.STRING_VALUE().getText()); + } else if (ctx.INDEX() != null && ctx.DIRECTORY() != null) { + target.setIndexDirectory(ctx.STRING_VALUE().getText()); + } else if (ctx.ENCRYPTION() != null) { + target.setEncryption(ctx.STRING_VALUE().getText()); + } else if (ctx.STATS_AUTO_RECALC() != null) { + target.setStatsAutoRecalc(ctx.INTNUM() != null ? ctx.INTNUM().getText() : ctx.DEFAULT().getText()); + } else if (ctx.STATS_PERSISTENT() != null) { + target.setStatsPersistent(ctx.INTNUM() != null ? ctx.INTNUM().getText() : ctx.DEFAULT().getText()); + } else if (ctx.STATS_SAMPLE_PAGES() != null) { + target.setStatsSamplePages(ctx.INTNUM() != null ? ctx.INTNUM().getText() : ctx.DEFAULT().getText()); + } else if (ctx.UNION() != null) { + target.setUnion(Collections.emptyList()); + if (ctx.table_list() != null) { + target.setUnion(ctx.table_list().relation_factor().stream() + .map(MySQLFromReferenceFactory::getRelationFactor).collect(Collectors.toList())); + } + } else if (ctx.INSERT_METHOD() != null) { + target.setInsertMethod(ctx.merge_insert_types().getText()); } return target; } @@ -202,4 +261,15 @@ public TableOptions visitParallel_option(Parallel_optionContext ctx) { return tableOptions; } + public static LobStorageOption getLobStorageOption(Lob_storage_clauseContext ctx) { + List lobStorageSizes = ctx.lob_storage_parameters().lob_storage_parameter() + .stream().map(i -> { + if (i.lob_chunk_size().INTNUM() != null) { + return i.lob_chunk_size().INTNUM().getText(); + } + return i.lob_chunk_size().STRING_VALUE().getText(); + }).collect(Collectors.toList()); + return new LobStorageOption(ctx, ctx.column_name().getText(), lobStorageSizes); + } + } diff --git a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/oracle/OracleAlterTableActionFactory.java b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/oracle/OracleAlterTableActionFactory.java index 3736972307..c77af1ee78 100644 --- a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/oracle/OracleAlterTableActionFactory.java +++ b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/oracle/OracleAlterTableActionFactory.java @@ -15,9 +15,7 @@ */ package com.oceanbase.tools.sqlparser.adapter.oracle; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; +import java.util.*; import java.util.stream.Collectors; import org.antlr.v4.runtime.CharStream; @@ -25,10 +23,12 @@ import org.antlr.v4.runtime.misc.Interval; import com.oceanbase.tools.sqlparser.adapter.StatementFactory; +import com.oceanbase.tools.sqlparser.oboracle.OBParser.Add_external_table_partition_actionsContext; import com.oceanbase.tools.sqlparser.oboracle.OBParser.Add_range_or_list_partitionContext; import com.oceanbase.tools.sqlparser.oboracle.OBParser.Add_range_or_list_subpartitionContext; -import com.oceanbase.tools.sqlparser.oboracle.OBParser.Alter_column_group_optionContext; +import com.oceanbase.tools.sqlparser.oboracle.OBParser.Alter_column_group_actionContext; import com.oceanbase.tools.sqlparser.oboracle.OBParser.Alter_column_optionContext; +import com.oceanbase.tools.sqlparser.oboracle.OBParser.Alter_external_table_actionContext; import com.oceanbase.tools.sqlparser.oboracle.OBParser.Alter_index_optionContext; import com.oceanbase.tools.sqlparser.oboracle.OBParser.Alter_partition_optionContext; import com.oceanbase.tools.sqlparser.oboracle.OBParser.Alter_table_actionContext; @@ -43,6 +43,7 @@ import com.oceanbase.tools.sqlparser.oboracle.OBParser.Split_list_partitionContext; import com.oceanbase.tools.sqlparser.oboracle.OBParser.Split_range_partitionContext; import com.oceanbase.tools.sqlparser.oboracle.OBParserBaseVisitor; +import com.oceanbase.tools.sqlparser.statement.Expression; import com.oceanbase.tools.sqlparser.statement.alter.table.AlterTableAction; import com.oceanbase.tools.sqlparser.statement.alter.table.PartitionSplitActions; import com.oceanbase.tools.sqlparser.statement.common.ColumnGroupElement; @@ -56,6 +57,7 @@ import com.oceanbase.tools.sqlparser.statement.createtable.SubPartitionElement; import com.oceanbase.tools.sqlparser.statement.createtable.TableOptions; import com.oceanbase.tools.sqlparser.statement.expression.ColumnReference; +import com.oceanbase.tools.sqlparser.statement.expression.ConstExpression; import lombok.NonNull; @@ -75,10 +77,14 @@ public OracleAlterTableActionFactory(@NonNull Alter_table_actionContext alterTab this.parserRuleContext = alterTableActionContext; } - public OracleAlterTableActionFactory(@NonNull Alter_column_group_optionContext alterColumnGroupOptionContext) { + public OracleAlterTableActionFactory(@NonNull Alter_column_group_actionContext alterColumnGroupOptionContext) { this.parserRuleContext = alterColumnGroupOptionContext; } + public OracleAlterTableActionFactory(@NonNull Alter_external_table_actionContext alterExternalTableActionContext) { + this.parserRuleContext = alterExternalTableActionContext; + } + @Override public AlterTableAction generate() { return visit(this.parserRuleContext); @@ -130,6 +136,20 @@ public AlterTableAction visitOpt_alter_compress_option(Opt_alter_compress_option return alterTableAction; } + @Override + public AlterTableAction visitAlter_external_table_action(Alter_external_table_actionContext ctx) { + AlterTableAction action = new AlterTableAction(ctx); + action.setExternalTableLocation(ctx.STRING_VALUE().getText()); + if (ctx.DROP() != null && ctx.PARTITION() != null) { + action.setDropExternalTablePartition(true); + } else if (ctx.ADD() != null && ctx.PARTITION() != null) { + Map externalTablePartition = new HashMap<>(); + visitAddExternalTablePartitionActions(externalTablePartition, ctx.add_external_table_partition_actions()); + action.setAddExternalTablePartition(externalTablePartition); + } + return action; + } + @Override public AlterTableAction visitAlter_column_option(Alter_column_optionContext ctx) { AlterTableAction alterTableAction = new AlterTableAction(ctx); @@ -248,6 +268,9 @@ public AlterTableAction visitAlter_partition_option(Alter_partition_optionContex .collect(Collectors.toList()); } alterTableAction.addSubpartitionElements(getRelationFactor(ctx.relation_factor()), subElts); + } else if (ctx.EXCHANGE() != null && ctx.PARTITION() != null) { + alterTableAction.setExchangePartition(ctx.relation_name(0).getText(), + OracleFromReferenceFactory.getRelationFactor(ctx.relation_factor())); } else if (ctx.add_range_or_list_partition() != null) { Add_range_or_list_partitionContext pCtx = ctx.add_range_or_list_partition(); List elts; @@ -329,7 +352,7 @@ public AlterTableAction visitModify_partition_info(Modify_partition_infoContext } @Override - public AlterTableAction visitAlter_column_group_option(Alter_column_group_optionContext ctx) { + public AlterTableAction visitAlter_column_group_action(Alter_column_group_actionContext ctx) { AlterTableAction action = new AlterTableAction(ctx); List columnGroupElements = ctx.column_group_list().column_group_element() .stream().map(c -> new OracleColumnGroupElementFactory(c).generate()).collect(Collectors.toList()); @@ -379,4 +402,14 @@ private List getSpecialPartitionElement(Special_partition_list }).collect(Collectors.toList()); } + private void visitAddExternalTablePartitionActions(Map externalTablePartition, + Add_external_table_partition_actionsContext context) { + if (context == null) { + return; + } + Expression value = new ConstExpression(context.add_external_table_partition_action().expr_const()); + externalTablePartition.put(context.add_external_table_partition_action().column_name().getText(), value); + visitAddExternalTablePartitionActions(externalTablePartition, context.add_external_table_partition_actions()); + } + } diff --git a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/oracle/OracleAlterTableFactory.java b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/oracle/OracleAlterTableFactory.java index e3c3bc5851..580ddb5ba6 100644 --- a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/oracle/OracleAlterTableFactory.java +++ b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/oracle/OracleAlterTableFactory.java @@ -58,9 +58,12 @@ public AlterTable visitAlter_table_stmt(Alter_table_stmtContext ctx) { List actions = ctx.alter_table_actions().alter_table_action().stream() .map(c -> new OracleAlterTableActionFactory(c).generate()).collect(Collectors.toList()); alterTable = new AlterTable(ctx, relationFactor, actions); + } else if (ctx.alter_external_table_action() != null) { + alterTable = new AlterTable(ctx, relationFactor, Collections.singletonList( + new OracleAlterTableActionFactory(ctx.alter_external_table_action()).generate())); } else { alterTable = new AlterTable(ctx, relationFactor, Collections.singletonList( - new OracleAlterTableActionFactory(ctx.alter_column_group_option()).generate())); + new OracleAlterTableActionFactory(ctx.alter_column_group_action()).generate())); } if (ctx.EXTERNAL() != null) { alterTable.setExternal(true); diff --git a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/oracle/OracleCreateIndexFactory.java b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/oracle/OracleCreateIndexFactory.java index 8b85428350..64638fe736 100644 --- a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/oracle/OracleCreateIndexFactory.java +++ b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/oracle/OracleCreateIndexFactory.java @@ -57,6 +57,9 @@ public CreateIndex visitCreate_index_stmt(Create_index_stmtContext ctx) { CreateIndex index = new CreateIndex(ctx, OracleFromReferenceFactory.getRelationFactor(ctx.normal_relation_factor()), OracleFromReferenceFactory.getRelationFactor(ctx.relation_factor()), columns); + if (ctx.INDEXTYPE() != null && ctx.MDSYS() != null && ctx.SPATIAL_INDEX() != null) { + index.setMdSysDotSpatialIndex(true); + } if (ctx.opt_index_options() != null) { IndexOptions options = new OracleIndexOptionsFactory(ctx.opt_index_options()).generate(); Index_using_algorithmContext context = ctx.index_using_algorithm(); diff --git a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/oracle/OracleDataTypeFactory.java b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/oracle/OracleDataTypeFactory.java index 137c18e030..2ff47c98cc 100644 --- a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/oracle/OracleDataTypeFactory.java +++ b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/oracle/OracleDataTypeFactory.java @@ -255,6 +255,8 @@ public boolean isBinary() { public DataType visitTreat_data_type(Treat_data_typeContext ctx) { if (ctx.JSON() != null) { return new GeneralDataType(ctx, ctx.JSON().getText(), null); + } else if (ctx.obj_access_ref_cast() != null) { + return new GeneralDataType(ctx, ctx.obj_access_ref_cast().getText(), null); } return visitChildren(ctx); } diff --git a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/oracle/OracleExpressionFactory.java b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/oracle/OracleExpressionFactory.java index 0166b88715..75bc21a2a4 100644 --- a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/oracle/OracleExpressionFactory.java +++ b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/oracle/OracleExpressionFactory.java @@ -51,11 +51,14 @@ import com.oceanbase.tools.sqlparser.oboracle.OBParser.Dot_notation_pathContext; import com.oceanbase.tools.sqlparser.oboracle.OBParser.Dot_notation_path_obj_access_refContext; import com.oceanbase.tools.sqlparser.oboracle.OBParser.Entry_opContext; +import com.oceanbase.tools.sqlparser.oboracle.OBParser.Environment_id_functionContext; import com.oceanbase.tools.sqlparser.oboracle.OBParser.Evalname_exprContext; import com.oceanbase.tools.sqlparser.oboracle.OBParser.ExprContext; +import com.oceanbase.tools.sqlparser.oboracle.OBParser.Extract_functionContext; import com.oceanbase.tools.sqlparser.oboracle.OBParser.Func_access_refContext; import com.oceanbase.tools.sqlparser.oboracle.OBParser.Func_paramContext; import com.oceanbase.tools.sqlparser.oboracle.OBParser.Func_param_with_assignContext; +import com.oceanbase.tools.sqlparser.oboracle.OBParser.Hierarchical_functionContext; import com.oceanbase.tools.sqlparser.oboracle.OBParser.In_exprContext; import com.oceanbase.tools.sqlparser.oboracle.OBParser.Insert_child_xmlContext; import com.oceanbase.tools.sqlparser.oboracle.OBParser.Is_json_constrainContext; @@ -87,6 +90,7 @@ import com.oceanbase.tools.sqlparser.oboracle.OBParser.Json_value_on_optContext; import com.oceanbase.tools.sqlparser.oboracle.OBParser.Json_value_on_responseContext; import com.oceanbase.tools.sqlparser.oboracle.OBParser.Nstring_length_iContext; +import com.oceanbase.tools.sqlparser.oboracle.OBParser.Numeric_functionContext; import com.oceanbase.tools.sqlparser.oboracle.OBParser.Obj_access_refContext; import com.oceanbase.tools.sqlparser.oboracle.OBParser.Obj_access_ref_normalContext; import com.oceanbase.tools.sqlparser.oboracle.OBParser.Opt_js_value_returning_typeContext; @@ -103,9 +107,12 @@ import com.oceanbase.tools.sqlparser.oboracle.OBParser.Regular_entry_objContext; import com.oceanbase.tools.sqlparser.oboracle.OBParser.Relation_nameContext; import com.oceanbase.tools.sqlparser.oboracle.OBParser.Scalars_optContext; +import com.oceanbase.tools.sqlparser.oboracle.OBParser.Sdo_relate_exprContext; import com.oceanbase.tools.sqlparser.oboracle.OBParser.Signed_literalContext; import com.oceanbase.tools.sqlparser.oboracle.OBParser.Simple_exprContext; import com.oceanbase.tools.sqlparser.oboracle.OBParser.Single_row_functionContext; +import com.oceanbase.tools.sqlparser.oboracle.OBParser.Spatial_cellid_exprContext; +import com.oceanbase.tools.sqlparser.oboracle.OBParser.Spatial_mbr_exprContext; import com.oceanbase.tools.sqlparser.oboracle.OBParser.Special_func_exprContext; import com.oceanbase.tools.sqlparser.oboracle.OBParser.String_length_iContext; import com.oceanbase.tools.sqlparser.oboracle.OBParser.Table_element_access_listContext; @@ -150,14 +157,14 @@ import com.oceanbase.tools.sqlparser.statement.expression.FullTextSearch; import com.oceanbase.tools.sqlparser.statement.expression.FunctionCall; import com.oceanbase.tools.sqlparser.statement.expression.FunctionParam; -import com.oceanbase.tools.sqlparser.statement.expression.JsonConstraint; -import com.oceanbase.tools.sqlparser.statement.expression.JsonConstraint.ScalarsMode; -import com.oceanbase.tools.sqlparser.statement.expression.JsonConstraint.StrictMode; -import com.oceanbase.tools.sqlparser.statement.expression.JsonConstraint.UniqueMode; -import com.oceanbase.tools.sqlparser.statement.expression.JsonConstraint.WrapperMode; import com.oceanbase.tools.sqlparser.statement.expression.JsonKeyValue; import com.oceanbase.tools.sqlparser.statement.expression.JsonOnOption; import com.oceanbase.tools.sqlparser.statement.expression.JsonOnOption.OnMismatch; +import com.oceanbase.tools.sqlparser.statement.expression.JsonOption; +import com.oceanbase.tools.sqlparser.statement.expression.JsonOption.ScalarsMode; +import com.oceanbase.tools.sqlparser.statement.expression.JsonOption.StrictMode; +import com.oceanbase.tools.sqlparser.statement.expression.JsonOption.UniqueMode; +import com.oceanbase.tools.sqlparser.statement.expression.JsonOption.WrapperMode; import com.oceanbase.tools.sqlparser.statement.expression.NullExpression; import com.oceanbase.tools.sqlparser.statement.expression.ParamWithAssign; import com.oceanbase.tools.sqlparser.statement.expression.RelationReference; @@ -666,7 +673,7 @@ public Expression visitBool_pri_in_pl_func(Bool_pri_in_pl_funcContext ctx) { @Override public Expression visitIs_json_constrain(Is_json_constrainContext ctx) { - JsonConstraint constraint = new JsonConstraint(ctx); + JsonOption constraint = new JsonOption(ctx); if (ctx.strict_opt() != null) { constraint.setStrictMode(ctx.strict_opt().LAX() != null ? StrictMode.LAX : StrictMode.STRICT); } @@ -699,42 +706,49 @@ public Expression visitJson_object_expr(Json_object_exprContext ctx) { FunctionCall fCall = new FunctionCall(ctx, "json_object", params); if (ctx.opt_json_object_content().opt_json_object_clause() != null) { Opt_json_object_clauseContext oCtx = ctx.opt_json_object_content().opt_json_object_clause(); + JsonOption jsonOpt = null; + if (oCtx.STRICT() != null || oCtx.json_obj_unique_key() != null) { + jsonOpt = getJsonOption(oCtx.STRICT(), oCtx.json_obj_unique_key()); + } if (oCtx.js_on_null() != null) { + if (jsonOpt == null) { + jsonOpt = new JsonOption(oCtx.js_on_null()); + } JsonOnOption onOption = new JsonOnOption(oCtx.js_on_null()); if (oCtx.js_on_null().ABSENT() != null) { onOption.setOnNull(new ConstExpression(oCtx.js_on_null().ABSENT())); } else { onOption.setOnNull(new NullExpression(oCtx.js_on_null().NULLX(0))); } - fCall.addOption(onOption); + jsonOpt.setOnOption(onOption); } if (oCtx.json_obj_returning_type() != null) { fCall.addOption(new OracleDataTypeFactory( oCtx.json_obj_returning_type().js_return_type()).generate()); } - if (oCtx.STRICT() != null || oCtx.json_obj_unique_key() != null) { - fCall.addOption(getJsonConstraint(oCtx.STRICT(), oCtx.json_obj_unique_key())); + if (jsonOpt != null) { + fCall.addOption(jsonOpt); } } else if (ctx.opt_json_object_content().STRICT() != null) { - fCall.addOption(getJsonConstraint(ctx.opt_json_object_content().STRICT(), + fCall.addOption(getJsonOption(ctx.opt_json_object_content().STRICT(), ctx.opt_json_object_content().json_obj_unique_key())); } else { - fCall.addOption(getJsonConstraint(null, ctx.opt_json_object_content().json_obj_unique_key())); + fCall.addOption(getJsonOption(null, ctx.opt_json_object_content().json_obj_unique_key())); } return fCall; } - private JsonConstraint getJsonConstraint(TerminalNode strict, Json_obj_unique_keyContext ctx) { - JsonConstraint jc; + private JsonOption getJsonOption(TerminalNode strict, Json_obj_unique_keyContext ctx) { + JsonOption jc; if (strict != null && ctx != null) { - jc = new JsonConstraint(strict, ctx); + jc = new JsonOption(strict, ctx); jc.setStrictMode(StrictMode.STRICT); jc.setUniqueMode(UniqueMode.WITH_UNIQUE_KEYS); } else if (strict != null) { - jc = new JsonConstraint(strict); + jc = new JsonOption(strict); jc.setStrictMode(StrictMode.STRICT); } else { - jc = new JsonConstraint(ctx); + jc = new JsonOption(ctx); jc.setUniqueMode(UniqueMode.WITH_UNIQUE_KEYS); } return jc; @@ -781,23 +795,28 @@ public Expression visitJson_query_expr(Json_query_exprContext ctx) { if (ctx.js_query_return_type() != null) { fCall.addOption(new OracleDataTypeFactory(ctx.js_query_return_type()).generate()); } - if (ctx.TRUNCATE() != null) { - fCall.addOption(new ConstExpression(ctx.TRUNCATE())); - } - if (ctx.PRETTY() != null) { - fCall.addOption(new ConstExpression(ctx.PRETTY())); - } - if (ctx.ASCII() != null) { - fCall.addOption(new ConstExpression(ctx.ASCII())); - } - if (ctx.scalars_opt() != null || ctx.wrapper_opts() != null) { - JsonConstraint constraint = new JsonConstraint( - ctx.scalars_opt() == null ? ctx.wrapper_opts() : ctx.scalars_opt()); - setScalarsMode(constraint, ctx.scalars_opt()); - setWrapperMode(constraint, ctx.wrapper_opts()); - fCall.addOption(constraint); + if (ctx.json_query_opt() != null) { + JsonOption jsonOpt = new JsonOption(ctx.json_query_opt()); + fCall.addOption(jsonOpt); + if (ctx.json_query_opt().TRUNCATE() != null) { + jsonOpt.setTruncate(true); + } + if (ctx.json_query_opt().PRETTY() != null) { + jsonOpt.setPretty(true); + } + if (ctx.json_query_opt().ASCII() != null) { + jsonOpt.setAscii(true); + } + setScalarsMode(jsonOpt, ctx.json_query_opt().scalars_opt()); + setWrapperMode(jsonOpt, ctx.json_query_opt().wrapper_opts()); + if (ctx.json_query_opt().ASIS() != null) { + jsonOpt.setAsis(true); + } + jsonOpt.setOnOption(getJsonOnOption(ctx.json_query_opt().json_query_on_opt())); + if (ctx.json_query_opt().MULTIVALUE() != null) { + jsonOpt.setMultiValue(true); + } } - fCall.addOption(getJsonOnOption(ctx.json_query_on_opt())); return fCall; } @@ -809,24 +828,27 @@ public Expression visitJson_mergepatch_expr(Json_mergepatch_exprContext ctx) { if (ctx.js_mp_return_clause() != null) { fCall.addOption(new OracleDataTypeFactory(ctx.js_mp_return_clause().js_return_type()).generate()); } - Opt_json_mergepatchContext oCtx = ctx.opt_json_mergepatch(); + JsonOption jsonOpt = new JsonOption(ctx.json_mergepatch_opt()); + fCall.addOption(jsonOpt); + Opt_json_mergepatchContext oCtx = ctx.json_mergepatch_opt().opt_json_mergepatch(); if (oCtx.TRUNCATE() != null) { - fCall.addOption(new ConstExpression(oCtx.TRUNCATE())); + jsonOpt.setTruncate(true); } if (oCtx.PRETTY() != null) { - fCall.addOption(new ConstExpression(oCtx.PRETTY())); + jsonOpt.setPretty(true); } if (oCtx.ASCII() != null) { - fCall.addOption(new ConstExpression(oCtx.ASCII())); + jsonOpt.setAscii(true); } - if (ctx.json_mergepatch_on_error() != null) { - JsonOnOption jsonOnOption = new JsonOnOption(ctx.json_mergepatch_on_error()); - if (ctx.json_mergepatch_on_error().NULLX() != null) { - jsonOnOption.setOnError(new NullExpression(ctx.json_mergepatch_on_error().NULLX())); + if (ctx.json_mergepatch_opt().json_mergepatch_on_error() != null) { + OBParser.Json_mergepatch_on_errorContext jCtx = ctx.json_mergepatch_opt().json_mergepatch_on_error(); + JsonOnOption jsonOnOption = new JsonOnOption(jCtx); + if (jCtx.NULLX() != null) { + jsonOnOption.setOnError(new NullExpression(jCtx.NULLX())); } else { - jsonOnOption.setOnError(new ConstExpression(ctx.json_mergepatch_on_error().ERROR_P(0))); + jsonOnOption.setOnError(new ConstExpression(jCtx.ERROR_P(0))); } - fCall.addOption(jsonOnOption); + jsonOpt.setOnOption(jsonOnOption); } return fCall; } @@ -846,22 +868,28 @@ public Expression visitJson_array_expr(Json_array_exprContext ctx) { return p; }).collect(Collectors.toList()); FunctionCall fCall = new FunctionCall(ctx, "json_array", params); + JsonOption jsonOpt = null; if (jCtx.json_array_on_null() != null) { + jsonOpt = new JsonOption(jCtx.json_array_on_null()); JsonOnOption jsonOnOption = new JsonOnOption(jCtx.json_array_on_null()); if (jCtx.json_array_on_null().ABSENT() != null) { jsonOnOption.setOnNull(new ConstExpression(jCtx.json_array_on_null().ABSENT())); } else { jsonOnOption.setOnNull(new NullExpression(jCtx.json_array_on_null().NULLX(0))); } - fCall.addOption(jsonOnOption); + jsonOpt.setOnOption(jsonOnOption); } if (jCtx.js_array_return_clause() != null) { fCall.addOption(new OracleDataTypeFactory(jCtx.js_array_return_clause().js_return_type()).generate()); } if (jCtx.STRICT() != null) { - JsonConstraint jsonConstraint = new JsonConstraint(jCtx.STRICT()); - jsonConstraint.setStrictMode(StrictMode.STRICT); - fCall.addOption(jsonConstraint); + if (jsonOpt == null) { + jsonOpt = new JsonOption(jCtx.STRICT()); + } + jsonOpt.setStrictMode(StrictMode.STRICT); + } + if (jsonOpt != null) { + fCall.addOption(jsonOpt); } return fCall; } @@ -878,13 +906,17 @@ public Expression visitJson_value_expr(Json_value_exprContext ctx) { if (dataType != null) { fCall.addOption(dataType); } - if (ctx.TRUNCATE() != null) { - fCall.addOption(new ConstExpression(ctx.TRUNCATE())); - } - if (ctx.ASCII() != null) { - fCall.addOption(new ConstExpression(ctx.ASCII())); + if (ctx.json_value_opt() != null) { + JsonOption jsonOpt = new JsonOption(ctx.json_value_opt()); + fCall.addOption(jsonOpt); + if (ctx.json_value_opt().TRUNCATE() != null) { + jsonOpt.setTruncate(true); + } + if (ctx.json_value_opt().ASCII() != null) { + jsonOpt.setAscii(true); + } + jsonOpt.setOnOption(getJsonOnOption(ctx.json_value_opt().json_value_on_opt())); } - fCall.addOption(getJsonOnOption(ctx.json_value_on_opt())); return fCall; } @@ -912,7 +944,7 @@ public Expression visitJson_table_expr(Json_table_exprContext ctx) { params.add(new ExpressionParam(new ConstExpression(ctx.literal()))); } FunctionCall fCall = new FunctionCall(ctx, ctx.JSON_TABLE().getText(), params); - fCall.addOption(getJsonOnOption(ctx.opt_json_table_on_error_on_empty())); + fCall.addOption(getJsonOption(ctx.opt_json_table_on_error_on_empty())); ctx.json_table_columns_def_opt().json_table_columns_def().json_table_column_def() .forEach(c -> fCall.addOption(visitJsonTableColumnDef(c))); return fCall; @@ -927,7 +959,7 @@ public Expression visitJson_equal_expr(Json_equal_exprContext ctx) { } FunctionCall fCall = new FunctionCall(ctx, ctx.getChild(0).getText(), params); if (ctx.json_equal_option() != null) { - fCall.addOption(getJsonOnOption(ctx.json_equal_option())); + fCall.addOption(getJsonOption(ctx.json_equal_option())); } return fCall; } @@ -1230,7 +1262,7 @@ public FunctionCall getFunctionCall(Access_func_exprContext ctx) { } setJsonExistOpt(fCall, ctx.opt_json_exist()); if (ctx.json_equal_option() != null) { - fCall.addOption(getJsonOnOption(ctx.json_equal_option())); + fCall.addOption(getJsonOption(ctx.json_equal_option())); } return fCall; } @@ -1280,88 +1312,115 @@ public Expression visitObj_access_ref_normal(Obj_access_ref_normalContext ctx) { } @Override - public Expression visitSingle_row_function(Single_row_functionContext ctx) { + public Expression visitNumeric_function(Numeric_functionContext ctx) { + return new FunctionCall(ctx, ctx.MOD().getText(), ctx.bit_expr().stream() + .map(e -> new ExpressionParam(visit(e))).collect(Collectors.toList())); + } + + @Override + public Expression visitCharacter_function(Character_functionContext ctx) { String funcName = null; List functionOpts = new ArrayList<>(); List params = new ArrayList<>(); - if (ctx.numeric_function() != null) { - funcName = ctx.numeric_function().MOD().getText(); - params.addAll(ctx.numeric_function().bit_expr().stream() - .map(e -> new ExpressionParam(visit(e))).collect(Collectors.toList())); - } else if (ctx.character_function() != null) { - Character_functionContext characterFunc = ctx.character_function(); - if (characterFunc.TRANSLATE() != null) { - funcName = characterFunc.TRANSLATE().getText(); - } else if (characterFunc.TRIM() != null) { - funcName = characterFunc.TRIM().getText(); - } else if (characterFunc.ASCII() != null) { - funcName = characterFunc.ASCII().getText(); - } - if (characterFunc.parameterized_trim() != null) { - Parameterized_trimContext trim = characterFunc.parameterized_trim(); - FunctionParam param = new ExpressionParam(visit(trim.bit_expr(0))); - if (trim.bit_expr(1) != null) { - param.addOption(visit(trim.bit_expr(1))); - } - params.add(param); - for (int i = 0; i < trim.getChildCount(); i++) { - ParseTree p = trim.getChild(i); - if (p instanceof TerminalNode) { - functionOpts.add(new ConstExpression((TerminalNode) p)); - } else { - break; - } - } - } else { - params.addAll(characterFunc.bit_expr().stream().map(e -> new ExpressionParam(visit(e))) - .collect(Collectors.toList())); - if (params.size() > 0) { - params.get(params.size() - 1).addOption(new ConstExpression(characterFunc.translate_charset())); + if (ctx.TRANSLATE() != null) { + funcName = ctx.TRANSLATE().getText(); + } else if (ctx.TRIM() != null) { + funcName = ctx.TRIM().getText(); + } else if (ctx.ASCII() != null) { + funcName = ctx.ASCII().getText(); + } + if (funcName == null) { + throw new IllegalStateException("Missing function name"); + } + if (ctx.parameterized_trim() != null) { + Parameterized_trimContext trim = ctx.parameterized_trim(); + FunctionParam param = new ExpressionParam(visit(trim.bit_expr(0))); + if (trim.bit_expr(1) != null) { + param.addOption(visit(trim.bit_expr(1))); + } + params.add(param); + for (int i = 0; i < trim.getChildCount(); i++) { + ParseTree p = trim.getChild(i); + if (p instanceof TerminalNode) { + functionOpts.add(new ConstExpression((TerminalNode) p)); + } else { + break; } } - } else if (ctx.extract_function() != null) { - funcName = ctx.extract_function().EXTRACT().getText(); - FunctionParam p = new ExpressionParam(new ConstExpression(ctx.extract_function().date_unit_for_extract())); - p.addOption(visit(ctx.extract_function().bit_expr())); - params.add(p); - } else if (ctx.conversion_function() != null) { - Conversion_functionContext fCtx = ctx.conversion_function(); - if (fCtx.CAST() != null) { - funcName = fCtx.CAST().getText(); - FunctionParam functionParam = new ExpressionParam(visit(fCtx.bit_expr())); - functionParam.addOption(new OracleDataTypeFactory(fCtx.cast_data_type()).generate()); - params.add(functionParam); - } else { - funcName = fCtx.TREAT().getText(); - FunctionParam functionParam = new ExpressionParam(visit(fCtx.bit_expr())); - functionParam.addOption(new OracleDataTypeFactory(fCtx.treat_data_type()).generate()); - params.add(functionParam); - } - } else if (ctx.hierarchical_function() != null) { - funcName = ctx.hierarchical_function().SYS_CONNECT_BY_PATH().getText(); - params.addAll(ctx.hierarchical_function().bit_expr().stream().map(e -> new ExpressionParam(visit(e))) + } else { + params.addAll(ctx.bit_expr().stream().map(e -> new ExpressionParam(visit(e))) .collect(Collectors.toList())); - } else if (ctx.environment_id_function() != null) { - funcName = ctx.environment_id_function().getText(); - } else if (ctx.xml_function() != null) { - Expression fCall = visit(ctx.xml_function()); - if (ctx.obj_access_ref_normal() != null) { - fCall.reference(visit(ctx.obj_access_ref_normal()), ReferenceOperator.DOT); - } else if (ctx.table_element_access_list() != null) { - visitTableElementAccessList(fCall, ctx.table_element_access_list()); + if (params.size() > 0) { + params.get(params.size() - 1).addOption(new ConstExpression(ctx.translate_charset())); } - return fCall; - } else if (ctx.json_function() != null) { - return visit(ctx.json_function()); - } - if (funcName == null) { - throw new IllegalStateException("Missing function name"); } FunctionCall fCall = new FunctionCall(ctx, funcName, params); functionOpts.forEach(fCall::addOption); return fCall; } + @Override + public Expression visitExtract_function(Extract_functionContext ctx) { + FunctionParam p = new ExpressionParam(new ConstExpression(ctx.date_unit_for_extract())); + p.addOption(visit(ctx.bit_expr())); + return new FunctionCall(ctx, ctx.EXTRACT().getText(), Collections.singletonList(p)); + } + + @Override + public Expression visitConversion_function(Conversion_functionContext ctx) { + if (ctx.CAST() != null) { + FunctionParam functionParam = new ExpressionParam(visit(ctx.bit_expr())); + functionParam.addOption(new OracleDataTypeFactory(ctx.cast_data_type()).generate()); + return new FunctionCall(ctx, ctx.CAST().getText(), Collections.singletonList(functionParam)); + } + FunctionParam functionParam = new ExpressionParam(visit(ctx.bit_expr())); + functionParam.addOption(new OracleDataTypeFactory(ctx.treat_data_type()).generate()); + return new FunctionCall(ctx, ctx.TREAT().getText(), Collections.singletonList(functionParam)); + } + + @Override + public Expression visitHierarchical_function(Hierarchical_functionContext ctx) { + return new FunctionCall(ctx, ctx.SYS_CONNECT_BY_PATH().getText(), ctx.bit_expr() + .stream().map(e -> new ExpressionParam(visit(e))).collect(Collectors.toList())); + } + + @Override + public Expression visitEnvironment_id_function(Environment_id_functionContext ctx) { + return new FunctionCall(ctx, ctx.getText(), Collections.emptyList()); + } + + @Override + public Expression visitSpatial_cellid_expr(Spatial_cellid_exprContext ctx) { + return new FunctionCall(ctx, ctx.SPATIAL_CELLID().getText(), + Collections.singletonList(new ExpressionParam(visit(ctx.bit_expr())))); + } + + @Override + public Expression visitSpatial_mbr_expr(Spatial_mbr_exprContext ctx) { + return new FunctionCall(ctx, ctx.SPATIAL_MBR().getText(), + Collections.singletonList(new ExpressionParam(visit(ctx.bit_expr())))); + } + + @Override + public Expression visitSdo_relate_expr(Sdo_relate_exprContext ctx) { + return new FunctionCall(ctx, ctx.SDO_RELATE().getText(), ctx.bit_expr() + .stream().map(e -> new ExpressionParam(visit(e))).collect(Collectors.toList())); + } + + @Override + public Expression visitSingle_row_function(Single_row_functionContext ctx) { + if (ctx.xml_function() == null) { + return visitChildren(ctx); + } + Expression fCall = visit(ctx.xml_function()); + if (ctx.obj_access_ref_normal() != null) { + fCall.reference(visit(ctx.obj_access_ref_normal()), ReferenceOperator.DOT); + } else if (ctx.table_element_access_list() != null) { + visitTableElementAccessList(fCall, ctx.table_element_access_list()); + } + return fCall; + } + @Override public Expression visitAggregate_function(Aggregate_functionContext ctx) { if (ctx.funcName == null) { @@ -1452,6 +1511,9 @@ public Expression visitSpecial_func_expr(Special_func_exprContext ctx) { funcName = ctx.VALUES() == null ? ctx.DEFAULT().getText() : ctx.VALUES().getText(); StatementFactory factory = new OracleColumnRefFactory(ctx.column_definition_ref()); params.add(new ExpressionParam(factory.generate())); + } else if (ctx.LAST_REFRESH_SCN() != null) { + funcName = ctx.LAST_REFRESH_SCN().getText(); + params.add(new ExpressionParam(new ConstExpression(ctx.INTNUM()))); } else { funcName = ctx.getChild(0).getText(); params.add(new ExpressionParam(visit(ctx.bit_expr(0)))); @@ -1676,14 +1738,14 @@ public boolean isBinary() { return new GeneralDataType(ctx, ctx.RAW().getText(), null); } - private void setScalarsMode(JsonConstraint c, Scalars_optContext ctx) { + private void setScalarsMode(JsonOption c, Scalars_optContext ctx) { if (ctx == null) { return; } c.setScalarsMode(ctx.ALLOW() != null ? ScalarsMode.ALLOW_SCALARS : ScalarsMode.DISALLOW_SCALARS); } - private void setWrapperMode(JsonConstraint c, Wrapper_optsContext ctx) { + private void setWrapperMode(JsonOption c, Wrapper_optsContext ctx) { if (ctx == null) { return; } @@ -1812,28 +1874,32 @@ private JsonOnOption getJsonOnOption(Js_agg_on_nullContext ctx) { return jsonOnOption; } - private JsonOnOption getJsonOnOption(Opt_json_table_on_error_on_emptyContext ctx) { + private JsonOption getJsonOption(Opt_json_table_on_error_on_emptyContext ctx) { if (ctx == null) { return null; } + JsonOption jsonOpt = new JsonOption(ctx); JsonOnOption jsonOnOption = new JsonOnOption(ctx); + jsonOpt.setOnOption(jsonOnOption); if (ctx.json_table_on_error() != null) { jsonOnOption.setOnError(visit(ctx.json_table_on_error().json_table_on_response())); } if (ctx.json_table_on_empty() != null) { jsonOnOption.setOnEmpty(visit(ctx.json_table_on_empty().json_table_on_response())); } - return jsonOnOption; + return jsonOpt; } - private JsonOnOption getJsonOnOption(Json_equal_optionContext ctx) { + private JsonOption getJsonOption(Json_equal_optionContext ctx) { + JsonOption jsonOpt = new JsonOption(ctx); JsonOnOption jsonOnOption = new JsonOnOption(ctx); + jsonOpt.setOnOption(jsonOnOption); if (ctx.BOOL_VALUE() != null) { jsonOnOption.setOnError(new BoolValue(ctx.BOOL_VALUE())); } else { jsonOnOption.setOnError(new ConstExpression(ctx.ERROR_P(0))); } - return jsonOnOption; + return jsonOpt; } private FunctionParam visitJsonTableColumnDef(Json_table_column_defContext ctx) { @@ -1860,12 +1926,28 @@ private FunctionParam visitJsonTableExistsColumnDef(Json_table_exists_column_def FunctionParam param = new ExpressionParam(new ColumnReference( ctx.column_name(), null, null, ctx.column_name().getText())); param.addOption(new OracleDataTypeFactory(ctx.opt_jt_value_type()).generate()); + JsonOption jsonOpt = null; if (ctx.TRUNCATE() != null) { - param.addOption(new ConstExpression(ctx.TRUNCATE())); + jsonOpt = new JsonOption(ctx.TRUNCATE()); + jsonOpt.setTruncate(true); } param.addOption(new ConstExpression(ctx.EXISTS())); param.addOption(visit(ctx.json_table_column_def_path())); - param.addOption(getJsonOnOption(ctx.opt_json_exists_on_error_on_empty())); + if (ctx.ASIS() != null) { + if (jsonOpt == null) { + jsonOpt = new JsonOption(ctx.ASIS()); + } + jsonOpt.setAsis(true); + } + if (ctx.opt_json_exists_on_error_on_empty() != null) { + if (jsonOpt == null) { + jsonOpt = new JsonOption(ctx.opt_json_exists_on_error_on_empty()); + } + jsonOpt.setOnOption(getJsonOnOption(ctx.opt_json_exists_on_error_on_empty())); + } + if (jsonOpt != null) { + param.addOption(jsonOpt); + } return param; } @@ -1880,24 +1962,40 @@ private FunctionParam visitJsonTableQueryColumnDef(Json_table_query_column_defCo } else if (ctx.JSON() != null) { param.addOption(new GeneralDataType(ctx.JSON(), ctx.JSON().getText(), null)); } + JsonOption jsonOpt = null; if (ctx.TRUNCATE() != null) { - param.addOption(new ConstExpression(ctx.TRUNCATE())); + jsonOpt = new JsonOption(ctx.TRUNCATE()); + jsonOpt.setTruncate(true); } if (ctx.scalars_opt() != null || ctx.wrapper_opts() != null) { - JsonConstraint jsonConstraint; - if (ctx.scalars_opt() != null && ctx.wrapper_opts() != null) { - jsonConstraint = new JsonConstraint(ctx.scalars_opt(), ctx.wrapper_opts()); - } else if (ctx.wrapper_opts() != null) { - jsonConstraint = new JsonConstraint(ctx.wrapper_opts()); - } else { - jsonConstraint = new JsonConstraint(ctx.scalars_opt()); + if (jsonOpt == null) { + if (ctx.scalars_opt() != null && ctx.wrapper_opts() != null) { + jsonOpt = new JsonOption(ctx.scalars_opt(), ctx.wrapper_opts()); + } else if (ctx.wrapper_opts() != null) { + jsonOpt = new JsonOption(ctx.wrapper_opts()); + } else { + jsonOpt = new JsonOption(ctx.scalars_opt()); + } } - setScalarsMode(jsonConstraint, ctx.scalars_opt()); - setWrapperMode(jsonConstraint, ctx.wrapper_opts()); - param.addOption(jsonConstraint); + setScalarsMode(jsonOpt, ctx.scalars_opt()); + setWrapperMode(jsonOpt, ctx.wrapper_opts()); } param.addOption(visit(ctx.json_table_column_def_path())); - param.addOption(getJsonOnOption(ctx.json_query_on_opt())); + if (ctx.ASIS() != null) { + if (jsonOpt == null) { + jsonOpt = new JsonOption(ctx.ASIS()); + } + jsonOpt.setAsis(true); + } + if (ctx.json_query_on_opt() != null) { + if (jsonOpt == null) { + jsonOpt = new JsonOption(ctx.json_query_on_opt()); + } + jsonOpt.setOnOption(getJsonOnOption(ctx.json_query_on_opt())); + } + if (jsonOpt != null) { + param.addOption(jsonOpt); + } return param; } @@ -1905,11 +2003,27 @@ private FunctionParam visitJsonTableValueColumnDef(Json_table_value_column_defCo FunctionParam param = new ExpressionParam(new ColumnReference( ctx.column_name(), null, null, ctx.column_name().getText())); param.addOption(new OracleDataTypeFactory(ctx.opt_jt_value_type()).generate()); + JsonOption jsonOpt = null; if (ctx.TRUNCATE() != null) { - param.addOption(new ConstExpression(ctx.TRUNCATE())); + jsonOpt = new JsonOption(ctx.TRUNCATE()); + jsonOpt.setTruncate(true); } param.addOption(visit(ctx.json_table_column_def_path())); - param.addOption(getJsonOnOption(ctx.json_value_on_opt())); + if (ctx.ASIS() != null) { + if (jsonOpt == null) { + jsonOpt = new JsonOption(ctx.ASIS()); + } + jsonOpt.setAsis(true); + } + if (ctx.json_value_on_opt() != null) { + if (jsonOpt == null) { + jsonOpt = new JsonOption(ctx.json_value_on_opt()); + } + jsonOpt.setOnOption(getJsonOnOption(ctx.json_value_on_opt())); + } + if (jsonOpt != null) { + param.addOption(jsonOpt); + } return param; } @@ -1930,7 +2044,11 @@ private void setJsonExistOpt(@NonNull FunctionCall functionCall, Opt_json_existC .map(c -> new ExpressionParam(visit(c.bit_expr()), c.sql_var_name().getText())) .forEach(functionCall::addOption); } - functionCall.addOption(getJsonOnOption(ctx.opt_json_exists_on_error_on_empty())); + if (ctx.opt_json_exists_on_error_on_empty() != null) { + JsonOption jsonOpt = new JsonOption(ctx.opt_json_exists_on_error_on_empty()); + functionCall.addOption(jsonOpt); + jsonOpt.setOnOption(getJsonOnOption(ctx.opt_json_exists_on_error_on_empty())); + } } private void setFunctionOptions(FunctionCall functionCall, Aggregate_functionContext ctx) { @@ -1947,7 +2065,16 @@ private void setFunctionOptions(FunctionCall functionCall, Aggregate_functionCon if (ctx.WITHIN() == null && ctx.DENSE_RANK() == null && ctx.order_by() != null) { functionCall.addOption(new OracleOrderByFactory(ctx.order_by()).generate()); } - functionCall.addOption(getJsonOnOption(ctx.js_agg_on_null())); + JsonOption jsonOpt = null; + if (ctx.STRICT() != null || ctx.json_obj_unique_key() != null) { + jsonOpt = getJsonOption(ctx.STRICT(), ctx.json_obj_unique_key()); + } + if (ctx.js_agg_on_null() != null) { + if (jsonOpt == null) { + jsonOpt = new JsonOption(ctx.js_agg_on_null()); + } + jsonOpt.setOnOption(getJsonOnOption(ctx.js_agg_on_null())); + } if (ctx.js_agg_returning_type_opt() != null) { Js_agg_returning_type_optContext jCtx = ctx.js_agg_returning_type_opt(); if (jCtx.js_return_type() != null) { @@ -1956,8 +2083,8 @@ private void setFunctionOptions(FunctionCall functionCall, Aggregate_functionCon functionCall.addOption(new OracleDataTypeFactory(jCtx.js_agg_returning_type()).generate()); } } - if (ctx.STRICT() != null || ctx.json_obj_unique_key() != null) { - functionCall.addOption(getJsonConstraint(ctx.STRICT(), ctx.json_obj_unique_key())); + if (jsonOpt != null) { + functionCall.addOption(jsonOpt); } } diff --git a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/oracle/OracleFromReferenceFactory.java b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/oracle/OracleFromReferenceFactory.java index 64ed4ead8c..244a97507c 100644 --- a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/oracle/OracleFromReferenceFactory.java +++ b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/oracle/OracleFromReferenceFactory.java @@ -165,7 +165,11 @@ public FromReference visitTable_factor(Table_factorContext ctx) { if (ctx.tbl_name() != null) { return visit(ctx.tbl_name()); } else if (ctx.table_subquery() != null) { - return visit(ctx.table_subquery()); + ExpressionReference reference = (ExpressionReference) visit(ctx.table_subquery()); + if (ctx.LATERAL() != null) { + reference.setLateral(true); + } + return reference; } else if (ctx.table_reference() != null) { return visit(ctx.table_reference()); } else if (ctx.simple_expr() != null) { diff --git a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/oracle/OracleInsertFactory.java b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/oracle/OracleInsertFactory.java index 5cf2e67530..ee9eebda35 100644 --- a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/oracle/OracleInsertFactory.java +++ b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/oracle/OracleInsertFactory.java @@ -20,6 +20,8 @@ import java.util.List; import java.util.stream.Collectors; +import org.antlr.v4.runtime.tree.TerminalNode; + import com.oceanbase.tools.sqlparser.adapter.StatementFactory; import com.oceanbase.tools.sqlparser.oboracle.OBParser.Condition_insert_clauseContext; import com.oceanbase.tools.sqlparser.oboracle.OBParser.Conditional_insert_clauseContext; @@ -130,15 +132,16 @@ public Insert visitConditional_insert_clause(Conditional_insert_clauseContext ct public Insert visitSingle_table_insert(Single_table_insertContext ctx) { InsertTable insertTable; Insert_table_clauseContext iCtx = ctx.insert_table_clause(); + TerminalNode beginNode = ctx.INTO() == null ? ctx.OVERWRITE() : ctx.INTO(); if (iCtx.dml_table_name() != null) { Dml_table_nameContext dCtx = iCtx.dml_table_name(); - insertTable = new InsertTable(ctx.INTO(), ctx.values_clause(), OracleFromReferenceFactory + insertTable = new InsertTable(beginNode, ctx.values_clause(), OracleFromReferenceFactory .getRelationFactor(dCtx.relation_factor())); if (dCtx.use_partition() != null) { insertTable.setPartitionUsage(new OraclePartitionUsageFactory(dCtx.use_partition()).generate()); } } else if (iCtx.select_with_parens() != null) { - insertTable = new InsertTable(ctx.INTO(), ctx.values_clause(), + insertTable = new InsertTable(beginNode, ctx.values_clause(), new OracleSelectBodyFactory(iCtx.select_with_parens()).generate()); } else { OracleSelectBodyFactory factory = new OracleSelectBodyFactory(iCtx.subquery()); @@ -153,7 +156,7 @@ public Insert visitSingle_table_insert(Single_table_insertContext ctx) { if (oCtx.with_check_option() != null) { select.getLastSelectBody().setWithCheckOption(true); } - insertTable = new InsertTable(ctx.INTO(), ctx.values_clause(), select); + insertTable = new InsertTable(beginNode, ctx.values_clause(), select); } if (iCtx.relation_name() != null) { insertTable.setAlias(iCtx.relation_name().getText()); @@ -192,6 +195,9 @@ public Insert visitSingle_table_insert(Single_table_insertContext ctx) { insert.setLogErrors(new OracleLogErrorsFactory(rCtx.log_error_clause()).generate()); } } + if (ctx.OVERWRITE() != null) { + insert.setOverwrite(true); + } return insert; } diff --git a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/oracle/OraclePartitionUsageFactory.java b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/oracle/OraclePartitionUsageFactory.java index 196671830a..239a31843c 100644 --- a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/oracle/OraclePartitionUsageFactory.java +++ b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/oracle/OraclePartitionUsageFactory.java @@ -16,12 +16,17 @@ package com.oceanbase.tools.sqlparser.adapter.oracle; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import com.oceanbase.tools.sqlparser.adapter.StatementFactory; +import com.oceanbase.tools.sqlparser.oboracle.OBParser.External_table_partitionsContext; import com.oceanbase.tools.sqlparser.oboracle.OBParser.Name_listContext; import com.oceanbase.tools.sqlparser.oboracle.OBParser.Use_partitionContext; import com.oceanbase.tools.sqlparser.oboracle.OBParserBaseVisitor; +import com.oceanbase.tools.sqlparser.statement.Expression; +import com.oceanbase.tools.sqlparser.statement.expression.ConstExpression; import com.oceanbase.tools.sqlparser.statement.select.PartitionType; import com.oceanbase.tools.sqlparser.statement.select.PartitionUsage; @@ -54,9 +59,24 @@ public PartitionUsage visitUse_partition(Use_partitionContext ctx) { if (ctx.SUBPARTITION() != null) { type = PartitionType.SUB_PARTITION; } - List nameList = new ArrayList<>(); - visitNameList(ctx.name_list(), nameList); - return new PartitionUsage(ctx, type, nameList); + if (ctx.name_list() != null) { + List nameList = new ArrayList<>(); + visitNameList(ctx.name_list(), nameList); + return new PartitionUsage(ctx, type, nameList); + } + Map externalTablePartition = new HashMap<>(); + visitExternalTablePartitions(ctx.external_table_partitions(), externalTablePartition); + return new PartitionUsage(ctx, type, externalTablePartition); + } + + private void visitExternalTablePartitions(External_table_partitionsContext ctx, + Map externalTablePartition) { + if (ctx == null) { + return; + } + externalTablePartition.put(ctx.external_table_partition().relation_name().getText(), + new ConstExpression(ctx.external_table_partition().expr_const())); + visitExternalTablePartitions(ctx.external_table_partitions(), externalTablePartition); } private void visitNameList(Name_listContext ctx, List nameList) { diff --git a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/oracle/OracleTableElementFactory.java b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/oracle/OracleTableElementFactory.java index 86bc36bf41..4b23b1c30f 100644 --- a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/oracle/OracleTableElementFactory.java +++ b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/oracle/OracleTableElementFactory.java @@ -387,6 +387,8 @@ private ColumnAttributes visitColumnAttribute(Column_attributeContext ctx) { attributes.setOrigDefault(visitNowOrSignedLiteral(ctx.now_or_signed_literal())); } else if (ctx.ID() != null) { attributes.setId(Integer.valueOf(ctx.INTNUM().getText())); + } else if (ctx.SRID() != null) { + attributes.setSrid(Integer.valueOf(ctx.INTNUM().getText())); } else if (ctx.SKIP_INDEX() != null) { List skipIndexTypes = new ArrayList<>(); if (ctx.opt_skip_index_type_list() != null) { @@ -459,6 +461,8 @@ private ColumnAttributes visitGeneratedColumnAttribute(Generated_column_attribut attributes.setId(Integer.valueOf(ctx.INTNUM().getText())); } else if (ctx.COMMENT() != null) { attributes.setComment(ctx.STRING_VALUE().getText()); + } else if (ctx.SRID() != null) { + attributes.setSrid(Integer.valueOf(ctx.INTNUM().getText())); } else { String name = null; if (ctx.constraint_and_name() != null) { diff --git a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/oracle/OracleTableOptionsFactory.java b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/oracle/OracleTableOptionsFactory.java index 8ce7cd3b93..875d7ae3e3 100644 --- a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/oracle/OracleTableOptionsFactory.java +++ b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/adapter/oracle/OracleTableOptionsFactory.java @@ -160,12 +160,32 @@ public TableOptions visitTable_option(Table_optionContext ctx) { .map(ex -> new OracleExpressionFactory(ex).generate()) .collect(Collectors.toList()); value = new CollectionExpression(e.expr_list(), exprs); + } else if (e.compression_name() != null) { + value = new ConstExpression(e.compression_name()); } formatMap.put(e.format_key.getText().toUpperCase(), value); }); target.setFormat(formatMap); } else if (ctx.PATTERN() != null) { target.setPattern(ctx.STRING_VALUE().getText()); + } else if (ctx.PROPERTIES() != null) { + Map externalProperties = new HashMap<>(); + ctx.external_properties_list().external_properties().forEach(e -> { + externalProperties.put(e.external_properties_key().getText(), e.STRING_VALUE().getText()); + }); + target.setExternalProperties(externalProperties); + } else if (ctx.PARTITION_TYPE() != null) { + target.setPartitionType(ctx.USER_SPECIFIED().getText()); + } else if (ctx.MICRO_INDEX_CLUSTERED() != null) { + target.setMicroIndexClustered(Boolean.valueOf(ctx.BOOL_VALUE().getText())); + } else if (ctx.AUTO_REFRESH() != null) { + if (ctx.OFF() != null) { + target.setAutoRefresh(ctx.OFF().getText()); + } else if (ctx.IMMEDIATE() != null) { + target.setAutoRefresh(ctx.IMMEDIATE().getText()); + } else if (ctx.INTERVAL() != null) { + target.setAutoRefresh(ctx.INTERVAL().getText()); + } } return target; } diff --git a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/alter/table/AlterTableAction.java b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/alter/table/AlterTableAction.java index 9050193876..f1eb1d0379 100644 --- a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/alter/table/AlterTableAction.java +++ b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/alter/table/AlterTableAction.java @@ -17,6 +17,7 @@ import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; import org.antlr.v4.runtime.ParserRuleContext; @@ -26,6 +27,7 @@ import com.oceanbase.tools.sqlparser.statement.Expression; import com.oceanbase.tools.sqlparser.statement.common.ColumnGroupElement; import com.oceanbase.tools.sqlparser.statement.common.RelationFactor; +import com.oceanbase.tools.sqlparser.statement.common.mysql.LobStorageOption; import com.oceanbase.tools.sqlparser.statement.createtable.ColumnDefinition; import com.oceanbase.tools.sqlparser.statement.createtable.ConstraintState; import com.oceanbase.tools.sqlparser.statement.createtable.OutOfLineConstraint; @@ -100,6 +102,9 @@ public class AlterTableAction extends BaseStatement { private List dropPartitionNames; private List dropSubPartitionNames; private List addPartitionElements; + private Map addExternalTablePartition; + private String externalTableLocation; + private boolean dropExternalTablePartition; @Setter(AccessLevel.NONE) private RelationFactor addSubPartitionElementTo; @Setter(AccessLevel.NONE) @@ -147,11 +152,19 @@ public class AlterTableAction extends BaseStatement { private String renameToSubPartitionName; private List addColumnGroupElements; private List dropColumnGroupElements; + private String exchangePartitionName; + private RelationFactor exchangePartitionTargetTable; + private LobStorageOption lobStorageOption; public AlterTableAction(@NonNull ParserRuleContext context) { super(context); } + public void setExchangePartition(@NonNull String partitionName, @NonNull RelationFactor targetTable) { + this.exchangePartitionName = partitionName; + this.exchangePartitionTargetTable = targetTable; + } + public void setDropColumn(@NonNull ColumnReference dropColumn, String dropColumnOption) { this.dropColumns = Collections.singletonList(dropColumn); @@ -249,6 +262,9 @@ public String toString() { .map(ColumnDefinition::toString) .collect(Collectors.joining(","))) .append(")"); + if (this.lobStorageOption != null) { + builder.append(" ").append(this.lobStorageOption); + } } } if (CollectionUtils.isNotEmpty(this.dropColumns)) { @@ -410,6 +426,10 @@ public String toString() { if (Boolean.TRUE.equals(this.removePartitioning)) { builder.append(" REMOVE PARTITIONING"); } + if (this.exchangePartitionTargetTable != null && this.exchangePartitionName != null) { + builder.append(" EXCHANGE PARTITION ").append(this.exchangePartitionName) + .append(" WITH TABLE ").append(this.exchangePartitionTargetTable).append(" WITHOUT VALIDATION"); + } if (addColumnGroupElements != null) { builder.append(" ADD COLUMN GROUP(") .append(addColumnGroupElements.stream().map(ColumnGroupElement::toString) @@ -422,6 +442,16 @@ public String toString() { .collect(Collectors.joining(","))) .append(")"); } + if (this.dropExternalTablePartition) { + builder.append(" DROP PARTITION"); + } + if (this.addExternalTablePartition != null) { + builder.append(" ADD PARTITION(").append(this.addExternalTablePartition.entrySet().stream() + .map(e -> e.getKey() + "=" + e.getValue()).collect(Collectors.joining(", "))).append(")"); + } + if (this.externalTableLocation != null) { + builder.append(" LOCATION ").append(this.externalTableLocation); + } return builder.length() == 0 ? "" : builder.substring(1); } @@ -453,7 +483,7 @@ public boolean isSetDefault() { @Override public String toString() { - return this.isSetDefault() ? "SET DEFAULT" + this.defaultValue : "DROP DEFAULT"; + return this.isSetDefault() ? "SET DEFAULT (" + this.defaultValue + ")" : "DROP DEFAULT"; } } diff --git a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/common/mysql/ArrayType.java b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/common/mysql/ArrayType.java index d6567dca70..9f907d807c 100644 --- a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/common/mysql/ArrayType.java +++ b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/common/mysql/ArrayType.java @@ -37,7 +37,7 @@ @Setter @EqualsAndHashCode(callSuper = false) public class ArrayType extends BaseStatement implements DataType { - private final String typeName = "ARRAY"; + private final DataType elementType; public ArrayType(@NonNull ParserRuleContext context, @NonNull DataType elementType) { @@ -45,17 +45,23 @@ public ArrayType(@NonNull ParserRuleContext context, @NonNull DataType elementTy this.elementType = elementType; } - public ArrayType(DataType elementType) { + public ArrayType(@NonNull DataType elementType) { this.elementType = elementType; } @Override public String getName() { - return this.typeName; + return "ARRAY"; } @Override public List getArguments() { return Collections.emptyList(); } + + @Override + public String toString() { + return getName() + "(" + this.elementType + ")"; + } + } diff --git a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/common/mysql/LobStorageOption.java b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/common/mysql/LobStorageOption.java new file mode 100644 index 0000000000..e6b86c2a3f --- /dev/null +++ b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/common/mysql/LobStorageOption.java @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.tools.sqlparser.statement.common.mysql; + +import java.util.List; +import java.util.stream.Collectors; + +import org.antlr.v4.runtime.ParserRuleContext; + +import com.oceanbase.tools.sqlparser.statement.BaseStatement; + +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.NonNull; + +/** + * {@link LobStorageOption} + * + * @author yh263208 + * @date 2024-10-25 15:29 + * @since ODC_release_4.3.2 + */ +@Getter +@EqualsAndHashCode(callSuper = false) +public class LobStorageOption extends BaseStatement { + + private final String columnName; + private final List lobChunkSizes; + + public LobStorageOption(@NonNull ParserRuleContext context, + @NonNull String columnName, @NonNull List lobChunkSizes) { + super(context); + this.columnName = columnName; + this.lobChunkSizes = lobChunkSizes; + } + + public LobStorageOption(@NonNull String columnName, @NonNull List lobChunkSizes) { + this.columnName = columnName; + this.lobChunkSizes = lobChunkSizes; + } + + @Override + public String toString() { + String lobChunkSize = this.lobChunkSizes.stream().map(s -> "CHUNK " + s).collect(Collectors.joining(" ")); + return "JSON(" + this.columnName + ")" + " STORE AS (" + lobChunkSize + ")"; + } + +} diff --git a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/common/mysql/VectorType.java b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/common/mysql/VectorType.java index fe16933523..29a895291a 100644 --- a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/common/mysql/VectorType.java +++ b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/common/mysql/VectorType.java @@ -37,6 +37,7 @@ @Setter @EqualsAndHashCode(callSuper = false) public class VectorType extends BaseStatement implements DataType { + private final String typeName; private final Integer dimension; @@ -64,8 +65,7 @@ public List getArguments() { @Override public String toString() { - StringBuilder builder = new StringBuilder(getName()); - builder.append("(").append(dimension).append(")"); - return builder.toString(); + return getName() + "(" + this.dimension + ")"; } + } diff --git a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/createindex/CreateIndex.java b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/createindex/CreateIndex.java index 0dd3035a70..156972b160 100644 --- a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/createindex/CreateIndex.java +++ b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/createindex/CreateIndex.java @@ -53,6 +53,7 @@ public class CreateIndex extends BaseStatement { private RelationFactor relation; private IndexOptions indexOptions; private Partition partition; + private boolean mdSysDotSpatialIndex; private final List columns; private List columnGroupElements; @@ -96,6 +97,9 @@ public String toString() { .append(" (\n\t").append(this.columns.stream() .map(SortColumn::toString).collect(Collectors.joining(",\n\t"))) .append("\n)"); + if (this.mdSysDotSpatialIndex) { + builder.append(" INDEXTYPE IS MDSYS.SPATIAL_INDEX"); + } if (this.indexOptions != null) { builder.append(" ").append(this.indexOptions); } diff --git a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/createtable/ColumnAttributes.java b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/createtable/ColumnAttributes.java index 7067779422..2ad06647f2 100644 --- a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/createtable/ColumnAttributes.java +++ b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/createtable/ColumnAttributes.java @@ -56,6 +56,9 @@ public class ColumnAttributes extends BaseOptions { private Expression onUpdate; private String collation; private Integer srid; + private String lobChunkSize; + private String columnFormat; + private String storage; private List constraints; private List skipIndexTypes; @@ -130,6 +133,15 @@ public String toString() { .append(String.join(",", skipIndexTypes)) .append(")"); } + if (this.lobChunkSize != null) { + builder.append(" CHUNK ").append(this.lobChunkSize); + } + if (this.columnFormat != null) { + builder.append(" COLUMN_FORMAT ").append(this.columnFormat); + } + if (this.storage != null) { + builder.append(" STORAGE ").append(this.storage); + } return builder.length() == 0 ? "" : builder.substring(1); } diff --git a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/createtable/ColumnDefinition.java b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/createtable/ColumnDefinition.java index e70bf9ac86..d9fcfcfcbb 100644 --- a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/createtable/ColumnDefinition.java +++ b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/createtable/ColumnDefinition.java @@ -43,7 +43,12 @@ public class ColumnDefinition extends BaseStatement implements TableElement { private ColumnAttributes columnAttributes; private Boolean visible; private Location location; + /** + * if this field is true, means `BIGINT UNSIGNED NOT NULL AUTO_INCREMENT UNIQUE` + */ + private boolean serial; private GenerateOption generateOption; + private ForeignReference foreignReference; private final DataType dataType; private final ColumnReference columnReference; @@ -64,6 +69,8 @@ public String toString() { StringBuilder builder = new StringBuilder(this.columnReference.toString()); if (this.dataType != null) { builder.append(" ").append(this.dataType.toString()); + } else if (this.serial) { + builder.append(" SERIAL"); } if (this.visible != null) { if (this.visible) { @@ -78,6 +85,9 @@ public String toString() { if (this.columnAttributes != null) { builder.append(" ").append(this.columnAttributes); } + if (this.foreignReference != null) { + builder.append(" ").append(this.foreignReference); + } if (this.location != null) { builder.append(" ").append(this.location); } diff --git a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/createtable/CreateTable.java b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/createtable/CreateTable.java index 601229d1cc..a9a1f029e0 100644 --- a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/createtable/CreateTable.java +++ b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/createtable/CreateTable.java @@ -46,6 +46,8 @@ public class CreateTable extends BaseStatement { private RelationFactor relation; + private boolean ignore; + private boolean replace; private String userVariable; private boolean global; private boolean temporary; @@ -178,6 +180,12 @@ public String toString() { .map(ColumnGroupElement::toString).collect(Collectors.joining(","))) .append(")"); } + if (this.ignore) { + builder.append(" IGNORE"); + } + if (this.replace) { + builder.append(" REPLACE"); + } if (this.as != null) { builder.append(" AS ").append(this.as); } diff --git a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/createtable/IndexOptions.java b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/createtable/IndexOptions.java index a480face47..a81fe01893 100644 --- a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/createtable/IndexOptions.java +++ b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/createtable/IndexOptions.java @@ -16,6 +16,7 @@ package com.oceanbase.tools.sqlparser.statement.createtable; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; import org.antlr.v4.runtime.ParserRuleContext; @@ -66,6 +67,8 @@ public class IndexOptions extends BaseOptions { private Boolean reverse; private List storing; private List ctxcat; + private Integer keyBlockSize; + private Map vectorIndexParams; public IndexOptions(@NonNull ParserRuleContext context) { super(context); @@ -149,6 +152,13 @@ public String toString() { if (this.tableSpace != null) { builder.append(" TABLESPACE ").append(this.tableSpace); } + if (this.keyBlockSize != null) { + builder.append(" KEY_BLOCK_SIZE ").append(this.keyBlockSize); + } + if (this.vectorIndexParams != null) { + builder.append(" WITH (").append(this.vectorIndexParams.entrySet().stream() + .map(e -> e.getKey() + "=" + e.getValue()).collect(Collectors.joining(", "))).append(")"); + } return builder.length() == 0 ? "" : builder.substring(1); } diff --git a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/createtable/OutOfLineIndex.java b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/createtable/OutOfLineIndex.java index d794ee4812..24f6c671a6 100644 --- a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/createtable/OutOfLineIndex.java +++ b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/createtable/OutOfLineIndex.java @@ -69,6 +69,8 @@ public String toString() { builder = new StringBuilder("FULLTEXT KEY"); } else if (this.spatial) { builder = new StringBuilder("SPATIAL KEY"); + } else if (this.vector) { + builder = new StringBuilder("VECTOR KEY"); } else { builder = new StringBuilder("INDEX"); } diff --git a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/createtable/TableOptions.java b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/createtable/TableOptions.java index 5ccebaa8a0..3c6a229e6a 100644 --- a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/createtable/TableOptions.java +++ b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/createtable/TableOptions.java @@ -26,6 +26,8 @@ import com.oceanbase.tools.sqlparser.statement.BaseStatement; import com.oceanbase.tools.sqlparser.statement.Expression; import com.oceanbase.tools.sqlparser.statement.common.BaseOptions; +import com.oceanbase.tools.sqlparser.statement.common.RelationFactor; +import com.oceanbase.tools.sqlparser.statement.common.mysql.LobStorageOption; import com.oceanbase.tools.sqlparser.statement.expression.ColumnReference; import lombok.EqualsAndHashCode; @@ -96,6 +98,26 @@ public class TableOptions extends BaseOptions { private String kvAttributes; private Integer defaultLobInRowThreshold; private Integer lobInRowThreshold; + private Integer keyBlockSize; + private Integer autoIncrementCacheSize; + private String partitionType; + private Map externalProperties; + private LobStorageOption lobStorageOption; + private Boolean microIndexClustered; + private String autoRefresh; + private Integer maxRows; + private Integer minRows; + private String password; + private String packKeys; + private String connection; + private String dataDirectory; + private String indexDirectory; + private String encryption; + private String statsAutoRecalc; + private String statsPersistent; + private String statsSamplePages; + private List union; + private String insertMethod; public TableOptions(@NonNull ParserRuleContext context) { super(context); @@ -255,6 +277,71 @@ public String toString() { if (this.lobInRowThreshold != null) { builder.append(" LOB_INROW_THRESHOLD=").append(this.lobInRowThreshold); } + if (this.keyBlockSize != null) { + builder.append(" KEY_BLOCK_SIZE=").append(this.keyBlockSize); + } + if (this.autoIncrementCacheSize != null) { + builder.append(" AUTO_INCREMENT_CACHE_SIZE=").append(this.autoIncrementCacheSize); + } + if (this.partitionType != null) { + builder.append(" PARTITION_TYPE=").append(this.partitionType); + } + if (this.externalProperties != null) { + builder.append(" PROPERTIES=(") + .append(this.externalProperties.entrySet().stream() + .map(e -> e.getKey() + "=" + e.getValue()).collect(Collectors.joining(","))) + .append(")"); + } + if (this.lobStorageOption != null) { + builder.append(" ").append(this.lobStorageOption); + } + if (this.microIndexClustered != null) { + builder.append(" MICRO_INDEX_CLUSTERED=").append( + Boolean.TRUE.equals(this.microIndexClustered) ? "TRUE" : "FALSE"); + } + if (this.autoRefresh != null) { + builder.append(" AUTO_REFRESH=").append(this.autoRefresh); + } + if (this.maxRows != null) { + builder.append(" MAX_ROWS=").append(this.maxRows); + } + if (this.minRows != null) { + builder.append(" MIN_ROWS=").append(this.minRows); + } + if (this.password != null) { + builder.append(" PASSWORD=").append(this.password); + } + if (this.packKeys != null) { + builder.append(" PACK_KEYS=").append(this.packKeys); + } + if (this.connection != null) { + builder.append(" CONNECTION=").append(this.connection); + } + if (this.dataDirectory != null) { + builder.append(" DATA DIRECTORY=").append(this.dataDirectory); + } + if (this.indexDirectory != null) { + builder.append(" INDEX DIRECTORY=").append(this.indexDirectory); + } + if (this.encryption != null) { + builder.append(" ENCRYPTION=").append(this.encryption); + } + if (this.statsAutoRecalc != null) { + builder.append(" STATS_AUTO_RECALC=").append(this.statsAutoRecalc); + } + if (this.statsPersistent != null) { + builder.append(" STATS_PERSISTENT=").append(this.statsPersistent); + } + if (this.statsSamplePages != null) { + builder.append(" STATS_SAMPLE_PAGES=").append(this.statsSamplePages); + } + if (this.union != null) { + builder.append(" UNION=(").append(this.union.stream() + .map(RelationFactor::toString).collect(Collectors.joining(", "))).append(")"); + } + if (this.insertMethod != null) { + builder.append(" INSERT_METHOD=").append(this.insertMethod); + } return builder.length() == 0 ? "" : builder.substring(1); } diff --git a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/expression/ArrayExpression.java b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/expression/ArrayExpression.java index 1fd49fce70..bbc7804d4c 100644 --- a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/expression/ArrayExpression.java +++ b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/expression/ArrayExpression.java @@ -15,7 +15,6 @@ */ package com.oceanbase.tools.sqlparser.statement.expression; -import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; @@ -35,7 +34,8 @@ @Getter @EqualsAndHashCode(callSuper = true) public class ArrayExpression extends BaseExpression { - private List expressions = new ArrayList<>(); + + private final List expressions; public ArrayExpression(@NonNull ParserRuleContext context, @NonNull List expressions) { super(context); @@ -50,4 +50,5 @@ public ArrayExpression(@NonNull List expressions) { protected String doToString() { return "[" + this.expressions.stream().map(Object::toString).collect(Collectors.joining(",")) + "]"; } + } diff --git a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/expression/FullTextSearch.java b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/expression/FullTextSearch.java index 668c7b5212..aa2b56ec59 100644 --- a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/expression/FullTextSearch.java +++ b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/expression/FullTextSearch.java @@ -38,6 +38,8 @@ public class FullTextSearch extends FunctionCall { private final String against; @Setter private TextSearchMode searchMode; + @Setter + private boolean withQueryExpansion; public FullTextSearch(@NonNull ParserRuleContext context, @NonNull List params, @NonNull String against) { @@ -59,6 +61,9 @@ public String doToString() { if (this.searchMode != null) { builder.append(" ").append(this.searchMode.getValue()); } + if (this.withQueryExpansion) { + builder.append(" WITH QUERY EXPANSION"); + } return builder.append(")").toString(); } diff --git a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/expression/JsonConstraint.java b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/expression/JsonOption.java similarity index 72% rename from libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/expression/JsonConstraint.java rename to libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/expression/JsonOption.java index 4e045fe059..48997ec171 100644 --- a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/expression/JsonConstraint.java +++ b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/expression/JsonOption.java @@ -26,7 +26,7 @@ import lombok.Setter; /** - * {@link JsonConstraint} + * {@link JsonOption} * * @author yh263208 * @date 2023-09-26 15:09 @@ -36,44 +36,68 @@ @Setter @NoArgsConstructor @EqualsAndHashCode(callSuper = true) -public class JsonConstraint extends BaseExpression { +public class JsonOption extends BaseExpression { + private boolean truncate; + private boolean pretty; + private boolean ascii; + private boolean asis; + private boolean multiValue; + private JsonOnOption onOption; private StrictMode strictMode; private ScalarsMode scalarsMode; private UniqueMode uniqueMode; private WrapperMode wrapperMode; - public JsonConstraint(@NonNull ParserRuleContext context) { + public JsonOption(@NonNull ParserRuleContext context) { super(context); } - public JsonConstraint(@NonNull TerminalNode terminalNode) { + public JsonOption(@NonNull TerminalNode terminalNode) { super(terminalNode); } - public JsonConstraint(@NonNull TerminalNode beginNode, @NonNull ParserRuleContext endRule) { + public JsonOption(@NonNull TerminalNode beginNode, @NonNull ParserRuleContext endRule) { super(beginNode, endRule); } - public JsonConstraint(@NonNull ParserRuleContext beginRule, @NonNull ParserRuleContext endRule) { + public JsonOption(@NonNull ParserRuleContext beginRule, @NonNull ParserRuleContext endRule) { super(beginRule, endRule); } @Override protected String doToString() { - StringBuilder builder = new StringBuilder("JSON"); + StringBuilder builder = new StringBuilder(); if (this.strictMode != null) { builder.append(" ").append(this.strictMode.name()); } + if (this.truncate) { + builder.append(" TRUNCATE"); + } if (this.scalarsMode != null) { builder.append(" ").append(this.scalarsMode.name().replace("_", " ")); } + if (this.pretty) { + builder.append(" PRETTY"); + } + if (this.ascii) { + builder.append(" ASCII"); + } if (this.uniqueMode != null) { builder.append(" ").append(this.uniqueMode.name().replace("_", " ")); } if (this.wrapperMode != null) { builder.append(" ").append(this.wrapperMode.name().replace("_", " ")); } + if (this.asis) { + builder.append(" ASIS"); + } + if (this.onOption != null) { + builder.append(" ").append(this.onOption); + } + if (this.multiValue) { + builder.append(" MULTIVALUE"); + } return builder.toString(); } diff --git a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/insert/Insert.java b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/insert/Insert.java index 005fd3c735..dc1c87ab20 100644 --- a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/insert/Insert.java +++ b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/insert/Insert.java @@ -54,6 +54,9 @@ public class Insert extends BaseStatement { private boolean first; private boolean replace; private boolean ignore; + private boolean highPriority; + private boolean lowPriority; + private boolean overwrite; private List onDuplicateKeyUpdateColumns; private final List tableInsert; private final ConditionalInsert conditionalInsert; @@ -78,6 +81,9 @@ public Insert(@NonNull ParserRuleContext context, @NonNull Insert target) { this.ignore = target.ignore; this.onDuplicateKeyUpdateColumns = target.onDuplicateKeyUpdateColumns; this.tableInsert = target.tableInsert; + this.overwrite = target.overwrite; + this.highPriority = target.highPriority; + this.lowPriority = target.lowPriority; this.conditionalInsert = target.conditionalInsert; } @@ -95,6 +101,11 @@ public String toString() { } else { builder.append("INSERT"); } + if (this.highPriority) { + builder.append(" HIGH_PRIORITY"); + } else if (this.lowPriority) { + builder.append(" LOW_PRIORITY"); + } if (this.all) { builder.append(" ALL"); } else if (this.first) { @@ -102,6 +113,9 @@ public String toString() { } else if (this.ignore) { builder.append(" IGNORE"); } + if (this.overwrite) { + builder.append(" OVERWRITE"); + } if (CollectionUtils.isNotEmpty(this.tableInsert)) { builder.append(" ").append(this.tableInsert.stream() .map(InsertTable::toString).collect(Collectors.joining("\n"))); diff --git a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/select/ExpressionReference.java b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/select/ExpressionReference.java index e586f48d26..c15143fefc 100644 --- a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/select/ExpressionReference.java +++ b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/select/ExpressionReference.java @@ -51,6 +51,8 @@ public class ExpressionReference extends BaseStatement implements FromReference @Setter private FlashbackUsage flashbackUsage; @Setter + private boolean lateral; + @Setter private List aliasColumns; public ExpressionReference(@NonNull ParserRuleContext context, @@ -67,7 +69,11 @@ public ExpressionReference(@NonNull Expression target, String alias) { @Override public String toString() { - StringBuilder builder = new StringBuilder(this.target.toString()); + StringBuilder builder = new StringBuilder(); + if (this.lateral) { + builder.append("LATERAL "); + } + builder.append(this.target.toString()); if (this.flashbackUsage != null) { builder.append(" ").append(this.flashbackUsage.toString()); } diff --git a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/select/PartitionUsage.java b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/select/PartitionUsage.java index c9d0c02bc8..fbd2d6711c 100644 --- a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/select/PartitionUsage.java +++ b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/select/PartitionUsage.java @@ -16,10 +16,13 @@ package com.oceanbase.tools.sqlparser.statement.select; import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; import org.antlr.v4.runtime.ParserRuleContext; import com.oceanbase.tools.sqlparser.statement.BaseStatement; +import com.oceanbase.tools.sqlparser.statement.Expression; import lombok.EqualsAndHashCode; import lombok.Getter; @@ -39,16 +42,33 @@ public class PartitionUsage extends BaseStatement { private final PartitionType type; private final List nameList; + private final Map externalTablePartition; public PartitionUsage(@NonNull ParserRuleContext context, PartitionType type, @NonNull List nameList) { super(context); this.type = type; this.nameList = nameList; + this.externalTablePartition = null; } public PartitionUsage(PartitionType type, @NonNull List nameList) { this.type = type; this.nameList = nameList; + this.externalTablePartition = null; + } + + public PartitionUsage(@NonNull ParserRuleContext context, PartitionType type, + @NonNull Map externalTablePartition) { + super(context); + this.type = type; + this.nameList = null; + this.externalTablePartition = externalTablePartition; + } + + public PartitionUsage(PartitionType type, @NonNull Map externalTablePartition) { + this.type = type; + this.nameList = null; + this.externalTablePartition = externalTablePartition; } @Override @@ -59,7 +79,11 @@ public String toString() { } else { buffer.append("SUBPARTITION "); } - return buffer.append("(").append(String.join(",", nameList)).append(")").toString(); + if (this.nameList != null) { + return buffer.append("(").append(String.join(",", nameList)).append(")").toString(); + } + return buffer.append("(").append(this.externalTablePartition.entrySet().stream() + .map(e -> e.getKey() + "=" + e.getValue()).collect(Collectors.joining(", "))).append(")").toString(); } } diff --git a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/select/SelectBody.java b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/select/SelectBody.java index 54886df64c..5423e22d38 100644 --- a/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/select/SelectBody.java +++ b/libs/ob-sql-parser/src/main/java/com/oceanbase/tools/sqlparser/statement/select/SelectBody.java @@ -54,14 +54,16 @@ public class SelectBody extends BaseExpression { private List with = new ArrayList<>(); private boolean recursive; private List groupBy = new ArrayList<>(); - boolean withRollUp; - boolean withCheckOption; + private boolean withRollUp; + private boolean withCheckOption; + private boolean approximate; private List windows = new ArrayList<>(); private Fetch fetch; private Limit limit; private ForUpdate forUpdate; private OrderBy orderBy; private boolean lockInShareMode; + private RelatedSelectBody relatedSelect; private final List> values; private final List froms; @@ -92,6 +94,7 @@ public SelectBody(@NonNull ParserRuleContext context, @NonNull SelectBody other) this.with = other.with; this.recursive = other.recursive; this.groupBy = other.groupBy; + this.lockInShareMode = other.lockInShareMode; this.withRollUp = other.withRollUp; this.withCheckOption = other.withCheckOption; this.windows = other.windows; @@ -102,6 +105,7 @@ public SelectBody(@NonNull ParserRuleContext context, @NonNull SelectBody other) this.froms = other.froms; this.selectItems = other.selectItems; this.forUpdate = other.forUpdate; + this.approximate = other.approximate; this.values = other.values; } @@ -182,6 +186,9 @@ public String doToString() { if (this.orderBy != null) { builder.append(" ").append(this.orderBy.toString()); } + if (this.approximate) { + builder.append(" APPROXIMATE"); + } if (this.fetch != null) { builder.append(" ").append(this.fetch.toString()); } diff --git a/libs/ob-sql-parser/src/main/resources/obmysql/sql/OBParser.g4 b/libs/ob-sql-parser/src/main/resources/obmysql/sql/OBParser.g4 index 16a1c8e18b..529f4bb9e0 100644 --- a/libs/ob-sql-parser/src/main/resources/obmysql/sql/OBParser.g4 +++ b/libs/ob-sql-parser/src/main/resources/obmysql/sql/OBParser.g4 @@ -323,11 +323,6 @@ simple_expr | LeftBracket expr_list RightBracket ; -search_expr - : expr_const - | func_expr - ; - expr : (NOT|USER_VARIABLE SET_VAR) expr | LeftParen expr RightParen @@ -1025,11 +1020,6 @@ reference_action | SET DEFAULT ; -opt_match_option - : MATCH match_action - | empty - ; - match_action : SIMPLE | FULL @@ -1037,8 +1027,8 @@ match_action ; column_definition - : column_definition_ref data_type opt_column_attribute_list? (REFERENCES relation_factor LeftParen column_name_list RightParen opt_match_option opt_reference_option_list)? (FIRST | (BEFORE column_name) | (AFTER column_name))? - | column_definition_ref data_type (GENERATED opt_generated_option_list)? AS LeftParen expr RightParen (VIRTUAL | STORED)? opt_generated_column_attribute_list? (REFERENCES relation_factor LeftParen column_name_list RightParen opt_match_option opt_reference_option_list)? (FIRST | (BEFORE column_name) | (AFTER column_name))? + : column_definition_ref data_type opt_column_attribute_list? references_clause? (FIRST | (BEFORE column_name) | (AFTER column_name))? + | column_definition_ref data_type (GENERATED opt_generated_option_list)? AS LeftParen expr RightParen (VIRTUAL | STORED)? opt_generated_column_attribute_list? references_clause? (FIRST | (BEFORE column_name) | (AFTER column_name))? | column_definition_ref SERIAL opt_column_attribute_list? (FIRST | (BEFORE column_name) | (AFTER column_name))? ; @@ -1283,8 +1273,8 @@ column_attribute | COLLATE collation_name | SKIP_INDEX LeftParen (skip_index_type | (opt_skip_index_type_list Comma skip_index_type))? RightParen | lob_chunk_size - | COLUMN_FORMAT (DEFAULT|FIXED|DYNAMIC) - | STORAGE (DEFAULT|DISK|MEMORY) + | COLUMN_FORMAT col_attri_value=(DEFAULT|FIXED|DYNAMIC) + | STORAGE col_attri_value=(DEFAULT|DISK|MEMORY) ; now_or_signed_literal @@ -1359,7 +1349,6 @@ table_option | AUTO_INCREMENT_CACHE_SIZE COMP_EQ? INTNUM | PARTITION_TYPE COMP_EQ? USER_SPECIFIED | PROPERTIES COMP_EQ? LeftParen external_properties_list RightParen - | lob_storage_clause | MICRO_INDEX_CLUSTERED COMP_EQ? BOOL_VALUE | AUTO_REFRESH COMP_EQ? (OFF|IMMEDIATE|INTERVAL) @@ -1672,7 +1661,11 @@ external_properties_list ; external_properties - : ((((ACCESSID|ACCESSKEY)|(ACCESSTYPE|TYPE))|((ENDPOINT|STSTOKEN)|(PROJECT_NAME|SCHEMA_NAME)))|((COMPRESSION_CODE|QUOTA_NAME)|TABLE_NAME)) COMP_EQ STRING_VALUE + : external_properties_key COMP_EQ STRING_VALUE + ; + +external_properties_key + : ((((ACCESSID|ACCESSKEY)|(ACCESSTYPE|TYPE))|((ENDPOINT|STSTOKEN)|(PROJECT_NAME|SCHEMA_NAME)))|((COMPRESSION_CODE|QUOTA_NAME)|TABLE_NAME)) ; external_file_format_list @@ -1868,7 +1861,7 @@ index_option | COMMENT STRING_VALUE | (STORING|CTXCAT) LeftParen column_name_list RightParen | WITH ROWID - | WITH PARSER STRING_VALUE + | WITH PARSER relation_name | WITH LeftParen vec_index_params RightParen | index_using_algorithm | visibility_option @@ -2027,7 +2020,6 @@ select_stmt : with_clause? (select_no_parens into_clause? |select_with_parens) ; - select_with_parens : LeftParen with_clause? (select_no_parens |select_with_parens) RightParen ; @@ -3488,14 +3480,17 @@ rename_table_action alter_table_stmt : ALTER EXTERNAL? TABLE relation_factor alter_table_actions? - | ALTER TABLE relation_factor alter_column_group_option - | ALTER EXTERNAL TABLE relation_factor ADD PARTITION LeftParen add_external_table_partition_actions RightParen LOCATION STRING_VALUE - | ALTER EXTERNAL TABLE relation_factor DROP PARTITION LOCATION STRING_VALUE + | ALTER TABLE relation_factor alter_column_group_action + | ALTER EXTERNAL TABLE relation_factor alter_external_table_action + ; + +alter_external_table_action + : ADD PARTITION LeftParen add_external_table_partition_actions? RightParen LOCATION STRING_VALUE + | DROP PARTITION LOCATION STRING_VALUE ; add_external_table_partition_actions : add_external_table_partition_action - | empty | add_external_table_partition_actions Comma add_external_table_partition_action ; @@ -3585,13 +3580,13 @@ visibility_option | INVISIBLE ; -alter_column_group_option +alter_column_group_action : (ADD|DROP) COLUMN GROUP LeftParen column_group_list RightParen ; alter_column_option : ADD COLUMN? column_definition - | ADD COLUMN? LeftParen column_definition_list RightParen + | ADD COLUMN? LeftParen column_definition_list RightParen lob_storage_clause? | DROP column_definition_ref (CASCADE | RESTRICT)? | DROP COLUMN column_definition_ref (CASCADE | RESTRICT)? | ALTER COLUMN? column_definition_ref alter_column_behavior @@ -4468,7 +4463,39 @@ vec_index_param_value ; json_query_expr - : JSON_QUERY LeftParen simple_expr Comma complex_string_literal (RETURNING cast_data_type)? TRUNCATE? ((ALLOW SCALARS) | (DISALLOW SCALARS))? PRETTY? ASCII? ((WITHOUT WRAPPER) | (WITHOUT ARRAY WRAPPER) | (WITH WRAPPER) | (WITH ARRAY WRAPPER) | (WITH UNCONDITIONAL WRAPPER) | (WITH CONDITIONAL WRAPPER) | (WITH UNCONDITIONAL ARRAY WRAPPER) | (WITH CONDITIONAL ARRAY WRAPPER))? ASIS? (on_empty_query | on_error_query | on_mismatch_query | (on_error_query on_empty_query) | (on_empty_query on_error_query) | (on_error_query on_mismatch_query) | (on_empty_query on_mismatch_query) | (on_error_query on_empty_query on_mismatch_query) | (on_empty_query on_error_query on_mismatch_query))? MULTIVALUE? RightParen + : JSON_QUERY LeftParen simple_expr Comma complex_string_literal (RETURNING cast_data_type)? json_query_opt RightParen + ; + +json_query_opt + : TRUNCATE? scalars_opt? PRETTY? ASCII? wrapper_opts? ASIS? json_query_on_opt? MULTIVALUE? + ; + +scalars_opt + : ALLOW SCALARS + | DISALLOW SCALARS + ; + +wrapper_opts + : WITHOUT WRAPPER + | WITHOUT ARRAY WRAPPER + | WITH WRAPPER + | WITH ARRAY WRAPPER + | WITH UNCONDITIONAL WRAPPER + | WITH CONDITIONAL WRAPPER + | WITH UNCONDITIONAL ARRAY WRAPPER + | WITH CONDITIONAL ARRAY WRAPPER + ; + +json_query_on_opt + : on_empty_query + | on_error_query + | on_mismatch_query + | on_error_query on_empty_query + | on_empty_query on_error_query + | on_error_query on_mismatch_query + | on_empty_query on_mismatch_query + | on_error_query on_empty_query on_mismatch_query + | on_empty_query on_error_query on_mismatch_query ; opt_response_query @@ -4489,7 +4516,17 @@ on_empty_query ; json_value_expr - : JSON_VALUE LeftParen simple_expr Comma complex_string_literal (RETURNING cast_data_type)? TRUNCATE? ASCII? (on_empty | on_error | (on_empty on_error))? RightParen + : JSON_VALUE LeftParen simple_expr Comma complex_string_literal (RETURNING cast_data_type)? json_value_opt RightParen + ; + +json_value_opt + : TRUNCATE? ASCII? json_value_on_opt? + ; + +json_value_on_opt + : on_empty + | on_error + | on_empty on_error ; opt_on_empty_or_error diff --git a/libs/ob-sql-parser/src/main/resources/oboracle/sql/OBParser.g4 b/libs/ob-sql-parser/src/main/resources/oboracle/sql/OBParser.g4 index 35e0e9df9a..fc6045b429 100644 --- a/libs/ob-sql-parser/src/main/resources/oboracle/sql/OBParser.g4 +++ b/libs/ob-sql-parser/src/main/resources/oboracle/sql/OBParser.g4 @@ -1590,9 +1590,7 @@ table_option | PATTERN COMP_EQ? STRING_VALUE | PARTITION_TYPE COMP_EQ? USER_SPECIFIED | MICRO_INDEX_CLUSTERED COMP_EQ? BOOL_VALUE - | AUTO_REFRESH COMP_EQ? OFF - | AUTO_REFRESH COMP_EQ? IMMEDIATE - | AUTO_REFRESH COMP_EQ? INTERVAL + | AUTO_REFRESH COMP_EQ? (OFF|IMMEDIATE|INTERVAL) ; parallel_option @@ -1923,7 +1921,11 @@ external_properties_list ; external_properties - : ((((ACCESSID|ACCESSKEY)|(ACCESSTYPE|TYPE))|((ENDPOINT|STSTOKEN)|(PROJECT_NAME|SCHEMA_NAME)))|((COMPRESSION_CODE|QUOTA_NAME)|TABLE_NAME)) COMP_EQ STRING_VALUE + : external_properties_key COMP_EQ STRING_VALUE + ; + +external_properties_key + : ((((ACCESSID|ACCESSKEY)|(ACCESSTYPE|TYPE))|((ENDPOINT|STSTOKEN)|(PROJECT_NAME|SCHEMA_NAME)))|((COMPRESSION_CODE|QUOTA_NAME)|TABLE_NAME)) ; external_file_format_list @@ -2892,12 +2894,11 @@ table_subquery use_partition : (PARTITION|SUBPARTITION) LeftParen name_list RightParen - | PARTITION LeftParen external_table_partitions RightParen + | PARTITION LeftParen external_table_partitions? RightParen ; external_table_partitions : external_table_partition - | empty | external_table_partitions Comma external_table_partition ; @@ -3704,14 +3705,17 @@ alter_index_option_oracle alter_table_stmt : ALTER EXTERNAL? TABLE relation_factor alter_table_actions - | ALTER TABLE relation_factor alter_column_group_option - | ALTER EXTERNAL TABLE relation_factor ADD PARTITION LeftParen add_external_table_partition_actions RightParen LOCATION STRING_VALUE - | ALTER EXTERNAL TABLE relation_factor DROP PARTITION LOCATION STRING_VALUE + | ALTER TABLE relation_factor alter_column_group_action + | ALTER EXTERNAL TABLE relation_factor alter_external_table_action + ; + +alter_external_table_action + : ADD PARTITION LeftParen add_external_table_partition_actions? RightParen LOCATION STRING_VALUE + | DROP PARTITION LOCATION STRING_VALUE ; add_external_table_partition_actions : add_external_table_partition_action - | empty | add_external_table_partition_actions Comma add_external_table_partition_action ; @@ -3815,7 +3819,7 @@ visibility_option | INVISIBLE ; -alter_column_group_option +alter_column_group_action : (ADD|DROP) COLUMN GROUP LeftParen column_group_list RightParen ; @@ -4579,7 +4583,11 @@ date_unit_for_extract ; json_mergepatch_expr - : JSON_MERGEPATCH LeftParen bit_expr Comma bit_expr js_mp_return_clause? opt_json_mergepatch json_mergepatch_on_error? RightParen + : JSON_MERGEPATCH LeftParen bit_expr Comma bit_expr js_mp_return_clause? json_mergepatch_opt RightParen + ; + +json_mergepatch_opt + : opt_json_mergepatch json_mergepatch_on_error? ; json_mergepatch_on_error @@ -4623,7 +4631,11 @@ js_array_return_clause ; json_value_expr - : JSON_VALUE LeftParen js_doc_expr Comma js_literal opt_js_value_returning_type TRUNCATE? ASCII? json_value_on_opt? RightParen + : JSON_VALUE LeftParen js_doc_expr Comma js_literal opt_js_value_returning_type json_value_opt RightParen + ; + +json_value_opt + : TRUNCATE? ASCII? json_value_on_opt? ; json_equal_expr @@ -4735,7 +4747,11 @@ json_exists_response_type ; json_query_expr - : JSON_QUERY LeftParen js_doc_expr Comma js_literal (RETURNING js_query_return_type)? TRUNCATE? scalars_opt? PRETTY? ASCII? wrapper_opts? ASIS? json_query_on_opt? MULTIVALUE? RightParen + : JSON_QUERY LeftParen js_doc_expr Comma js_literal (RETURNING js_query_return_type)? json_query_opt RightParen + ; + +json_query_opt + : TRUNCATE? scalars_opt? PRETTY? ASCII? wrapper_opts? ASIS? json_query_on_opt? MULTIVALUE? ; json_query_on_opt diff --git a/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/MySQLAlterTableActionFactoryTest.java b/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/MySQLAlterTableActionFactoryTest.java index 08c8c26752..0684519be0 100644 --- a/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/MySQLAlterTableActionFactoryTest.java +++ b/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/MySQLAlterTableActionFactoryTest.java @@ -35,6 +35,7 @@ import com.oceanbase.tools.sqlparser.statement.common.CharacterType; import com.oceanbase.tools.sqlparser.statement.common.GeneralDataType; import com.oceanbase.tools.sqlparser.statement.common.RelationFactor; +import com.oceanbase.tools.sqlparser.statement.common.mysql.LobStorageOption; import com.oceanbase.tools.sqlparser.statement.createtable.ColumnDefinition; import com.oceanbase.tools.sqlparser.statement.createtable.ConstraintState; import com.oceanbase.tools.sqlparser.statement.createtable.HashPartition; @@ -147,6 +148,23 @@ public void generate_addColumns_succeed() { Assert.assertEquals(expect, actual); } + @Test + public void generate_addColumnsWithLobStorage_succeed() { + StatementFactory factory = new MySQLAlterTableActionFactory( + getActionContext("add (id varchar(64), id1 blob) json(col) store as (chunk 'aas' chunk 123)")); + AlterTableAction actual = factory.generate(); + + AlterTableAction expect = new AlterTableAction(); + CharacterType t1 = new CharacterType("varchar", new BigDecimal("64")); + ColumnDefinition d1 = new ColumnDefinition(new ColumnReference(null, null, "id"), t1); + GeneralDataType t2 = new GeneralDataType("blob", null); + ColumnDefinition d2 = new ColumnDefinition(new ColumnReference(null, null, "id1"), t2); + expect.setAddColumns(Arrays.asList(d1, d2)); + LobStorageOption storageOption = new LobStorageOption("col", Arrays.asList("'aas'", "123")); + expect.setLobStorageOption(storageOption); + Assert.assertEquals(expect, actual); + } + @Test public void generate_dropColumnCascade_succeed() { StatementFactory factory = new MySQLAlterTableActionFactory( @@ -208,6 +226,19 @@ public void generate_alterColumn_succeed() { Assert.assertEquals(expect, actual); } + @Test + public void generate_alterColumn1_succeed() { + StatementFactory factory = new MySQLAlterTableActionFactory( + getActionContext("alter column a.b set default (12)")); + AlterTableAction actual = factory.generate(); + + AlterTableAction expect = new AlterTableAction(); + AlterColumnBehavior behavior = new AlterColumnBehavior(); + behavior.setDefaultValue(new ConstExpression("12")); + expect.alterColumnBehavior(new ColumnReference(null, "a", "b"), behavior); + Assert.assertEquals(expect, actual); + } + @Test public void generate_alterColumnDropDefault_succeed() { StatementFactory factory = new MySQLAlterTableActionFactory( @@ -642,6 +673,17 @@ public void generate_dropForeignConstraints_succeed() { Assert.assertEquals(expect, actual); } + @Test + public void generate_exchangePartition_succeed() { + StatementFactory factory = new MySQLAlterTableActionFactory( + getActionContext("exchange partition p1 with table tbl without validation")); + AlterTableAction actual = factory.generate(); + + AlterTableAction expect = new AlterTableAction(); + expect.setExchangePartition("p1", new RelationFactor("tbl")); + Assert.assertEquals(expect, actual); + } + @Test public void generate_dropPrimaryKey_succeed() { StatementFactory factory = new MySQLAlterTableActionFactory( diff --git a/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/MySQLAlterTableFactoryTest.java b/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/MySQLAlterTableFactoryTest.java index 7cce672361..cff06c4a86 100644 --- a/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/MySQLAlterTableFactoryTest.java +++ b/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/MySQLAlterTableFactoryTest.java @@ -18,6 +18,8 @@ import java.math.BigDecimal; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; +import java.util.Map; import org.antlr.v4.runtime.BailErrorStrategy; import org.antlr.v4.runtime.CharStreams; @@ -29,6 +31,7 @@ import com.oceanbase.tools.sqlparser.obmysql.OBLexer; import com.oceanbase.tools.sqlparser.obmysql.OBParser; import com.oceanbase.tools.sqlparser.obmysql.OBParser.Alter_table_stmtContext; +import com.oceanbase.tools.sqlparser.statement.Expression; import com.oceanbase.tools.sqlparser.statement.alter.table.AlterTable; import com.oceanbase.tools.sqlparser.statement.alter.table.AlterTableAction; import com.oceanbase.tools.sqlparser.statement.common.CharacterType; @@ -37,6 +40,7 @@ import com.oceanbase.tools.sqlparser.statement.createtable.ColumnDefinition; import com.oceanbase.tools.sqlparser.statement.createtable.TableOptions; import com.oceanbase.tools.sqlparser.statement.expression.ColumnReference; +import com.oceanbase.tools.sqlparser.statement.expression.ConstExpression; public class MySQLAlterTableFactoryTest { @@ -71,6 +75,52 @@ public void generate_alterTable1_succeed() { Assert.assertEquals(expect, actual); } + @Test + public void generate_alterExternalTableAddEmptyPartition_succeed() { + StatementFactory factory = new MySQLAlterTableFactory( + getAlterContext("alter external table a.b add partition () location 'abcd'")); + AlterTable actual = factory.generate(); + + AlterTableAction action = new AlterTableAction(); + action.setExternalTableLocation("'abcd'"); + action.setAddExternalTablePartition(new HashMap<>()); + AlterTable expect = new AlterTable(getRelationFactor("a", "b"), Collections.singletonList(action)); + expect.setExternal(true); + Assert.assertEquals(expect, actual); + } + + @Test + public void generate_alterExternalTableAddPartition_succeed() { + StatementFactory factory = new MySQLAlterTableFactory( + getAlterContext( + "alter external table a.b add partition (col='aaa', col1=@@global.ss) location 'abcd'")); + AlterTable actual = factory.generate(); + + AlterTableAction action = new AlterTableAction(); + action.setExternalTableLocation("'abcd'"); + Map partitions = new HashMap<>(); + partitions.put("col", new ConstExpression("'aaa'")); + partitions.put("col1", new ConstExpression("@@global.ss")); + action.setAddExternalTablePartition(partitions); + AlterTable expect = new AlterTable(getRelationFactor("a", "b"), Collections.singletonList(action)); + expect.setExternal(true); + Assert.assertEquals(expect, actual); + } + + @Test + public void generate_alterExternalTableDropPartition_succeed() { + StatementFactory factory = new MySQLAlterTableFactory( + getAlterContext("alter external table a.b drop partition location 'abcd'")); + AlterTable actual = factory.generate(); + + AlterTableAction action = new AlterTableAction(); + action.setExternalTableLocation("'abcd'"); + action.setDropExternalTablePartition(true); + AlterTable expect = new AlterTable(getRelationFactor("a", "b"), Collections.singletonList(action)); + expect.setExternal(true); + Assert.assertEquals(expect, actual); + } + @Test public void generate_alterExternalTable1_succeed() { StatementFactory factory = new MySQLAlterTableFactory( @@ -121,4 +171,5 @@ private RelationFactor getRelationFactor(String schema, String relation) { relationFactor.setSchema(schema); return relationFactor; } + } diff --git a/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/MySQLCreateIndexFactoryTest.java b/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/MySQLCreateIndexFactoryTest.java index 7fddcceb7e..7317b280cd 100644 --- a/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/MySQLCreateIndexFactoryTest.java +++ b/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/MySQLCreateIndexFactoryTest.java @@ -15,9 +15,7 @@ */ package com.oceanbase.tools.sqlparser.adapter; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; +import java.util.*; import org.antlr.v4.runtime.BailErrorStrategy; import org.antlr.v4.runtime.CharStreams; @@ -194,13 +192,20 @@ public void generate_withColumnGroup_customGroup_succeed() { @Test public void generate_CreateVectorIndex_Succeed() { StatementFactory factory = new MySQLCreateIndexFactory( - getCreateIdxContext("create vector index vec_idx1 on t_vec(c2) with (distance=l2, type=hnsw);")); + getCreateIdxContext("create vector index vec_idx1 on t_vec(c2) " + + "key_block_size=1234 with (distance=l2, type=hnsw);")); CreateIndex actual = factory.generate(); CreateIndex expected = new CreateIndex(new RelationFactor("vec_idx1"), - new RelationFactor("t_vec"), Arrays.asList( + new RelationFactor("t_vec"), Collections.singletonList( new SortColumn(new ColumnReference(null, null, "c2")))); - expected.setIndexOptions(new IndexOptions()); + IndexOptions indexOptions = new IndexOptions(); + indexOptions.setKeyBlockSize(1234); + Map params = new HashMap<>(); + params.put("distance", "l2"); + params.put("type", "hnsw"); + indexOptions.setVectorIndexParams(params); + expected.setIndexOptions(indexOptions); expected.setVector(true); Assert.assertEquals(expected, actual); diff --git a/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/MySQLCreateTableFactoryTest.java b/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/MySQLCreateTableFactoryTest.java index 7ff458254f..0d203c3201 100644 --- a/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/MySQLCreateTableFactoryTest.java +++ b/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/MySQLCreateTableFactoryTest.java @@ -39,6 +39,7 @@ import com.oceanbase.tools.sqlparser.statement.common.ColumnGroupElement; import com.oceanbase.tools.sqlparser.statement.common.DataType; import com.oceanbase.tools.sqlparser.statement.common.RelationFactor; +import com.oceanbase.tools.sqlparser.statement.common.mysql.LobStorageOption; import com.oceanbase.tools.sqlparser.statement.common.mysql.VectorType; import com.oceanbase.tools.sqlparser.statement.createtable.ColumnAttributes; import com.oceanbase.tools.sqlparser.statement.createtable.ColumnDefinition; @@ -143,6 +144,23 @@ public void generate_createTableAsSelect_generateSucceed() { Assert.assertEquals(expect, actual); } + @Test + public void generate_createTableAsSelectIgnore_generateSucceed() { + Create_table_stmtContext context = + getCreateTableContext("create table any_schema.abcd ignore as select * from tab"); + StatementFactory factory = new MySQLCreateTableFactory(context); + CreateTable actual = factory.generate(); + + CreateTable expect = new CreateTable(context, getRelationFactor("any_schema", "abcd")); + expect.setIgnore(true); + NameReference from = new NameReference(null, "tab", null); + SelectBody selectBody = + new SelectBody(Collections.singletonList(new Projection()), Collections.singletonList(from)); + Select select = new Select(selectBody); + expect.setAs(select); + Assert.assertEquals(expect, actual); + } + @Test public void generate_sortKey_succeed() { Create_table_stmtContext context = @@ -365,18 +383,60 @@ public void generate_physicalAttrs_succeed() { @Test public void generate_formatTableOp_succeed() { - Create_table_stmtContext context = getCreateTableContext( - "create table any_schema.abcd (id varchar(64)) kv_attributes='12' format=(ENCODING='aaaa',LINE_DELIMITER=123," - + "SKIP_HEADER=12," - + "EMPTY_FIELD_AS_NULL=true,NULL_IF_EXETERNAL=(1,2,3))"); + Create_table_stmtContext context = getCreateTableContext("create table any_schema.abcd (id varchar(64)) " + + "kv_attributes='12' " + + "key_block_size=456 " + + "AUTO_INCREMENT_CACHE_SIZE=667 " + + "partition_type=USER_SPECIFIED " + + "properties=(ACCESSID='abcd', PROJECT_NAME='ooop') " + + "json(col) store as (chunk 'aas' chunk 123) " + + "micro_index_clustered=true " + + "auto_refresh=OFF " + + "max_rows=123445 " + + "min_rows=9879 " + + "password='asdasd' " + + "pack_keys=default " + + "connection='asasdasd' " + + "data directory='ooopi' " + + "index directory='indxasdasd' " + + "encryption='asdasdasdas' " + + "stats_auto_recalc=default " + + "stats_persistent=default " + + "stats_sample_pages=3345 " + + "union=(col1, col2) " + + "insert_method=FIRST " + + "format=(ENCODING='aaaa',LINE_DELIMITER=123,SKIP_HEADER=12,EMPTY_FIELD_AS_NULL=true," + + "NULL_IF_EXETERNAL=(1,2,3),COMPRESSION=col3)"); StatementFactory factory = new MySQLCreateTableFactory(context); CreateTable actual = factory.generate(); - CreateTable expect = new CreateTable(context, getRelationFactor("any_schema", "abcd")); DataType dataType = new CharacterType("varchar", new BigDecimal("64")); expect.setTableElements( Collections.singletonList(new ColumnDefinition(new ColumnReference(null, null, "id"), dataType))); TableOptions tableOptions = new TableOptions(); + tableOptions.setAutoIncrementCacheSize(667); + tableOptions.setDataDirectory("'ooopi'"); + tableOptions.setIndexDirectory("'indxasdasd'"); + tableOptions.setStatsAutoRecalc("default"); + tableOptions.setStatsPersistent("default"); + tableOptions.setStatsSamplePages("3345"); + tableOptions.setUnion(Arrays.asList(new RelationFactor("col1"), new RelationFactor("col2"))); + tableOptions.setInsertMethod("FIRST"); + tableOptions.setEncryption("'asdasdasdas'"); + tableOptions.setPartitionType("USER_SPECIFIED"); + Map externalProperties = new HashMap<>(); + externalProperties.put("ACCESSID", "'abcd'"); + externalProperties.put("PROJECT_NAME", "'ooop'"); + tableOptions.setExternalProperties(externalProperties); + LobStorageOption storageOption = new LobStorageOption("col", Arrays.asList("'aas'", "123")); + tableOptions.setMicroIndexClustered(true); + tableOptions.setLobStorageOption(storageOption); + tableOptions.setAutoRefresh("OFF"); + tableOptions.setMaxRows(123445); + tableOptions.setMinRows(9879); + tableOptions.setPassword("'asdasd'"); + tableOptions.setPackKeys("default"); + tableOptions.setConnection("'asasdasd'"); Map map = new HashMap<>(); map.put("ENCODING", new ConstExpression("'aaaa'")); map.put("EMPTY_FIELD_AS_NULL", new BoolValue(true)); @@ -386,8 +446,13 @@ public void generate_formatTableOp_succeed() { es.addExpression(new ConstExpression("2")); es.addExpression(new ConstExpression("3")); map.put("NULL_IF_EXETERNAL", es); + map.put("COMPRESSION", new ConstExpression("col3")); map.put("LINE_DELIMITER", new ConstExpression("123")); + tableOptions.setKeyBlockSize(456); + tableOptions.setFormat(map); + tableOptions.setKeyBlockSize(456); + tableOptions.setAutoIncrementCacheSize(667); tableOptions.setKvAttributes("'12'"); expect.setTableOptions(tableOptions); Assert.assertEquals(expect, actual); @@ -507,7 +572,12 @@ public void generate_WithVectorIndex_Succeed() { SortColumn indexColumn = new SortColumn(new ColumnReference(null, null, "c1")); OutOfLineIndex index = new OutOfLineIndex("idx1", Collections.singletonList(indexColumn)); index.setVector(true); - index.setIndexOptions(new IndexOptions()); + IndexOptions indexOptions = new IndexOptions(); + index.setIndexOptions(indexOptions); + Map params = new HashMap<>(); + params.put("distance", "L2"); + params.put("type", "hnsw"); + indexOptions.setVectorIndexParams(params); expect.setTableElements( Arrays.asList(new ColumnDefinition(new ColumnReference(null, null, "c1"), dataType), index)); Assert.assertEquals(expect, actual); @@ -534,4 +604,5 @@ private RelationFactor getRelationFactor(String schema, String relation) { relationFactor.setSchema(schema); return relationFactor; } + } diff --git a/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/MySQLExpressionFactoryTest.java b/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/MySQLExpressionFactoryTest.java index 3f875de38c..16c299ccf2 100644 --- a/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/MySQLExpressionFactoryTest.java +++ b/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/MySQLExpressionFactoryTest.java @@ -43,23 +43,7 @@ import com.oceanbase.tools.sqlparser.statement.common.WindowOffsetType; import com.oceanbase.tools.sqlparser.statement.common.WindowSpec; import com.oceanbase.tools.sqlparser.statement.common.WindowType; -import com.oceanbase.tools.sqlparser.statement.expression.ArrayExpression; -import com.oceanbase.tools.sqlparser.statement.expression.BoolValue; -import com.oceanbase.tools.sqlparser.statement.expression.CaseWhen; -import com.oceanbase.tools.sqlparser.statement.expression.CollectionExpression; -import com.oceanbase.tools.sqlparser.statement.expression.ColumnReference; -import com.oceanbase.tools.sqlparser.statement.expression.CompoundExpression; -import com.oceanbase.tools.sqlparser.statement.expression.ConstExpression; -import com.oceanbase.tools.sqlparser.statement.expression.ExpressionParam; -import com.oceanbase.tools.sqlparser.statement.expression.FullTextSearch; -import com.oceanbase.tools.sqlparser.statement.expression.FunctionCall; -import com.oceanbase.tools.sqlparser.statement.expression.FunctionParam; -import com.oceanbase.tools.sqlparser.statement.expression.GroupConcat; -import com.oceanbase.tools.sqlparser.statement.expression.IntervalExpression; -import com.oceanbase.tools.sqlparser.statement.expression.JsonOnOption; -import com.oceanbase.tools.sqlparser.statement.expression.NullExpression; -import com.oceanbase.tools.sqlparser.statement.expression.TextSearchMode; -import com.oceanbase.tools.sqlparser.statement.expression.WhenClause; +import com.oceanbase.tools.sqlparser.statement.expression.*; import com.oceanbase.tools.sqlparser.statement.select.OrderBy; import com.oceanbase.tools.sqlparser.statement.select.SortDirection; import com.oceanbase.tools.sqlparser.statement.select.SortKey; @@ -222,6 +206,23 @@ public void generate_matchNaturalMode_generateSucceed() { Assert.assertEquals(expect, actual); } + @Test + public void generate_matchNaturalModeWithQueryExpansion_generateSucceed() { + ExprContext context = getExprContext( + "match(col,tab.col,chz.tab.col) against ('abc' IN NATURAL LANGUAGE MODE WITH QUERY EXPANSION)"); + StatementFactory factory = new MySQLExpressionFactory(context); + Expression actual = factory.generate(); + + List params = new ArrayList<>(); + params.add(new ExpressionParam(new ColumnReference(null, null, "col"))); + params.add(new ExpressionParam(new ColumnReference(null, "tab", "col"))); + params.add(new ExpressionParam(new ColumnReference("chz", "tab", "col"))); + FullTextSearch expect = new FullTextSearch(params, "'abc'"); + expect.setWithQueryExpansion(true); + expect.setSearchMode(TextSearchMode.NATURAL_LANGUAGE_MODE); + Assert.assertEquals(expect, actual); + } + @Test public void generate_matchBooleanMode_generateSucceed() { ExprContext context = getExprContext("match(col,tab.col,chz.tab.col) against ('abc' IN BOOLEAN MODE)"); @@ -338,6 +339,28 @@ public void generate_st_asmvt_generateFunctionCallSucceed() { Assert.assertEquals(expect, actual); } + @Test + public void generate_lastRefreshScn_generateFunctionCallSucceed() { + ExprContext context = getExprContext("last_refresh_scn(123)"); + StatementFactory factory = new MySQLExpressionFactory(context); + Expression actual = factory.generate(); + + FunctionCall expect = new FunctionCall("last_refresh_scn", + Collections.singletonList(new ExpressionParam(new ConstExpression("123")))); + Assert.assertEquals(expect, actual); + } + + @Test + public void generate_sumOpnsize_generateFunctionCallSucceed() { + ExprContext context = getExprContext("sum_Opnsize(123)"); + StatementFactory factory = new MySQLExpressionFactory(context); + Expression actual = factory.generate(); + + FunctionCall expect = new FunctionCall("sum_Opnsize", + Collections.singletonList(new ExpressionParam(new ConstExpression("123")))); + Assert.assertEquals(expect, actual); + } + @Test public void generate_stddevPopExpr_generateFunctionCallSucceed() { ExprContext context = getExprContext("STDDEV_POP(all 1)"); @@ -453,7 +476,7 @@ public void generate_groupConcat_generateFunctionCallSucceed() { @Test public void generate_castAsChar_generateFunctionCallSucceed() { - ExprContext context = getExprContext("cast('abc' as character(15) binary)"); + ExprContext context = getExprContext("cast('abc' as character(15) binary array)"); StatementFactory factory = new MySQLExpressionFactory(context); Expression actual = factory.generate(); @@ -462,6 +485,7 @@ public void generate_castAsChar_generateFunctionCallSucceed() { CharacterType type = new CharacterType("character", new BigDecimal(15)); type.setBinary(true); p.addOption(type); + p.addOption(new ConstExpression("array")); params.add(p); FunctionCall expect = new FunctionCall("cast", params); Assert.assertEquals(expect, actual); @@ -756,6 +780,258 @@ public void generate_weightString3Int_generateFunctionCallSucceed() { Assert.assertEquals(expect, actual); } + @Test + public void generate_jsonQueryExpr_generateFunctionCallSucceed() { + ExprContext context = getExprContext("JSON_QUERY('123', _utf8 'abc')"); + StatementFactory factory = new MySQLExpressionFactory(context); + Expression actual = factory.generate(); + + List params = new ArrayList<>(); + params.add(new ExpressionParam(new ConstExpression("'123'"))); + params.add(new ExpressionParam(new ConstExpression("_utf8 'abc'"))); + FunctionCall expect = new FunctionCall("JSON_QUERY", params); + expect.addOption(new JsonOption()); + Assert.assertEquals(expect, actual); + } + + @Test + public void generate_jsonQueryExprFullOpts_generateFunctionCallSucceed() { + ExprContext context = getExprContext("JSON_QUERY('123', _utf8 'abc' " + + "returning double " + + "TRUNCATE " + + "allow scalars " + + "pretty " + + "ASCII " + + "with unconditional array wrapper " + + "asis " + + "empty on empty empty array on error_p error_p on mismatch " + + "MULTIVALUE)"); + StatementFactory factory = new MySQLExpressionFactory(context); + Expression actual = factory.generate(); + + List params = new ArrayList<>(); + params.add(new ExpressionParam(new ConstExpression("'123'"))); + params.add(new ExpressionParam(new ConstExpression("_utf8 'abc'"))); + FunctionCall expect = new FunctionCall("JSON_QUERY", params); + expect.addOption(new NumberType("double", null, null)); + JsonOption jsonOpt = new JsonOption(); + jsonOpt.setTruncate(true); + jsonOpt.setScalarsMode(JsonOption.ScalarsMode.ALLOW_SCALARS); + jsonOpt.setPretty(true); + jsonOpt.setAscii(true); + jsonOpt.setMultiValue(true); + jsonOpt.setWrapperMode(JsonOption.WrapperMode.WITH_UNCONDITIONAL_ARRAY_WRAPPER); + jsonOpt.setAsis(true); + JsonOnOption jsonOnOption = new JsonOnOption(); + jsonOnOption.setOnEmpty(new ConstExpression("empty")); + jsonOnOption.setOnError(new ConstExpression("empty array")); + jsonOnOption.setOnMismatches(Collections.singletonList( + new JsonOnOption.OnMismatch(new ConstExpression("error_p"), null))); + jsonOpt.setOnOption(jsonOnOption); + expect.addOption(jsonOpt); + Assert.assertEquals(expect, actual); + } + + @Test + public void generate_jsonQueryExprFullOpts2_generateFunctionCallSucceed() { + ExprContext context = getExprContext("JSON_QUERY('123', _utf8 'abc' " + + "returning double " + + "empty array on empty empty array on error_p error_p on mismatch)"); + StatementFactory factory = new MySQLExpressionFactory(context); + Expression actual = factory.generate(); + + List params = new ArrayList<>(); + params.add(new ExpressionParam(new ConstExpression("'123'"))); + params.add(new ExpressionParam(new ConstExpression("_utf8 'abc'"))); + FunctionCall expect = new FunctionCall("JSON_QUERY", params); + expect.addOption(new NumberType("double", null, null)); + JsonOption jsonOpt = new JsonOption(); + JsonOnOption jsonOnOption = new JsonOnOption(); + jsonOnOption.setOnEmpty(new ConstExpression("empty array")); + jsonOnOption.setOnError(new ConstExpression("empty array")); + jsonOnOption.setOnMismatches(Collections.singletonList( + new JsonOnOption.OnMismatch(new ConstExpression("error_p"), null))); + jsonOpt.setOnOption(jsonOnOption); + expect.addOption(jsonOpt); + Assert.assertEquals(expect, actual); + } + + @Test + public void generate_jsonQueryExprFullOpts4_generateFunctionCallSucceed() { + ExprContext context = getExprContext("JSON_QUERY('123', _utf8 'abc' " + + "returning double " + + "empty object on empty empty array on error_p error_p on mismatch)"); + StatementFactory factory = new MySQLExpressionFactory(context); + Expression actual = factory.generate(); + + List params = new ArrayList<>(); + params.add(new ExpressionParam(new ConstExpression("'123'"))); + params.add(new ExpressionParam(new ConstExpression("_utf8 'abc'"))); + FunctionCall expect = new FunctionCall("JSON_QUERY", params); + expect.addOption(new NumberType("double", null, null)); + JsonOption jsonOpt = new JsonOption(); + JsonOnOption jsonOnOption = new JsonOnOption(); + jsonOnOption.setOnEmpty(new ConstExpression("empty object")); + jsonOnOption.setOnError(new ConstExpression("empty array")); + jsonOnOption.setOnMismatches(Collections.singletonList( + new JsonOnOption.OnMismatch(new ConstExpression("error_p"), null))); + jsonOpt.setOnOption(jsonOnOption); + expect.addOption(jsonOpt); + Assert.assertEquals(expect, actual); + } + + @Test + public void generate_jsonQueryExpr1_generateFunctionCallSucceed() { + ExprContext context = getExprContext("JSON_QUERY('123', _utf8 'abc' " + + "returning double " + + "with array wrapper)"); + StatementFactory factory = new MySQLExpressionFactory(context); + Expression actual = factory.generate(); + + List params = new ArrayList<>(); + params.add(new ExpressionParam(new ConstExpression("'123'"))); + params.add(new ExpressionParam(new ConstExpression("_utf8 'abc'"))); + FunctionCall expect = new FunctionCall("JSON_QUERY", params); + expect.addOption(new NumberType("double", null, null)); + JsonOption jsonOpt = new JsonOption(); + jsonOpt.setWrapperMode(JsonOption.WrapperMode.WITH_ARRAY_WRAPPER); + expect.addOption(jsonOpt); + Assert.assertEquals(expect, actual); + } + + @Test + public void generate_jsonQueryExpr2_generateFunctionCallSucceed() { + ExprContext context = getExprContext("JSON_QUERY('123', _utf8 'abc' " + + "returning double " + + "with conditional wrapper)"); + StatementFactory factory = new MySQLExpressionFactory(context); + Expression actual = factory.generate(); + + List params = new ArrayList<>(); + params.add(new ExpressionParam(new ConstExpression("'123'"))); + params.add(new ExpressionParam(new ConstExpression("_utf8 'abc'"))); + FunctionCall expect = new FunctionCall("JSON_QUERY", params); + expect.addOption(new NumberType("double", null, null)); + JsonOption jsonOpt = new JsonOption(); + jsonOpt.setWrapperMode(JsonOption.WrapperMode.WITH_CONDITIONAL_WRAPPER); + expect.addOption(jsonOpt); + Assert.assertEquals(expect, actual); + } + + @Test + public void generate_jsonQueryExpr3_generateFunctionCallSucceed() { + ExprContext context = getExprContext("JSON_QUERY('123', _utf8 'abc' " + + "returning double " + + "with unconditional wrapper)"); + StatementFactory factory = new MySQLExpressionFactory(context); + Expression actual = factory.generate(); + + List params = new ArrayList<>(); + params.add(new ExpressionParam(new ConstExpression("'123'"))); + params.add(new ExpressionParam(new ConstExpression("_utf8 'abc'"))); + FunctionCall expect = new FunctionCall("JSON_QUERY", params); + expect.addOption(new NumberType("double", null, null)); + JsonOption jsonOpt = new JsonOption(); + jsonOpt.setWrapperMode(JsonOption.WrapperMode.WITH_UNCONDITIONAL_WRAPPER); + expect.addOption(jsonOpt); + Assert.assertEquals(expect, actual); + } + + @Test + public void generate_jsonQueryExpr4_generateFunctionCallSucceed() { + ExprContext context = getExprContext("JSON_QUERY('123', _utf8 'abc' " + + "returning double " + + "with wrapper)"); + StatementFactory factory = new MySQLExpressionFactory(context); + Expression actual = factory.generate(); + + List params = new ArrayList<>(); + params.add(new ExpressionParam(new ConstExpression("'123'"))); + params.add(new ExpressionParam(new ConstExpression("_utf8 'abc'"))); + FunctionCall expect = new FunctionCall("JSON_QUERY", params); + expect.addOption(new NumberType("double", null, null)); + JsonOption jsonOpt = new JsonOption(); + jsonOpt.setWrapperMode(JsonOption.WrapperMode.WITH_WRAPPER); + expect.addOption(jsonOpt); + Assert.assertEquals(expect, actual); + } + + @Test + public void generate_jsonQueryExpr5_generateFunctionCallSucceed() { + ExprContext context = getExprContext("JSON_QUERY('123', _utf8 'abc' " + + "returning double " + + "without wrapper)"); + StatementFactory factory = new MySQLExpressionFactory(context); + Expression actual = factory.generate(); + + List params = new ArrayList<>(); + params.add(new ExpressionParam(new ConstExpression("'123'"))); + params.add(new ExpressionParam(new ConstExpression("_utf8 'abc'"))); + FunctionCall expect = new FunctionCall("JSON_QUERY", params); + expect.addOption(new NumberType("double", null, null)); + JsonOption jsonOpt = new JsonOption(); + jsonOpt.setWrapperMode(JsonOption.WrapperMode.WITHOUT_WRAPPER); + expect.addOption(jsonOpt); + Assert.assertEquals(expect, actual); + } + + @Test + public void generate_jsonQueryExpr6_generateFunctionCallSucceed() { + ExprContext context = getExprContext("JSON_QUERY('123', _utf8 'abc' " + + "returning double " + + "without array wrapper)"); + StatementFactory factory = new MySQLExpressionFactory(context); + Expression actual = factory.generate(); + + List params = new ArrayList<>(); + params.add(new ExpressionParam(new ConstExpression("'123'"))); + params.add(new ExpressionParam(new ConstExpression("_utf8 'abc'"))); + FunctionCall expect = new FunctionCall("JSON_QUERY", params); + expect.addOption(new NumberType("double", null, null)); + JsonOption jsonOpt = new JsonOption(); + jsonOpt.setWrapperMode(JsonOption.WrapperMode.WITHOUT_ARRAY_WRAPPER); + expect.addOption(jsonOpt); + Assert.assertEquals(expect, actual); + } + + @Test + public void generate_jsonQueryExprFullOpts1_generateFunctionCallSucceed() { + ExprContext context = getExprContext("JSON_QUERY('123', _utf8 'abc' " + + "returning double " + + "TRUNCATE " + + "disallow scalars " + + "pretty " + + "ASCII " + + "with conditional array wrapper " + + "asis " + + "error_p on empty empty object on error_p dot on mismatch " + + "MULTIVALUE)"); + StatementFactory factory = new MySQLExpressionFactory(context); + Expression actual = factory.generate(); + + List params = new ArrayList<>(); + params.add(new ExpressionParam(new ConstExpression("'123'"))); + params.add(new ExpressionParam(new ConstExpression("_utf8 'abc'"))); + FunctionCall expect = new FunctionCall("JSON_QUERY", params); + expect.addOption(new NumberType("double", null, null)); + JsonOption jsonOpt = new JsonOption(); + jsonOpt.setTruncate(true); + jsonOpt.setScalarsMode(JsonOption.ScalarsMode.DISALLOW_SCALARS); + jsonOpt.setPretty(true); + jsonOpt.setAscii(true); + jsonOpt.setWrapperMode(JsonOption.WrapperMode.WITH_CONDITIONAL_ARRAY_WRAPPER); + jsonOpt.setAsis(true); + jsonOpt.setMultiValue(true); + JsonOnOption jsonOnOption = new JsonOnOption(); + jsonOnOption.setOnEmpty(new ConstExpression("error_p")); + jsonOnOption.setOnError(new ConstExpression("empty object")); + jsonOnOption.setOnMismatches(Collections.singletonList( + new JsonOnOption.OnMismatch(new ConstExpression("dot"), null))); + jsonOpt.setOnOption(jsonOnOption); + expect.addOption(jsonOpt); + Assert.assertEquals(expect, actual); + } + @Test public void generate_jsonValueExpr_generateFunctionCallSucceed() { ExprContext context = getExprContext("JSON_VALUE('123', _utf8 'abc' returning double TRUNCATE ASCII)"); @@ -768,8 +1044,10 @@ public void generate_jsonValueExpr_generateFunctionCallSucceed() { params.add(p); FunctionCall expect = new FunctionCall("JSON_VALUE", params); expect.addOption(new NumberType("double", null, null)); - expect.addOption(new ConstExpression("TRUNCATE")); - expect.addOption(new ConstExpression("ASCII")); + JsonOption jsonOpt = new JsonOption(); + jsonOpt.setTruncate(true); + jsonOpt.setAscii(true); + expect.addOption(jsonOpt); Assert.assertEquals(expect, actual); } @@ -786,11 +1064,13 @@ public void generate_jsonValueExprOnEmpty_generateFunctionCallSucceed() { params.add(p); FunctionCall expect = new FunctionCall("JSON_VALUE", params); expect.addOption(new NumberType("double", null, null)); - expect.addOption(new ConstExpression("TRUNCATE")); - expect.addOption(new ConstExpression("ASCII")); + JsonOption jsonOpt = new JsonOption(); + jsonOpt.setTruncate(true); + jsonOpt.setAscii(true); JsonOnOption jsonOnOption = new JsonOnOption(); jsonOnOption.setOnEmpty(new ConstExpression("error_p")); - expect.addOption(jsonOnOption); + jsonOpt.setOnOption(jsonOnOption); + expect.addOption(jsonOpt); Assert.assertEquals(expect, actual); } @@ -807,11 +1087,13 @@ public void generate_jsonValueExprOnError_generateFunctionCallSucceed() { params.add(p); FunctionCall expect = new FunctionCall("JSON_VALUE", params); expect.addOption(new NumberType("double", null, null)); - expect.addOption(new ConstExpression("TRUNCATE")); - expect.addOption(new ConstExpression("ASCII")); + JsonOption jsonOpt = new JsonOption(); + jsonOpt.setTruncate(true); + jsonOpt.setAscii(true); JsonOnOption jsonOnOption = new JsonOnOption(); jsonOnOption.setOnError(new NullExpression()); - expect.addOption(jsonOnOption); + jsonOpt.setOnOption(jsonOnOption); + expect.addOption(jsonOpt); Assert.assertEquals(expect, actual); } @@ -828,12 +1110,30 @@ public void generate_jsonValueExprOnErrorEmpty_generateFunctionCallSucceed() { params.add(p); FunctionCall expect = new FunctionCall("JSON_VALUE", params); expect.addOption(new NumberType("double", null, null)); - expect.addOption(new ConstExpression("TRUNCATE")); - expect.addOption(new ConstExpression("ASCII")); + JsonOption jsonOpt = new JsonOption(); + jsonOpt.setTruncate(true); + jsonOpt.setAscii(true); JsonOnOption jsonOnOption = new JsonOnOption(); jsonOnOption.setOnError(new NullExpression()); jsonOnOption.setOnEmpty(new ConstExpression("12")); - expect.addOption(jsonOnOption); + jsonOpt.setOnOption(jsonOnOption); + expect.addOption(jsonOpt); + Assert.assertEquals(expect, actual); + } + + @Test + public void generate_jsonValueExprNoOpt_generateFunctionCallSucceed() { + ExprContext context = getExprContext("JSON_VALUE('123', _utf8 'abc' returning double)"); + StatementFactory factory = new MySQLExpressionFactory(context); + Expression actual = factory.generate(); + + List params = new ArrayList<>(); + params.add(new ExpressionParam(new ConstExpression("'123'"))); + FunctionParam p = new ExpressionParam(new ConstExpression("_utf8 'abc'")); + params.add(p); + FunctionCall expect = new FunctionCall("JSON_VALUE", params); + expect.addOption(new NumberType("double", null, null)); + expect.addOption(new JsonOption()); Assert.assertEquals(expect, actual); } @@ -1430,6 +1730,20 @@ public void generate_vectorDistanceExpr_Succeed() { Assert.assertEquals(expected, actual); } + @Test + public void generate_vectorDistanceExpr1_Succeed() { + ExprContext context = getExprContext("VECTOR_DISTANCE(vector1, vector2, COSINE)"); + StatementFactory factory = new MySQLExpressionFactory(context); + Expression actual = factory.generate(); + + List params = new ArrayList<>(); + params.add(new ExpressionParam(new ColumnReference(null, null, "vector1"))); + params.add(new ExpressionParam(new ColumnReference(null, null, "vector2"))); + params.add(new ExpressionParam(new ConstExpression("COSINE"))); + FunctionCall expected = new FunctionCall("VECTOR_DISTANCE", params); + Assert.assertEquals(expected, actual); + } + @Test public void generate_AnyArrayExpr_1_Succeed() { ExprContext context = getExprContext("\"hel\" = ANY([\"hello\", \"hi\"])"); diff --git a/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/MySQLFromReferenceFactoryTest.java b/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/MySQLFromReferenceFactoryTest.java index c840f8744d..62d9da22a0 100644 --- a/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/MySQLFromReferenceFactoryTest.java +++ b/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/MySQLFromReferenceFactoryTest.java @@ -15,10 +15,7 @@ */ package com.oceanbase.tools.sqlparser.adapter; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; +import java.util.*; import org.antlr.v4.runtime.BailErrorStrategy; import org.antlr.v4.runtime.CharStreams; @@ -38,14 +35,7 @@ import com.oceanbase.tools.sqlparser.statement.common.BraceBlock; import com.oceanbase.tools.sqlparser.statement.common.GeneralDataType; import com.oceanbase.tools.sqlparser.statement.common.NumberType; -import com.oceanbase.tools.sqlparser.statement.expression.ColumnReference; -import com.oceanbase.tools.sqlparser.statement.expression.CompoundExpression; -import com.oceanbase.tools.sqlparser.statement.expression.ConstExpression; -import com.oceanbase.tools.sqlparser.statement.expression.ExpressionParam; -import com.oceanbase.tools.sqlparser.statement.expression.FunctionCall; -import com.oceanbase.tools.sqlparser.statement.expression.FunctionParam; -import com.oceanbase.tools.sqlparser.statement.expression.JsonOnOption; -import com.oceanbase.tools.sqlparser.statement.expression.NullExpression; +import com.oceanbase.tools.sqlparser.statement.expression.*; import com.oceanbase.tools.sqlparser.statement.select.ExpressionReference; import com.oceanbase.tools.sqlparser.statement.select.FlashBackType; import com.oceanbase.tools.sqlparser.statement.select.FlashbackUsage; @@ -169,6 +159,7 @@ public void generate_jsonTable2_generateSucceed() { FunctionParam op2 = new ExpressionParam(new ColumnReference(null, null, "col1")); op2.addOption(new NumberType("int", null, null)); op2.addOption(new ConstExpression("123")); + op2.addOption(new JsonOption()); fcall.addOption(op2); FunctionParam op3 = new ExpressionParam(new ColumnReference(null, null, "col2")); @@ -177,7 +168,9 @@ public void generate_jsonTable2_generateSucceed() { op3.addOption(new ConstExpression("123")); JsonOnOption onOption = new JsonOnOption(); onOption.setOnEmpty(new NullExpression()); - op3.addOption(onOption); + JsonOption jsonOpt = new JsonOption(); + jsonOpt.setOnOption(onOption); + op3.addOption(jsonOpt); fcall.addOption(op3); FunctionParam op4 = new ExpressionParam(new ColumnReference(null, null, "col3")); @@ -186,7 +179,9 @@ public void generate_jsonTable2_generateSucceed() { op4.addOption(new ConstExpression("123")); onOption = new JsonOnOption(); onOption.setOnError(new ConstExpression("error_p")); - op4.addOption(onOption); + jsonOpt = new JsonOption(); + jsonOpt.setOnOption(onOption); + op4.addOption(jsonOpt); fcall.addOption(op4); FunctionParam op5 = new ExpressionParam(new ColumnReference(null, null, "col4")); @@ -196,7 +191,9 @@ public void generate_jsonTable2_generateSucceed() { onOption = new JsonOnOption(); onOption.setOnEmpty(new NullExpression()); onOption.setOnError(new ConstExpression("error_p")); - op5.addOption(onOption); + jsonOpt = new JsonOption(); + jsonOpt.setOnOption(onOption); + op5.addOption(jsonOpt); fcall.addOption(op5); ExpressionReference expect = new ExpressionReference(fcall, null); Assert.assertEquals(expect, actual); @@ -298,6 +295,21 @@ public void generate_nameRefWithPartition_generateNameRefSucceed() { Assert.assertEquals(expect, actual); } + @Test + public void generate_nameRefWithExternalPartition_generateNameRefSucceed() { + Table_referenceContext context = getTableReferenceContext( + "select a from mysql.tab partition (col1=@@global.lalal, col2='aaa')"); + StatementFactory factory = new MySQLFromReferenceFactory(context); + FromReference actual = factory.generate(); + + NameReference expect = new NameReference("mysql", "tab", null); + Map partitionMap = new HashMap<>(); + partitionMap.put("col1", new ConstExpression("@@global.lalal")); + partitionMap.put("col2", new ConstExpression("'aaa'")); + expect.setPartitionUsage(new PartitionUsage(PartitionType.PARTITION, partitionMap)); + Assert.assertEquals(expect, actual); + } + @Test public void generate_nameRefWithFlashback_generateNameRefSucceed() { Table_referenceContext context = diff --git a/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/MySQLInsertFactoryTest.java b/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/MySQLInsertFactoryTest.java index 5d58c3fe27..e0f89ab1f6 100644 --- a/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/MySQLInsertFactoryTest.java +++ b/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/MySQLInsertFactoryTest.java @@ -71,6 +71,24 @@ public void generate_simpleInsert_succeed() { Assert.assertEquals(actual, expect); } + @Test + public void generate_insertWithHighPriority_succeed() { + StatementFactory factory = new MySQLInsertFactory( + getInsertContext("insert high_priority overwrite a.b values(1,default)")); + Insert actual = factory.generate(); + + RelationFactor factor = new RelationFactor("b"); + factor.setSchema("a"); + InsertTable insertTable = new InsertTable(factor); + List> values = new ArrayList<>(); + values.add(Arrays.asList(new ConstExpression("1"), new ConstExpression("default"))); + insertTable.setValues(values); + Insert expect = new Insert(Collections.singletonList(insertTable), null); + expect.setHighPriority(true); + expect.setOverwrite(true); + Assert.assertEquals(actual, expect); + } + @Test public void generate_simpleReplace_succeed() { StatementFactory factory = new MySQLInsertFactory(getInsertContext( diff --git a/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/MySQLSelectFactoryTest.java b/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/MySQLSelectFactoryTest.java index f5ee27b5cc..3ac36da245 100644 --- a/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/MySQLSelectFactoryTest.java +++ b/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/MySQLSelectFactoryTest.java @@ -116,7 +116,7 @@ public void generate_parentSelectUnionDual_generateSelectSucceed() { @Test public void generate_orderAndLimitUnion_generateSelectSucceed() { Select_stmtContext context = getSelectContext( - "select col.* abc from tab order by col desc limit 3 union distinct select * from dual"); + "select col.* abc from tab order by col desc approx limit 3 union distinct select * from dual"); StatementFactory factory = new MySQLSelectFactory(context); Select actual = factory.generate(); @@ -249,6 +251,7 @@ public void generate_fromSelectStatment_generateSelectSucceed() { fromBody.setOrderBy(orderBy); FlashbackUsage flashbackUsage = new FlashbackUsage(FlashBackType.AS_OF_SNAPSHOT, new ConstExpression("1")); ExpressionReference from = new ExpressionReference(fromBody, "abc"); + from.setLateral(true); from.setFlashbackUsage(flashbackUsage); Select expect = new Select(new SelectBody(Collections.singletonList(p), Collections.singletonList(from))); Assert.assertEquals(expect, actual); diff --git a/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/MySQLTableElementFactoryTest.java b/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/MySQLTableElementFactoryTest.java index d11b8a92e8..6e928f7b81 100644 --- a/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/MySQLTableElementFactoryTest.java +++ b/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/MySQLTableElementFactoryTest.java @@ -16,10 +16,7 @@ package com.oceanbase.tools.sqlparser.adapter; import java.math.BigDecimal; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; +import java.util.*; import org.antlr.v4.runtime.BailErrorStrategy; import org.antlr.v4.runtime.CharStreams; @@ -86,6 +83,36 @@ public void generate_columnDef_generateSuccees() { Assert.assertEquals(expect, actual); } + @Test + public void generate_serialColumnDef_generateSuccees() { + StatementFactory factory = + new MySQLTableElementFactory(getTableElementContext("tb.col serial")); + ColumnDefinition actual = (ColumnDefinition) factory.generate(); + + ColumnDefinition expect = new ColumnDefinition(new ColumnReference(null, "tb", "col"), null); + expect.setSerial(true); + Assert.assertEquals(expect, actual); + } + + @Test + public void generate_columnDefWithForeignKeyIndexName_succeed() { + StatementFactory factory = new MySQLTableElementFactory(getTableElementContext( + "tb.col varchar(64) references a.b (col2, col3) match simple on delete cascade on update set default")); + TableElement actual = factory.generate(); + + DataType dataType = new CharacterType("varchar", new BigDecimal("64")); + ColumnDefinition expect = new ColumnDefinition(new ColumnReference(null, "tb", "col"), dataType); + + ColumnReference r1 = new ColumnReference(null, null, "col2"); + ColumnReference r2 = new ColumnReference(null, null, "col3"); + ForeignReference reference = new ForeignReference("a", "b", Arrays.asList(r1, r2)); + reference.setDeleteOption(OnOption.CASCADE); + reference.setUpdateOption(OnOption.SET_DEFAULT); + reference.setMatchOption(MatchOption.SIMPLE); + expect.setForeignReference(reference); + Assert.assertEquals(expect, actual); + } + @Test public void generate_columnDefFirst_generateSuccees() { StatementFactory factory = @@ -200,10 +227,25 @@ public void generate_columnDefDefaultExpr_generateSuccees() { Assert.assertEquals(expect, actual); } + @Test + public void generate_columnDefDefaultExpr1_generateSuccees() { + StatementFactory factory = + new MySQLTableElementFactory(getTableElementContext("tb.col varchar(64) default (1) chunk '123'")); + ColumnDefinition actual = (ColumnDefinition) factory.generate(); + + DataType dataType = new CharacterType("varchar", new BigDecimal("64")); + ColumnDefinition expect = new ColumnDefinition(new ColumnReference(null, "tb", "col"), dataType); + ColumnAttributes attributes = new ColumnAttributes(); + attributes.setDefaultValue(new ConstExpression("1")); + attributes.setLobChunkSize("'123'"); + expect.setColumnAttributes(attributes); + Assert.assertEquals(expect, actual); + } + @Test public void generate_columnDefKey_generateSuccees() { StatementFactory factory = - new MySQLTableElementFactory(getTableElementContext("tb.col varchar(64) key")); + new MySQLTableElementFactory(getTableElementContext("tb.col varchar(64) key chunk 123")); ColumnDefinition actual = (ColumnDefinition) factory.generate(); DataType dataType = new CharacterType("varchar", new BigDecimal("64")); @@ -212,6 +254,7 @@ public void generate_columnDefKey_generateSuccees() { InLineConstraint constraint = new InLineConstraint(null, null); constraint.setPrimaryKey(true); attributes.setConstraints(Collections.singletonList(constraint)); + attributes.setLobChunkSize("123"); expect.setColumnAttributes(attributes); Assert.assertEquals(expect, actual); } @@ -219,7 +262,8 @@ public void generate_columnDefKey_generateSuccees() { @Test public void generate_columnDefOrigDefaultExpr_generateSuccees() { StatementFactory factory = new MySQLTableElementFactory( - getTableElementContext("tb.col varchar(64) orig_default current_timestamp(1)")); + getTableElementContext( + "tb.col varchar(64) orig_default current_timestamp(1) column_format default storage disk")); ColumnDefinition actual = (ColumnDefinition) factory.generate(); DataType dataType = new CharacterType("varchar", new BigDecimal("64")); @@ -228,6 +272,8 @@ public void generate_columnDefOrigDefaultExpr_generateSuccees() { FunctionCall expr = new FunctionCall("current_timestamp", Collections.singletonList(new ExpressionParam(new ConstExpression("1")))); attributes.setOrigDefault(expr); + attributes.setColumnFormat("default"); + attributes.setStorage("disk"); expect.setColumnAttributes(attributes); Assert.assertEquals(expect, actual); } @@ -647,7 +693,7 @@ public void generate_columnDefCheckConstraint1_generateSuccees() { @Test public void generate_indexBtree_succeed() { StatementFactory factory = new MySQLTableElementFactory( - getTableElementContext("index idx_name using btree (col, col1) global with parser 'aaaa'")); + getTableElementContext("index idx_name using btree (col, col1) global with parser `aaaa`")); OutOfLineIndex actual = (OutOfLineIndex) factory.generate(); SortColumn s1 = new SortColumn(new ColumnReference(null, null, "col")); @@ -656,7 +702,7 @@ public void generate_indexBtree_succeed() { IndexOptions indexOptions = new IndexOptions(); indexOptions.setUsingBtree(true); indexOptions.setGlobal(true); - indexOptions.setWithParser("'aaaa'"); + indexOptions.setWithParser("`aaaa`"); expect.setIndexOptions(indexOptions); Assert.assertEquals(expect, actual); } @@ -665,7 +711,7 @@ public void generate_indexBtree_succeed() { public void generate_exprIndexBtree_succeed() { StatementFactory factory = new MySQLTableElementFactory( getTableElementContext( - "index idx_name using btree ((CASE a WHEN 1 THEN 11 WHEN 2 THEN 22 ELSE 33 END)) global with parser 'aaaa'")); + "index idx_name using btree ((CASE a WHEN 1 THEN 11 WHEN 2 THEN 22 ELSE 33 END)) global with parser `aaaa`")); OutOfLineIndex actual = (OutOfLineIndex) factory.generate(); List whenClauses = new ArrayList<>(); @@ -679,7 +725,7 @@ public void generate_exprIndexBtree_succeed() { IndexOptions indexOptions = new IndexOptions(); indexOptions.setUsingBtree(true); indexOptions.setGlobal(true); - indexOptions.setWithParser("'aaaa'"); + indexOptions.setWithParser("`aaaa`"); expect.setIndexOptions(indexOptions); Assert.assertEquals(expect, actual); } @@ -865,7 +911,7 @@ public void generate_indexPrimaryKeyNoyAlgorithmAndComment_succeed() { public void generate_uniqueIndexColumnAscId_succeed() { StatementFactory factory = new MySQLTableElementFactory( getTableElementContext( - "unique index idx_name using btree (col asc id 16, col1) global with parser 'aaaa'")); + "unique index idx_name using btree (col asc id 16, col1) global with parser `aaaa`")); TableElement actual = factory.generate(); SortColumn s1 = new SortColumn(new ColumnReference(null, null, "col")); @@ -876,7 +922,7 @@ public void generate_uniqueIndexColumnAscId_succeed() { IndexOptions indexOptions = new IndexOptions(); indexOptions.setUsingBtree(true); indexOptions.setGlobal(true); - indexOptions.setWithParser("'aaaa'"); + indexOptions.setWithParser("`aaaa`"); state.setIndexOptions(indexOptions); OutOfLineConstraint expect = new OutOfLineConstraint(state, Arrays.asList(s1, s2)); expect.setUniqueKey(true); @@ -931,7 +977,7 @@ public void generate_uniqueIndexAutoPartition_succeed() { @Test public void generate_uniqueIndexColumnAscIdNoIndexOps_succeed() { StatementFactory factory = new MySQLTableElementFactory( - getTableElementContext("unique index idx_name (col asc id 16, col1) global with parser 'aaaa'")); + getTableElementContext("unique index idx_name (col asc id 16, col1) global with parser `aaaa`")); TableElement actual = factory.generate(); SortColumn s1 = new SortColumn(new ColumnReference(null, null, "col")); @@ -941,7 +987,7 @@ public void generate_uniqueIndexColumnAscIdNoIndexOps_succeed() { ConstraintState state = new ConstraintState(); IndexOptions indexOptions = new IndexOptions(); indexOptions.setGlobal(true); - indexOptions.setWithParser("'aaaa'"); + indexOptions.setWithParser("`aaaa`"); state.setIndexOptions(indexOptions); OutOfLineConstraint expect = new OutOfLineConstraint(state, Arrays.asList(s1, s2)); expect.setUniqueKey(true); @@ -1118,8 +1164,29 @@ public void generate_outOfLineIndexVectorIndex_succeed() { TableElement actual = factory.generate(); OutOfLineIndex expected = new OutOfLineIndex("idx1", Collections.singletonList(new SortColumn(new ColumnReference(null, null, "c2")))); - expected.setIndexOptions(new IndexOptions()); + IndexOptions indexOptions = new IndexOptions(); + expected.setIndexOptions(indexOptions); expected.setVector(true); + Map params = new HashMap<>(); + params.put("distance", "L2"); + params.put("type", "hnsw"); + indexOptions.setVectorIndexParams(params); + Assert.assertEquals(expected, actual); + } + + @Test + public void generate_outOfLineIndexFullTextIndex_getWithParser_Succeed() { + StatementFactory factory = new MySQLTableElementFactory( + getTableElementContext("FULLTEXT KEY idx1 (name) WITH PARSER space")); + TableElement actual = factory.generate(); + + OutOfLineIndex expected = new OutOfLineIndex("idx1", + Collections.singletonList(new SortColumn(new ColumnReference(null, null, "name")))); + expected.setFullText(true); + IndexOptions indexOptions = new IndexOptions(); + indexOptions.setWithParser("space"); + expected.setIndexOptions(indexOptions); + Assert.assertEquals(expected, actual); } diff --git a/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/OracleAlterTableActionFactoryTest.java b/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/OracleAlterTableActionFactoryTest.java index 94415e83de..c82d947b05 100644 --- a/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/OracleAlterTableActionFactoryTest.java +++ b/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/OracleAlterTableActionFactoryTest.java @@ -82,6 +82,17 @@ public void generate_tableOptions_succeed() { Assert.assertEquals(expect, actual); } + @Test + public void generate_exchangePartition_succeed() { + StatementFactory factory = new OracleAlterTableActionFactory( + getActionContext("exchange partition p1 with table tbl including indexes without validation")); + AlterTableAction actual = factory.generate(); + + AlterTableAction expect = new AlterTableAction(); + expect.setExchangePartition("p1", new RelationFactor("tbl")); + Assert.assertEquals(expect, actual); + } + @Test public void generate_setInterval_succeed() { StatementFactory factory = new OracleAlterTableActionFactory( diff --git a/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/OracleAlterTableFactoryTest.java b/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/OracleAlterTableFactoryTest.java index 95b8339865..65b33773b9 100644 --- a/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/OracleAlterTableFactoryTest.java +++ b/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/OracleAlterTableFactoryTest.java @@ -18,6 +18,8 @@ import java.math.BigDecimal; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; +import java.util.Map; import org.antlr.v4.runtime.BailErrorStrategy; import org.antlr.v4.runtime.CharStreams; @@ -29,6 +31,7 @@ import com.oceanbase.tools.sqlparser.oboracle.OBLexer; import com.oceanbase.tools.sqlparser.oboracle.OBParser; import com.oceanbase.tools.sqlparser.oboracle.OBParser.Alter_table_stmtContext; +import com.oceanbase.tools.sqlparser.statement.Expression; import com.oceanbase.tools.sqlparser.statement.alter.table.AlterTable; import com.oceanbase.tools.sqlparser.statement.alter.table.AlterTableAction; import com.oceanbase.tools.sqlparser.statement.common.CharacterType; @@ -37,6 +40,7 @@ import com.oceanbase.tools.sqlparser.statement.createtable.ColumnDefinition; import com.oceanbase.tools.sqlparser.statement.createtable.TableOptions; import com.oceanbase.tools.sqlparser.statement.expression.ColumnReference; +import com.oceanbase.tools.sqlparser.statement.expression.ConstExpression; /** * Test cases for {@link OracleAlterTableFactory} @@ -100,6 +104,52 @@ public void generate_alterTable_dropColumnGroup_succeed() { Assert.assertEquals(expect, actual); } + @Test + public void generate_alterExternalTableAddEmptyPartition_succeed() { + StatementFactory factory = new OracleAlterTableFactory( + getAlterContext("alter external table a.b add partition () location 'abcd'")); + AlterTable actual = factory.generate(); + + AlterTableAction action = new AlterTableAction(); + action.setExternalTableLocation("'abcd'"); + action.setAddExternalTablePartition(new HashMap<>()); + AlterTable expect = new AlterTable(getRelationFactor("a", "b"), Collections.singletonList(action)); + expect.setExternal(true); + Assert.assertEquals(expect, actual); + } + + @Test + public void generate_alterExternalTableAddPartition_succeed() { + StatementFactory factory = new OracleAlterTableFactory( + getAlterContext( + "alter external table a.b add partition (col='aaa', col1=@@global.ss) location 'abcd'")); + AlterTable actual = factory.generate(); + + AlterTableAction action = new AlterTableAction(); + action.setExternalTableLocation("'abcd'"); + Map partitions = new HashMap<>(); + partitions.put("col", new ConstExpression("'aaa'")); + partitions.put("col1", new ConstExpression("@@global.ss")); + action.setAddExternalTablePartition(partitions); + AlterTable expect = new AlterTable(getRelationFactor("a", "b"), Collections.singletonList(action)); + expect.setExternal(true); + Assert.assertEquals(expect, actual); + } + + @Test + public void generate_alterExternalTableDropPartition_succeed() { + StatementFactory factory = new OracleAlterTableFactory( + getAlterContext("alter external table a.b drop partition location 'abcd'")); + AlterTable actual = factory.generate(); + + AlterTableAction action = new AlterTableAction(); + action.setExternalTableLocation("'abcd'"); + action.setDropExternalTablePartition(true); + AlterTable expect = new AlterTable(getRelationFactor("a", "b"), Collections.singletonList(action)); + expect.setExternal(true); + Assert.assertEquals(expect, actual); + } + private Alter_table_stmtContext getAlterContext(String action) { OBLexer lexer = new OBLexer(CharStreams.fromString(action)); CommonTokenStream tokens = new CommonTokenStream(lexer); @@ -113,4 +163,5 @@ private RelationFactor getRelationFactor(String schema, String relation) { relationFactor.setSchema(schema); return relationFactor; } + } diff --git a/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/OracleCreateIndexFactoryTest.java b/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/OracleCreateIndexFactoryTest.java index 4b6ccbb093..b9bd719255 100644 --- a/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/OracleCreateIndexFactoryTest.java +++ b/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/OracleCreateIndexFactoryTest.java @@ -59,6 +59,20 @@ public void generate_createIndex_succeed() { Assert.assertEquals(expect, actual); } + @Test + public void generate_createMdSysIndex_succeed() { + StatementFactory factory = new OracleCreateIndexFactory( + getCreateIdxContext("create index abc on tb (col, col1) indextype is mdsys.spatial_index ")); + CreateIndex actual = factory.generate(); + + CreateIndex expect = new CreateIndex(new RelationFactor("abc"), + new RelationFactor("tb"), Arrays.asList( + new SortColumn(new RelationReference("col", null)), + new SortColumn(new RelationReference("col1", null)))); + expect.setMdSysDotSpatialIndex(true); + Assert.assertEquals(expect, actual); + } + @Test public void generate_createUniqueIndex_succeed() { StatementFactory factory = new OracleCreateIndexFactory( diff --git a/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/OracleCreateTableFactoryTest.java b/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/OracleCreateTableFactoryTest.java index 6277c73730..a4ab213ef0 100644 --- a/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/OracleCreateTableFactoryTest.java +++ b/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/OracleCreateTableFactoryTest.java @@ -358,9 +358,13 @@ public void generate_tableWithPartition_succeed() { @Test public void generate_formatTableOp_succeed() { - Create_table_stmtContext context = getCreateTableContext( - "create table any_schema.abcd (id varchar(64)) format=(ENCODING='aaaa',LINE_DELIMITER=123,SKIP_HEADER=12," - + "EMPTY_FIELD_AS_NULL=true,NULL_IF_EXETERNAL=(1,2,3))"); + Create_table_stmtContext context = getCreateTableContext("create table any_schema.abcd (id varchar(64)) " + + "properties=(ACCESSID='abcd', PROJECT_NAME='ooop') " + + "partition_type=USER_SPECIFIED " + + "micro_index_clustered=true " + + "auto_refresh=OFF " + + "format=(ENCODING='aaaa',LINE_DELIMITER=123,SKIP_HEADER=12," + + "EMPTY_FIELD_AS_NULL=true,NULL_IF_EXETERNAL=(1,2,3))"); StatementFactory factory = new OracleCreateTableFactory(context); CreateTable actual = factory.generate(); @@ -380,6 +384,13 @@ public void generate_formatTableOp_succeed() { map.put("NULL_IF_EXETERNAL", es); map.put("LINE_DELIMITER", new ConstExpression("123")); tableOptions.setFormat(map); + tableOptions.setPartitionType("USER_SPECIFIED"); + tableOptions.setMicroIndexClustered(true); + tableOptions.setAutoRefresh("OFF"); + Map externalProperties = new HashMap<>(); + externalProperties.put("ACCESSID", "'abcd'"); + externalProperties.put("PROJECT_NAME", "'ooop'"); + tableOptions.setExternalProperties(externalProperties); expect.setTableOptions(tableOptions); Assert.assertEquals(expect, actual); } diff --git a/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/OracleExpressionFactoryTest.java b/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/OracleExpressionFactoryTest.java index 6276205688..36457234c1 100644 --- a/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/OracleExpressionFactoryTest.java +++ b/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/OracleExpressionFactoryTest.java @@ -33,6 +33,7 @@ import com.oceanbase.tools.sqlparser.oboracle.OBParser.Bit_exprContext; import com.oceanbase.tools.sqlparser.oboracle.OBParser.ExprContext; import com.oceanbase.tools.sqlparser.oboracle.OBParser.PredicateContext; +import com.oceanbase.tools.sqlparser.oboracle.OBParser.Single_row_functionContext; import com.oceanbase.tools.sqlparser.oboracle.OBParser.Xml_functionContext; import com.oceanbase.tools.sqlparser.statement.Expression; import com.oceanbase.tools.sqlparser.statement.Expression.ReferenceOperator; @@ -56,14 +57,14 @@ import com.oceanbase.tools.sqlparser.statement.expression.FullTextSearch; import com.oceanbase.tools.sqlparser.statement.expression.FunctionCall; import com.oceanbase.tools.sqlparser.statement.expression.FunctionParam; -import com.oceanbase.tools.sqlparser.statement.expression.JsonConstraint; -import com.oceanbase.tools.sqlparser.statement.expression.JsonConstraint.ScalarsMode; -import com.oceanbase.tools.sqlparser.statement.expression.JsonConstraint.StrictMode; -import com.oceanbase.tools.sqlparser.statement.expression.JsonConstraint.UniqueMode; -import com.oceanbase.tools.sqlparser.statement.expression.JsonConstraint.WrapperMode; import com.oceanbase.tools.sqlparser.statement.expression.JsonKeyValue; import com.oceanbase.tools.sqlparser.statement.expression.JsonOnOption; import com.oceanbase.tools.sqlparser.statement.expression.JsonOnOption.OnMismatch; +import com.oceanbase.tools.sqlparser.statement.expression.JsonOption; +import com.oceanbase.tools.sqlparser.statement.expression.JsonOption.ScalarsMode; +import com.oceanbase.tools.sqlparser.statement.expression.JsonOption.StrictMode; +import com.oceanbase.tools.sqlparser.statement.expression.JsonOption.UniqueMode; +import com.oceanbase.tools.sqlparser.statement.expression.JsonOption.WrapperMode; import com.oceanbase.tools.sqlparser.statement.expression.NullExpression; import com.oceanbase.tools.sqlparser.statement.expression.ParamWithAssign; import com.oceanbase.tools.sqlparser.statement.expression.RelationReference; @@ -1609,6 +1610,41 @@ public void generate_xmlParse_generateSucceed() { Assert.assertEquals(expect, actual); } + @Test + public void generate_spatialCellid_generateSucceed() { + Single_row_functionContext context = getSingleRowFunctionContext("spatial_cellid('aaa')"); + OracleExpressionFactory factory = new OracleExpressionFactory(); + Expression actual = factory.visit(context); + + FunctionCall expect = new FunctionCall("spatial_cellid", + Collections.singletonList(new ExpressionParam(new ConstExpression("'aaa'")))); + Assert.assertEquals(expect, actual); + } + + @Test + public void generate_spatialMbr_generateSucceed() { + Single_row_functionContext context = getSingleRowFunctionContext("spatial_mbr('aaa')"); + OracleExpressionFactory factory = new OracleExpressionFactory(); + Expression actual = factory.visit(context); + + FunctionCall expect = new FunctionCall("spatial_mbr", + Collections.singletonList(new ExpressionParam(new ConstExpression("'aaa'")))); + Assert.assertEquals(expect, actual); + } + + @Test + public void generate_sdoRelate_generateSucceed() { + Single_row_functionContext context = getSingleRowFunctionContext("sdo_relate('aaa', 123, 456)"); + OracleExpressionFactory factory = new OracleExpressionFactory(); + Expression actual = factory.visit(context); + + FunctionCall expect = new FunctionCall("sdo_relate", Arrays.asList( + new ExpressionParam(new ConstExpression("'aaa'")), + new ExpressionParam(new ConstExpression("123")), + new ExpressionParam(new ConstExpression("456")))); + Assert.assertEquals(expect, actual); + } + @Test public void generate_deleteXml_generateSucceed() { Xml_functionContext context = getXmlExprContext("deletexml(1,2,3)"); @@ -2032,7 +2068,7 @@ public void generate_jsonConstrain_generateSucceed() { Expression actual = factory.generate(); Expression left = new ConstExpression("'aaa'"); - JsonConstraint right = new JsonConstraint(); + JsonOption right = new JsonOption(); right.setStrictMode(StrictMode.LAX); right.setScalarsMode(ScalarsMode.ALLOW_SCALARS); right.setUniqueMode(UniqueMode.WITH_UNIQUE_KEYS); @@ -2047,7 +2083,7 @@ public void generate_jsonConstrain1_generateSucceed() { Expression actual = factory.generate(); Expression left = new ConstExpression("'aaa'"); - JsonConstraint right = new JsonConstraint(); + JsonOption right = new JsonOption(); Expression expect = new CompoundExpression(left, right, Operator.EQ); Assert.assertEquals(expect, actual); } @@ -2107,7 +2143,9 @@ public void generate_jsonExists2_generateSucceed() { JsonOnOption jsonOnOption = new JsonOnOption(); jsonOnOption.setOnError(new BoolValue(true)); jsonOnOption.setOnEmpty(new ConstExpression("error_p")); - expect.addOption(jsonOnOption); + JsonOption jsonOpt = new JsonOption(); + jsonOpt.setOnOption(jsonOnOption); + expect.addOption(jsonOpt); Assert.assertEquals(expect, actual); } @@ -2122,7 +2160,9 @@ public void generate_accessFunc1_generateSucceed() { FunctionCall expect = new FunctionCall("func", Arrays.asList(p1, p2)); JsonOnOption jsonOnOption = new JsonOnOption(); jsonOnOption.setOnError(new BoolValue(true)); - expect.addOption(jsonOnOption); + JsonOption jsonOpt = new JsonOption(); + jsonOpt.setOnOption(jsonOnOption); + expect.addOption(jsonOpt); Assert.assertEquals(expect, actual); } @@ -2135,9 +2175,11 @@ public void generate_accessFunc3_generateSucceed() { FunctionParam p1 = new ExpressionParam(new ConstExpression("12")); FunctionParam p2 = new ExpressionParam(new ConstExpression("12")); FunctionCall expect = new FunctionCall("func", Arrays.asList(p1, p2)); + JsonOption jsonOpt = new JsonOption(); JsonOnOption jsonOnOption = new JsonOnOption(); jsonOnOption.setOnError(new ConstExpression("error_p")); - expect.addOption(jsonOnOption); + jsonOpt.setOnOption(jsonOnOption); + expect.addOption(jsonOpt); Assert.assertEquals(expect, actual); } @@ -2167,10 +2209,10 @@ public void generate_jsonArrayAgg1_generateSucceed() { expect.addOption(new OrderBy(Collections.singletonList(s))); JsonOnOption jsonOnOption = new JsonOnOption(); jsonOnOption.setOnNull(new ConstExpression("absent")); - expect.addOption(jsonOnOption); expect.addOption(new GeneralDataType("raw", Collections.singletonList("12"))); - JsonConstraint c = new JsonConstraint(); + JsonOption c = new JsonOption(); c.setStrictMode(StrictMode.STRICT); + c.setOnOption(jsonOnOption); expect.addOption(c); Assert.assertEquals(expect, actual); } @@ -2210,11 +2252,11 @@ public void generate_jsonObjAgg1_generateSucceed() { expect.addOption(new ConstExpression("format json")); JsonOnOption jsonOnOption = new JsonOnOption(); jsonOnOption.setOnNull(new NullExpression()); - expect.addOption(jsonOnOption); expect.addOption(new CharacterType("nvarchar2", new BigDecimal("12"))); - JsonConstraint c = new JsonConstraint(); + JsonOption c = new JsonOption(); c.setStrictMode(StrictMode.STRICT); c.setUniqueMode(UniqueMode.WITH_UNIQUE_KEYS); + c.setOnOption(jsonOnOption); expect.addOption(c); Assert.assertEquals(expect, actual); } @@ -2235,7 +2277,9 @@ public void generate_jsonExists3_generateSucceed() { JsonOnOption jsonOnOption = new JsonOnOption(); jsonOnOption.setOnError(new BoolValue(true)); jsonOnOption.setOnEmpty(new ConstExpression("error_p")); - expect.addOption(jsonOnOption); + JsonOption jsonOpt = new JsonOption(); + jsonOpt.setOnOption(jsonOnOption); + expect.addOption(jsonOpt); Assert.assertEquals(expect, actual); } @@ -2246,7 +2290,7 @@ public void generate_jsonConstrain2_generateSucceed() { Expression actual = factory.generate(); Expression left = new ConstExpression("'aaa'"); - JsonConstraint right = new JsonConstraint(); + JsonOption right = new JsonOption(); right.setStrictMode(StrictMode.STRICT); right.setScalarsMode(ScalarsMode.DISALLOW_SCALARS); right.setUniqueMode(UniqueMode.WITHOUT_UNIQUE_KEYS); @@ -2273,7 +2317,9 @@ public void generate_jsonObject1_generateSucceed() { FunctionCall expect = new FunctionCall("json_object", Collections.emptyList()); JsonOnOption jsonOnOption = new JsonOnOption(); jsonOnOption.setOnNull(new ConstExpression("absent")); - expect.addOption(jsonOnOption); + JsonOption jsonOpt = new JsonOption(); + jsonOpt.setOnOption(jsonOnOption); + expect.addOption(jsonOpt); Assert.assertEquals(expect, actual); } @@ -2297,11 +2343,11 @@ public void generate_jsonObject3_generateSucceed() { FunctionCall expect = new FunctionCall("json_object", Collections.emptyList()); JsonOnOption jsonOnOption = new JsonOnOption(); jsonOnOption.setOnNull(new NullExpression()); - expect.addOption(jsonOnOption); expect.addOption(new GeneralDataType("json", null)); - JsonConstraint jc = new JsonConstraint(); + JsonOption jc = new JsonOption(); jc.setStrictMode(StrictMode.STRICT); jc.setUniqueMode(UniqueMode.WITH_UNIQUE_KEYS); + jc.setOnOption(jsonOnOption); expect.addOption(jc); Assert.assertEquals(expect, actual); } @@ -2316,11 +2362,11 @@ public void generate_jsonObject4_generateSucceed() { FunctionCall expect = new FunctionCall("json_object", Collections.singletonList(p)); JsonOnOption jsonOnOption = new JsonOnOption(); jsonOnOption.setOnNull(new NullExpression()); - expect.addOption(jsonOnOption); expect.addOption(new GeneralDataType("json", null)); - JsonConstraint jc = new JsonConstraint(); + JsonOption jc = new JsonOption(); jc.setStrictMode(StrictMode.STRICT); jc.setUniqueMode(UniqueMode.WITH_UNIQUE_KEYS); + jc.setOnOption(jsonOnOption); expect.addOption(jc); Assert.assertEquals(expect, actual); } @@ -2344,7 +2390,7 @@ public void generate_jsonObject5_generateSucceed() { new ExpressionParam( new JsonKeyValue(new ConstExpression("' asdasdas '"), new RelationReference("col3", null))); FunctionCall expect = new FunctionCall("json_object", Arrays.asList(p, p1, p2, p3, p4)); - JsonConstraint jc = new JsonConstraint(); + JsonOption jc = new JsonOption(); jc.setStrictMode(StrictMode.STRICT); expect.addOption(jc); Assert.assertEquals(expect, actual); @@ -2357,7 +2403,7 @@ public void generate_jsonObject6_generateSucceed() { Expression actual = factory.generate(); FunctionCall expect = new FunctionCall("json_object", Collections.emptyList()); - JsonConstraint jc = new JsonConstraint(); + JsonOption jc = new JsonOption(); jc.setUniqueMode(UniqueMode.WITH_UNIQUE_KEYS); expect.addOption(jc); Assert.assertEquals(expect, actual); @@ -2373,6 +2419,7 @@ public void generate_jsonQuery_generateSucceed() { p2.addOption(new ConstExpression("format json")); FunctionParam p3 = new ExpressionParam(new ConstExpression("'aaa'")); FunctionCall expect = new FunctionCall("json_query", Arrays.asList(p2, p3)); + expect.addOption(new JsonOption()); Assert.assertEquals(expect, actual); } @@ -2389,18 +2436,19 @@ public void generate_jsonQuery1_generateSucceed() { FunctionParam p3 = new ExpressionParam(new ConstExpression("'aaa'")); FunctionCall expect = new FunctionCall("json_query", Arrays.asList(p2, p3)); expect.addOption(new GeneralDataType("json", null)); - expect.addOption(new ConstExpression("truncate")); - expect.addOption(new ConstExpression("pretty")); - expect.addOption(new ConstExpression("ascii")); - JsonConstraint jsonConstraint = new JsonConstraint(); - jsonConstraint.setScalarsMode(ScalarsMode.ALLOW_SCALARS); - jsonConstraint.setWrapperMode(WrapperMode.WITH_CONDITIONAL_WRAPPER); - expect.addOption(jsonConstraint); + + JsonOption jsonOpt = new JsonOption(); + jsonOpt.setTruncate(true); + jsonOpt.setPretty(true); + jsonOpt.setAscii(true); + jsonOpt.setScalarsMode(ScalarsMode.ALLOW_SCALARS); + jsonOpt.setWrapperMode(WrapperMode.WITH_CONDITIONAL_WRAPPER); + expect.addOption(jsonOpt); JsonOnOption jsonOnOption = new JsonOnOption(); jsonOnOption.setOnMismatches(Collections.singletonList(new OnMismatch(new ConstExpression("dot"), null))); jsonOnOption.setOnEmpty(new ConstExpression("empty")); jsonOnOption.setOnError(new NullExpression()); - expect.addOption(jsonOnOption); + jsonOpt.setOnOption(jsonOnOption); Assert.assertEquals(expect, actual); } @@ -2426,12 +2474,12 @@ public void generate_jsonQuery2_generateSucceed() { p2.addOption(new ConstExpression("format json")); FunctionParam p3 = new ExpressionParam(new ConstExpression("'aaa'")); FunctionCall expect = new FunctionCall("json_query", Arrays.asList(p2, p3)); - JsonConstraint jsonConstraint = new JsonConstraint(); - jsonConstraint.setWrapperMode(WrapperMode.valueOf(s.replace(" ", "_"))); - expect.addOption(jsonConstraint); + JsonOption jsonOpt = new JsonOption(); + jsonOpt.setWrapperMode(WrapperMode.valueOf(s.replace(" ", "_"))); JsonOnOption jsonOnOption = new JsonOnOption(); jsonOnOption.setOnMismatches(Collections.singletonList(new OnMismatch(new NullExpression(), null))); - expect.addOption(jsonOnOption); + jsonOpt.setOnOption(jsonOnOption); + expect.addOption(jsonOpt); Assert.assertEquals(expect, actual); } } @@ -2445,8 +2493,10 @@ public void generate_jsonMergepatch_generateSucceed() { FunctionParam p2 = new ExpressionParam(new ConstExpression("'a'")); FunctionParam p3 = new ExpressionParam(new ConstExpression("2")); FunctionCall expect = new FunctionCall("json_mergepatch", Arrays.asList(p2, p3)); - expect.addOption(new ConstExpression("PRETTY")); - expect.addOption(new ConstExpression("ASCII")); + JsonOption jsonOpt = new JsonOption(); + jsonOpt.setAscii(true); + jsonOpt.setPretty(true); + expect.addOption(jsonOpt); Assert.assertEquals(expect, actual); } @@ -2460,11 +2510,13 @@ public void generate_jsonMergepatch1_generateSucceed() { FunctionParam p3 = new ExpressionParam(new ConstExpression("2")); FunctionCall expect = new FunctionCall("json_mergepatch", Arrays.asList(p2, p3)); expect.addOption(new GeneralDataType("json", null)); - expect.addOption(new ConstExpression("PRETTY")); - expect.addOption(new ConstExpression("ASCII")); + JsonOption jsonOpt = new JsonOption(); + jsonOpt.setPretty(true); + jsonOpt.setAscii(true); JsonOnOption jsonOnOption = new JsonOnOption(); jsonOnOption.setOnError(new NullExpression()); - expect.addOption(jsonOnOption); + jsonOpt.setOnOption(jsonOnOption); + expect.addOption(jsonOpt); Assert.assertEquals(expect, actual); } @@ -2482,12 +2534,14 @@ public void generate_jsonMergepatch2_generateSucceed() { CharacterType characterType = new CharacterType("varchar2", new BigDecimal("12")); characterType.setBinary(true); expect.addOption(characterType); - expect.addOption(new ConstExpression("TRUNCATE")); - expect.addOption(new ConstExpression("PRETTY")); - expect.addOption(new ConstExpression("ASCII")); + JsonOption jsonOpt = new JsonOption(); + jsonOpt.setTruncate(true); + jsonOpt.setAscii(true); + jsonOpt.setPretty(true); JsonOnOption jsonOnOption = new JsonOnOption(); jsonOnOption.setOnError(new ConstExpression("error_p")); - expect.addOption(jsonOnOption); + jsonOpt.setOnOption(jsonOnOption); + expect.addOption(jsonOpt); Assert.assertEquals(expect, actual); } @@ -2526,8 +2580,10 @@ public void generate_jsonArray2_generateSucceed() { FunctionCall expect = new FunctionCall("json_array", Arrays.asList(p2, p3)); JsonOnOption jsonOnOption = new JsonOnOption(); jsonOnOption.setOnNull(new ConstExpression("absent")); - expect.addOption(jsonOnOption); + JsonOption jsonOpt = new JsonOption(); + jsonOpt.setOnOption(jsonOnOption); expect.addOption(new GeneralDataType("json", null)); + expect.addOption(jsonOpt); Assert.assertEquals(expect, actual); } @@ -2543,9 +2599,9 @@ public void generate_jsonArray3_generateSucceed() { FunctionCall expect = new FunctionCall("json_array", Arrays.asList(p2, p3)); JsonOnOption jsonOnOption = new JsonOnOption(); jsonOnOption.setOnNull(new NullExpression()); - expect.addOption(jsonOnOption); - JsonConstraint jc = new JsonConstraint(); + JsonOption jc = new JsonOption(); jc.setStrictMode(StrictMode.STRICT); + jc.setOnOption(jsonOnOption); expect.addOption(jc); Assert.assertEquals(expect, actual); } @@ -2560,6 +2616,7 @@ public void generate_jsonValue_generateSucceed() { p2.addOption(new ConstExpression("format json")); FunctionParam p3 = new ExpressionParam(new ConstExpression("'aaa'")); FunctionCall expect = new FunctionCall("json_value", Arrays.asList(p2, p3)); + expect.addOption(new JsonOption()); Assert.assertEquals(expect, actual); } @@ -2575,15 +2632,17 @@ public void generate_jsonValue1_generateSucceed() { FunctionParam p3 = new ExpressionParam(new ConstExpression("'aaa'")); FunctionCall expect = new FunctionCall("json_value", Arrays.asList(p2, p3)); expect.addOption(new CharacterType("nchar", new BigDecimal("2"))); - expect.addOption(new ConstExpression("truncate")); - expect.addOption(new ConstExpression("ascii")); + JsonOption jsonOpt = new JsonOption(); + jsonOpt.setTruncate(true); + jsonOpt.setAscii(true); JsonOnOption jsonOnOption = new JsonOnOption(); jsonOnOption.setOnError(new ConstExpression("1")); jsonOnOption.setOnEmpty(new NullExpression()); jsonOnOption.setOnMismatches(Arrays.asList(new OnMismatch(new ConstExpression("ignore"), Collections.singletonList("MISSING DATA")), new OnMismatch(new NullExpression(), Collections.emptyList()))); - expect.addOption(jsonOnOption); + jsonOpt.setOnOption(jsonOnOption); + expect.addOption(jsonOpt); Assert.assertEquals(expect, actual); } @@ -2601,15 +2660,18 @@ public void generate_jsonValue2_generateSucceed() { CharacterType characterType = new CharacterType("varchar2", new BigDecimal("2")); characterType.setBinary(true); expect.addOption(characterType); - expect.addOption(new ConstExpression("truncate")); - expect.addOption(new ConstExpression("ascii")); + + JsonOption jsonOpt = new JsonOption(); + jsonOpt.setTruncate(true); + jsonOpt.setAscii(true); JsonOnOption jsonOnOption = new JsonOnOption(); jsonOnOption.setOnEmpty(new ConstExpression("1")); jsonOnOption.setOnError(new ConstExpression("error_p")); jsonOnOption.setOnMismatches(Arrays.asList(new OnMismatch(new ConstExpression("ignore"), Collections.singletonList("MISSING DATA")), new OnMismatch(new NullExpression(), Collections.emptyList()))); - expect.addOption(jsonOnOption); + jsonOpt.setOnOption(jsonOnOption); + expect.addOption(jsonOpt); Assert.assertEquals(expect, actual); } @@ -2626,15 +2688,17 @@ public void generate_jsonValue4_generateSucceed() { FunctionCall expect = new FunctionCall("json_value", Arrays.asList(p2, p3)); CharacterType characterType = new CharacterType("char", null); expect.addOption(characterType); - expect.addOption(new ConstExpression("truncate")); - expect.addOption(new ConstExpression("ascii")); + JsonOption jsonOpt = new JsonOption(); + jsonOpt.setTruncate(true); + jsonOpt.setAscii(true); JsonOnOption jsonOnOption = new JsonOnOption(); jsonOnOption.setOnEmpty(new ConstExpression("1")); jsonOnOption.setOnError(new ConstExpression("error_p")); jsonOnOption.setOnMismatches(Arrays.asList(new OnMismatch(new ConstExpression("ignore"), Collections.singletonList("MISSING DATA")), new OnMismatch(new NullExpression(), Collections.emptyList()))); - expect.addOption(jsonOnOption); + jsonOpt.setOnOption(jsonOnOption); + expect.addOption(jsonOpt); Assert.assertEquals(expect, actual); } @@ -2650,15 +2714,17 @@ public void generate_jsonValue5_generateSucceed() { FunctionParam p3 = new ExpressionParam(new ConstExpression("'aaa'")); FunctionCall expect = new FunctionCall("json_value", Arrays.asList(p2, p3)); expect.addOption(new GeneralDataType("raw", null)); - expect.addOption(new ConstExpression("truncate")); - expect.addOption(new ConstExpression("ascii")); + JsonOption jsonOpt = new JsonOption(); + jsonOpt.setTruncate(true); + jsonOpt.setAscii(true); JsonOnOption jsonOnOption = new JsonOnOption(); jsonOnOption.setOnEmpty(new ConstExpression("1")); jsonOnOption.setOnError(new ConstExpression("error_p")); jsonOnOption.setOnMismatches(Arrays.asList(new OnMismatch(new ConstExpression("ignore"), Collections.singletonList("MISSING DATA")), new OnMismatch(new NullExpression(), Collections.emptyList()))); - expect.addOption(jsonOnOption); + jsonOpt.setOnOption(jsonOnOption); + expect.addOption(jsonOpt); Assert.assertEquals(expect, actual); } @@ -2675,15 +2741,17 @@ public void generate_jsonValue3_generateSucceed() { FunctionCall expect = new FunctionCall("json_value", Arrays.asList(p2, p3)); CharacterType characterType = new CharacterType("nvarchar2", null); expect.addOption(characterType); - expect.addOption(new ConstExpression("truncate")); - expect.addOption(new ConstExpression("ascii")); + JsonOption jsonOpt = new JsonOption(); + jsonOpt.setTruncate(true); + jsonOpt.setAscii(true); JsonOnOption jsonOnOption = new JsonOnOption(); jsonOnOption.setOnEmpty(new ConstExpression("1")); jsonOnOption.setOnError(new ConstExpression("error_p")); jsonOnOption.setOnMismatches(Arrays.asList(new OnMismatch(new ConstExpression("ignore"), Collections.singletonList("MISSING DATA")), new OnMismatch(new NullExpression(), Collections.emptyList()))); - expect.addOption(jsonOnOption); + jsonOpt.setOnOption(jsonOnOption); + expect.addOption(jsonOpt); Assert.assertEquals(expect, actual); } @@ -2756,50 +2824,59 @@ public void generate_jsonTable2_generateSucceed() { p1.addOption(new ConstExpression("format json")); FunctionParam p2 = new ExpressionParam(new ConstExpression("'aaa'")); FunctionCall expect = new FunctionCall("json_table", Arrays.asList(p1, p2)); + JsonOption jsonOpt = new JsonOption(); JsonOnOption onOption = new JsonOnOption(); onOption.setOnError(new ConstExpression("error_p")); onOption.setOnEmpty(new NullExpression()); - expect.addOption(onOption); + jsonOpt.setOnOption(onOption); + expect.addOption(jsonOpt); FunctionParam op1 = new ExpressionParam(new ColumnReference(null, null, "\"abcd\"")); op1.addOption(new ConstExpression("FOR ORDINALITY")); expect.addOption(op1); FunctionParam op2 = new ExpressionParam(new ColumnReference(null, null, "col1")); op2.addOption(new NumberType("int", null, null)); - op2.addOption(new ConstExpression("truncate")); + JsonOption jsonOpt1 = new JsonOption(); + jsonOpt1.setAsis(true); + jsonOpt1.setTruncate(true); op2.addOption(new ConstExpression("exists")); op2.addOption(new ConstExpression("123")); onOption = new JsonOnOption(); onOption.setOnEmpty(new BoolValue(true)); - op2.addOption(onOption); + jsonOpt1.setOnOption(onOption); + op2.addOption(jsonOpt1); expect.addOption(op2); FunctionParam op3 = new ExpressionParam(new ColumnReference(null, null, "col2")); op3.addOption(new GeneralDataType("json", null)); - JsonConstraint jc = new JsonConstraint(); + JsonOption jc = new JsonOption(); + jc.setAsis(true); jc.setScalarsMode(ScalarsMode.DISALLOW_SCALARS); jc.setWrapperMode(WrapperMode.WITH_CONDITIONAL_ARRAY_WRAPPER); - op3.addOption(jc); op3.addOption(new ColumnReference(null, null, "col21")); onOption = new JsonOnOption(); onOption.setOnEmpty(new ConstExpression("empty")); - op3.addOption(onOption); + jc.setOnOption(onOption); + op3.addOption(jc); expect.addOption(op3); FunctionParam op4 = new ExpressionParam(new ColumnReference(null, null, "col3")); op4.addOption(new GeneralDataType("blob", null)); op4.addOption(new ConstExpression("format json")); - op4.addOption(new ConstExpression("truncate")); - jc = new JsonConstraint(); + jc = new JsonOption(); + jc.setAsis(true); + jc.setTruncate(true); jc.setScalarsMode(ScalarsMode.ALLOW_SCALARS); jc.setWrapperMode(WrapperMode.WITH_ARRAY_WRAPPER); - op4.addOption(jc); op4.addOption(new ColumnReference(null, null, "col31")); - op4.addOption(onOption); + op4.addOption(jc); + jc.setOnOption(onOption); expect.addOption(op4); FunctionParam op5 = new ExpressionParam(new ColumnReference(null, null, "col4")); op5.addOption(new CharacterType("nchar", new BigDecimal("12"))); - op5.addOption(new ConstExpression("truncate")); + jc = new JsonOption(); + jc.setTruncate(true); + jc.setAsis(true); ColumnReference rc = new ColumnReference(null, null, "col41"); CollectionExpression es = new CollectionExpression(); es.addExpression(new ConstExpression("*")); @@ -2807,7 +2884,8 @@ public void generate_jsonTable2_generateSucceed() { op5.addOption(rc); onOption = new JsonOnOption(); onOption.setOnEmpty(new CompoundExpression(new ConstExpression("3"), null, Operator.SUB)); - op5.addOption(onOption); + jc.setOnOption(onOption); + op5.addOption(jc); expect.addOption(op5); FunctionParam op6 = new ExpressionParam(new ConstExpression("nested path")); op6.addOption(new ConstExpression("123")); @@ -2833,48 +2911,57 @@ public void generate_jsonTable3_generateSucceed() { p1.addOption(new ConstExpression("format json")); FunctionParam p2 = new ExpressionParam(new ConstExpression("'aaa'")); FunctionCall expect = new FunctionCall("json_table", Arrays.asList(p1, p2)); + JsonOption jsonOpt = new JsonOption(); JsonOnOption onOption = new JsonOnOption(); onOption.setOnError(new ConstExpression("error_p")); onOption.setOnEmpty(new CompoundExpression(new ConstExpression("5"), null, Operator.SUB)); - expect.addOption(onOption); + jsonOpt.setOnOption(onOption); + expect.addOption(jsonOpt); FunctionParam op1 = new ExpressionParam(new ColumnReference(null, null, "\"abcd\"")); op1.addOption(new ConstExpression("FOR ORDINALITY")); expect.addOption(op1); FunctionParam op2 = new ExpressionParam(new ColumnReference(null, null, "col1")); op2.addOption(new NumberType("int", null, null)); - op2.addOption(new ConstExpression("truncate")); + JsonOption jsonOpt1 = new JsonOption(); + jsonOpt1.setAsis(true); + jsonOpt1.setTruncate(true); op2.addOption(new ConstExpression("exists")); op2.addOption(new ConstExpression("123")); onOption = new JsonOnOption(); onOption.setOnEmpty(new BoolValue(true)); - op2.addOption(onOption); + jsonOpt1.setOnOption(onOption); + op2.addOption(jsonOpt1); expect.addOption(op2); FunctionParam op3 = new ExpressionParam(new ColumnReference(null, null, "col2")); op3.addOption(new GeneralDataType("json", null)); - JsonConstraint jc = new JsonConstraint(); + JsonOption jc = new JsonOption(); + jc.setAsis(true); jc.setWrapperMode(WrapperMode.WITH_CONDITIONAL_ARRAY_WRAPPER); - op3.addOption(jc); op3.addOption(new ColumnReference(null, null, "col21")); onOption = new JsonOnOption(); onOption.setOnEmpty(new ConstExpression("empty")); - op3.addOption(onOption); + jc.setOnOption(onOption); + op3.addOption(jc); expect.addOption(op3); FunctionParam op4 = new ExpressionParam(new ColumnReference(null, null, "col3")); op4.addOption(new GeneralDataType("blob", null)); op4.addOption(new ConstExpression("format json")); - op4.addOption(new ConstExpression("truncate")); - jc = new JsonConstraint(); + jc = new JsonOption(); + jc.setAsis(true); + jc.setTruncate(true); jc.setScalarsMode(ScalarsMode.ALLOW_SCALARS); - op4.addOption(jc); op4.addOption(new ColumnReference(null, null, "col31")); - op4.addOption(onOption); + op4.addOption(jc); + jc.setOnOption(onOption); expect.addOption(op4); FunctionParam op5 = new ExpressionParam(new ColumnReference(null, null, "col4")); op5.addOption(new CharacterType("nchar", new BigDecimal("12"))); - op5.addOption(new ConstExpression("truncate")); + jc = new JsonOption(); + jc.setTruncate(true); + jc.setAsis(true); ColumnReference rc = new ColumnReference(null, null, "col41"); CollectionExpression es = new CollectionExpression(); es.addExpression(new ConstExpression("*")); @@ -2882,7 +2969,8 @@ public void generate_jsonTable3_generateSucceed() { op5.addOption(rc); onOption = new JsonOnOption(); onOption.setOnEmpty(new CompoundExpression(new ConstExpression("3"), null, Operator.SUB)); - op5.addOption(onOption); + jc.setOnOption(onOption); + op5.addOption(jc); expect.addOption(op5); FunctionParam op6 = new ExpressionParam(new ConstExpression("nested path")); op6.addOption(new ConstExpression("123")); @@ -2900,9 +2988,11 @@ public void generate_jsonEqualExpr_generateSucceed() { ExpressionParam p1 = new ExpressionParam(new ConstExpression("'[1,]'")); ExpressionParam p2 = new ExpressionParam(new ConstExpression("'[1]'")); FunctionCall expect = new FunctionCall("json_equal", Arrays.asList(p1, p2)); + JsonOption jsonOpt = new JsonOption(); JsonOnOption jsonOnOption = new JsonOnOption(); jsonOnOption.setOnError(new BoolValue(false)); - expect.addOption(jsonOnOption); + jsonOpt.setOnOption(jsonOnOption); + expect.addOption(jsonOpt); Assert.assertEquals(expect, actual); } @@ -2930,6 +3020,14 @@ private ExprContext getExprContext(String expr) { return parser.expr(); } + private Single_row_functionContext getSingleRowFunctionContext(String expr) { + OBLexer lexer = new OBLexer(CharStreams.fromString(expr)); + CommonTokenStream tokens = new CommonTokenStream(lexer); + OBParser parser = new OBParser(tokens); + parser.setErrorHandler(new BailErrorStrategy()); + return parser.single_row_function(); + } + private Xml_functionContext getXmlExprContext(String expr) { OBLexer lexer = new OBLexer(CharStreams.fromString(expr)); CommonTokenStream tokens = new CommonTokenStream(lexer); diff --git a/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/OracleInsertFactoryTest.java b/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/OracleInsertFactoryTest.java index 5657ef167b..d7ecc02e1b 100644 --- a/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/OracleInsertFactoryTest.java +++ b/libs/ob-sql-parser/src/test/java/com/oceanbase/tools/sqlparser/adapter/OracleInsertFactoryTest.java @@ -16,10 +16,7 @@ package com.oceanbase.tools.sqlparser.adapter; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; +import java.util.*; import org.antlr.v4.runtime.BailErrorStrategy; import org.antlr.v4.runtime.CharStreams; @@ -83,6 +80,44 @@ public void generate_simpleInsert_succeed() { Assert.assertEquals(actual, expect); } + @Test + public void generate_simpleInsertWithPartition_succeed() { + StatementFactory factory = new OracleInsertFactory( + getInsertContext("insert into a.b partition(col='111', col2=121) alias values(1,default)")); + Insert actual = factory.generate(); + + RelationFactor factor = new RelationFactor("b"); + factor.setSchema("a"); + InsertTable insertTable = new InsertTable(factor); + Map parti = new HashMap<>(); + parti.put("col", new ConstExpression("'111'")); + parti.put("col2", new ConstExpression("121")); + insertTable.setPartitionUsage(new PartitionUsage(PartitionType.PARTITION, parti)); + insertTable.setAlias("alias"); + List> values = new ArrayList<>(); + values.add(Arrays.asList(new ConstExpression("1"), new ConstExpression("default"))); + insertTable.setValues(values); + Insert expect = new Insert(Collections.singletonList(insertTable), null); + Assert.assertEquals(actual, expect); + } + + @Test + public void generate_overwriteInsert_succeed() { + StatementFactory factory = new OracleInsertFactory( + getInsertContext("insert overwrite a.b values(1,default)")); + Insert actual = factory.generate(); + + RelationFactor factor = new RelationFactor("b"); + factor.setSchema("a"); + InsertTable insertTable = new InsertTable(factor); + List> values = new ArrayList<>(); + values.add(Arrays.asList(new ConstExpression("1"), new ConstExpression("default"))); + insertTable.setValues(values); + Insert expect = new Insert(Collections.singletonList(insertTable), null); + expect.setOverwrite(true); + Assert.assertEquals(expect, actual); + } + @Test public void generate_insertSelect_succeed() { StatementFactory factory = new OracleInsertFactory( diff --git a/pom.xml b/pom.xml index 76db54145d..f7d81813cd 100644 --- a/pom.xml +++ b/pom.xml @@ -9,7 +9,7 @@ pom OceanBase Developer Center https://github.com/oceanbase/odc - 4.3.2-SNAPSHOT + 4.3.3-SNAPSHOT server/3rd-party/Libinjection server/odc-test @@ -93,7 +93,7 @@ 4.20.19.ALL 4.10.0 2.10.0 - 1.2.0 + 1.2.2 1.4.0 3.10.0 1.64 @@ -113,13 +113,13 @@ 1.2.19 1.5.4 - 3.3.4 + 3.3.6 4.1.94.Final 1.2.1 2.1.6 - 1.1.6.bp4 + 1.2.0 2.11.0 @@ -150,6 +150,9 @@ 3.9.0 4.11.0 5.13.3.202401111512-r + 1.76 + 1.70 + 1.15 1.9.5 2.7.12 @@ -165,6 +168,10 @@ org.slf4j slf4j-log4j12 + + com.amazonaws + aws-java-sdk-bundle + @@ -193,6 +200,11 @@ groovy-all ${groovy-all.version} + + commons-codec + commons-codec + ${commons-codec.version} + io.micrometer micrometer-core @@ -459,6 +471,12 @@ spring-security-oauth2-client ${spring-security.version} + + + org.springframework.security + spring-security-saml2-service-provider + ${spring-security.version} + org.springframework.security spring-security-crypto @@ -482,7 +500,7 @@ org.bouncycastle bcpkix-jdk15on - ${bcpkix-jdk15on.version} + ${bouncycastle.jdk15.version} org.springframework.integration @@ -499,10 +517,31 @@ spring-cloud-starter-bootstrap ${spring-cloud-starter-bootstrap.version} + + org.bouncycastle + bcprov-jdk18on + ${bouncycastle.jdk18.version} + + + org.bouncycastle + bcpkix-jdk18on + ${bouncycastle.jdk18.version} + + + org.bouncycastle + bcutil-jdk18on + ${bouncycastle.jdk18.version} + org.springframework.cloud spring-cloud-config-server ${spring-cloud-config-server.version} + + + org.eclipse.jgit + org.eclipse.jgit.ssh.apache + + org.springframework.cloud @@ -978,9 +1017,7 @@ org.bouncycastle bcprov-jdk15on - - - 1.70 + ${bouncycastle.jdk15.version} org.hamcrest diff --git a/script/start-job.sh b/script/start-job.sh index 84e7556939..e055e97b6c 100755 --- a/script/start-job.sh +++ b/script/start-job.sh @@ -16,6 +16,9 @@ gc_log_options="-Xloggc:${install_directory}/log/gc.log -XX:+UseGCLogFileRotatio default_heap_options="-XX:MaxRAMPercentage=60.0 -XX:InitialRAMPercentage=60.0" default_gc_options="${gc_basic_options} ${gc_log_options}" default_oom_options="-XX:+ExitOnOutOfMemoryError" +default_agent_main_class_name="com.oceanbase.odc.agent.OdcAgent" +default_spring_boot_loader="org.springframework.boot.loader.PropertiesLauncher" +main_class_caller="-Dloader.main=${default_agent_main_class_name} ${default_spring_boot_loader}" # define some helper functions function usage() { @@ -139,8 +142,8 @@ main() { echo "Starting odc-job..." local cmd="${java_exec} ${remote_debug_options} ${spacev_java_agent_options} ${gc_options} ${heap_options} ${oom_options} - ${extra_options} ${app_options} -jar - ${jar_file} ${app_args}" + ${extra_options} ${app_options} -cp + ${jar_file} ${main_class_caller} ${app_args}" echo "cmd=${cmd}" eval ${cmd} return $? diff --git a/server/3rd-party/Libinjection/pom.xml b/server/3rd-party/Libinjection/pom.xml index ab117e953e..96db6dc34d 100644 --- a/server/3rd-party/Libinjection/pom.xml +++ b/server/3rd-party/Libinjection/pom.xml @@ -8,7 +8,7 @@ com.oceanbase odc-parent - 4.3.2-SNAPSHOT + 4.3.3-SNAPSHOT ../../../pom.xml Libinjection diff --git a/server/integration-test/pom.xml b/server/integration-test/pom.xml index 0c9d27fcfe..a24391df3a 100644 --- a/server/integration-test/pom.xml +++ b/server/integration-test/pom.xml @@ -7,7 +7,7 @@ com.oceanbase odc-parent - 4.3.2-SNAPSHOT + 4.3.3-SNAPSHOT ../../pom.xml integration-test diff --git a/server/integration-test/src/test/java/com/oceanbase/odc/metadb/connection/ConnectionConfigRepositoryTest.java b/server/integration-test/src/test/java/com/oceanbase/odc/metadb/connection/ConnectionConfigRepositoryTest.java index bd24ebff4b..a460b95bd5 100644 --- a/server/integration-test/src/test/java/com/oceanbase/odc/metadb/connection/ConnectionConfigRepositoryTest.java +++ b/server/integration-test/src/test/java/com/oceanbase/odc/metadb/connection/ConnectionConfigRepositoryTest.java @@ -15,6 +15,7 @@ */ package com.oceanbase.odc.metadb.connection; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.Date; @@ -47,6 +48,8 @@ public class ConnectionConfigRepositoryTest extends ServiceTestEnv { private static final String CLUSTER_NAME = "C1"; private static final String TENANT_NAME = "T1"; private static final String USERNAME = "odcTest"; + private static final List INSERT_ORDER_NAME_LIST = Arrays.asList("C", "B", "A"); + private static final List LEXICAL_ORDER_NAME_LIST = Arrays.asList("A", "B", "C"); @Autowired private ConnectionConfigRepository repository; @@ -284,6 +287,41 @@ public void deleteByIds_servalConnsExist_deleteSucceed() { Assert.assertEquals(entities.size(), affectRows); } + @Test + public void findByOrganizationId_getListBySpecificOrganization_success() { + List entities = new ArrayList<>(); + for (String name : INSERT_ORDER_NAME_LIST) { + ConnectionEntity entity = createEntity(ConnectionVisibleScope.ORGANIZATION); + entity.setName(name); + entities.add(entity); + } + List savedEntities = repository.saveAll(entities); + List list = repository.findByOrganizationId( + ORGANIZATION_ID); + Assert.assertEquals(savedEntities.size(), list.size()); + for (int i = 0; i < list.size(); i++) { + Assert.assertEquals(list.get(i).getOrganizationId(), ORGANIZATION_ID); + } + } + + @Test + public void findByOrganizationIdOrderByNameAsc_getSortedListByNameAsc_success() { + List entities = new ArrayList<>(); + for (String name : INSERT_ORDER_NAME_LIST) { + ConnectionEntity entity = createEntity(ConnectionVisibleScope.ORGANIZATION); + entity.setName(name); + entities.add(entity); + } + List savedEntities = repository.saveAll(entities); + List list = repository.findByOrganizationIdOrderByNameAsc( + ORGANIZATION_ID); + Assert.assertEquals(savedEntities.size(), list.size()); + for (int i = 0; i < list.size(); i++) { + Assert.assertEquals(list.get(i).getOrganizationId(), ORGANIZATION_ID); + Assert.assertEquals(list.get(i).getName(), LEXICAL_ORDER_NAME_LIST.get(i)); + } + } + @Test public void findSyncableConnections_HasSyncable_Succeed() { ConnectionEntity c1 = createEntity(ConnectionVisibleScope.ORGANIZATION); diff --git a/server/integration-test/src/test/java/com/oceanbase/odc/metadb/connection/DatabaseRepositoryTest.java b/server/integration-test/src/test/java/com/oceanbase/odc/metadb/connection/DatabaseRepositoryTest.java index 2d34beb4da..3b3e74ac21 100644 --- a/server/integration-test/src/test/java/com/oceanbase/odc/metadb/connection/DatabaseRepositoryTest.java +++ b/server/integration-test/src/test/java/com/oceanbase/odc/metadb/connection/DatabaseRepositoryTest.java @@ -57,9 +57,10 @@ public void setObjectSyncStatusByObjectLastSyncTimeBefore_noObjectMatched_setNot d2.setObjectLastSyncTime(syncTime); d2.setObjectSyncStatus(DBObjectSyncStatus.SYNCED); this.databaseRepository.saveAll(Arrays.asList(d1, d2)); - int affectRows = this.databaseRepository.setObjectSyncStatusByObjectSyncStatusAndObjectLastSyncTimeBefore( - DBObjectSyncStatus.SYNCING, DBObjectSyncStatus.SYNCING, - new Date(System.currentTimeMillis() - 86400 * 2)); + int affectRows = + this.databaseRepository.setObjectSyncStatusByObjectSyncStatusAndObjectLastSyncTimeIsNullOrBefore( + DBObjectSyncStatus.SYNCING, DBObjectSyncStatus.SYNCING, + new Date(System.currentTimeMillis() - 86400 * 2)); Assert.assertEquals(0, affectRows); } @@ -72,7 +73,7 @@ public void setObjectSyncStatusByObjectLastSyncTimeBefore_oneObjectMatched_setSu d2.setObjectLastSyncTime(new Date(System.currentTimeMillis() - 86400)); d2.setObjectSyncStatus(DBObjectSyncStatus.SYNCED); this.databaseRepository.saveAll(Arrays.asList(d1, d2)); - this.databaseRepository.setObjectSyncStatusByObjectSyncStatusAndObjectLastSyncTimeBefore( + this.databaseRepository.setObjectSyncStatusByObjectSyncStatusAndObjectLastSyncTimeIsNullOrBefore( DBObjectSyncStatus.INITIALIZED, DBObjectSyncStatus.SYNCED, new Date(System.currentTimeMillis() - 86400 / 2)); Set actual = this.databaseRepository.findAll().stream() diff --git a/server/integration-test/src/test/java/com/oceanbase/odc/metadb/dbobject/DBObjectRepositoryTest.java b/server/integration-test/src/test/java/com/oceanbase/odc/metadb/dbobject/DBObjectRepositoryTest.java index cad40c1b9b..7c9c6b4c3c 100644 --- a/server/integration-test/src/test/java/com/oceanbase/odc/metadb/dbobject/DBObjectRepositoryTest.java +++ b/server/integration-test/src/test/java/com/oceanbase/odc/metadb/dbobject/DBObjectRepositoryTest.java @@ -35,6 +35,11 @@ */ public class DBObjectRepositoryTest extends ServiceTestEnv { + private static final Long DATABASE_ID = 1L; + private static final DBObjectType DATASOURCE_TYPE = DBObjectType.SCHEMA; + private static final List INSERT_ORDER_NAME_LIST = Arrays.asList("C", "B", "A"); + private static final List LEXICAL_ORDER_NAME_LIST = Arrays.asList("A", "B", "C"); + @Autowired private DBObjectRepository dbObjectRepository; @@ -73,4 +78,44 @@ public void test_batchCreate() { Assert.assertEquals(entities.size(), saved.size()); } + @Test + public void findByDatabaseIdAndType_getListByDatabaseIdAndType_success() { + List entities = new ArrayList<>(); + for (String name : INSERT_ORDER_NAME_LIST) { + DBObjectEntity entity = TestRandom.nextObject(DBObjectEntity.class); + entity.setName(name); + entity.setDatabaseId(DATABASE_ID); + entity.setType(DATASOURCE_TYPE); + entities.add(entity); + } + List savedEntities = dbObjectRepository.saveAll(entities); + List list = dbObjectRepository.findByDatabaseIdAndType( + DATABASE_ID, DATASOURCE_TYPE); + Assert.assertEquals(savedEntities.size(), list.size()); + for (int i = 0; i < list.size(); i++) { + Assert.assertEquals(list.get(i).getDatabaseId(), DATABASE_ID); + Assert.assertEquals(list.get(i).getType(), DATASOURCE_TYPE); + } + } + + @Test + public void findByDatabaseIdAndTypeOrderByNameAsc_getListSortedByName_success() { + List entities = new ArrayList<>(); + for (String name : INSERT_ORDER_NAME_LIST) { + DBObjectEntity entity = TestRandom.nextObject(DBObjectEntity.class); + entity.setName(name); + entity.setDatabaseId(DATABASE_ID); + entity.setType(DATASOURCE_TYPE); + entities.add(entity); + } + List savedEntities = dbObjectRepository.saveAll(entities); + List list = dbObjectRepository.findByDatabaseIdAndTypeOrderByNameAsc( + DATABASE_ID, DATASOURCE_TYPE); + Assert.assertEquals(savedEntities.size(), list.size()); + for (int i = 0; i < list.size(); i++) { + Assert.assertEquals(list.get(i).getDatabaseId(), DATABASE_ID); + Assert.assertEquals(list.get(i).getType(), DATASOURCE_TYPE); + Assert.assertEquals(list.get(i).getName(), LEXICAL_ORDER_NAME_LIST.get(i)); + } + } } diff --git a/server/integration-test/src/test/java/com/oceanbase/odc/metadb/schedule/ScheduleTaskRepositoryTest.java b/server/integration-test/src/test/java/com/oceanbase/odc/metadb/schedule/ScheduleTaskRepositoryTest.java index bef16984da..67f59a2142 100644 --- a/server/integration-test/src/test/java/com/oceanbase/odc/metadb/schedule/ScheduleTaskRepositoryTest.java +++ b/server/integration-test/src/test/java/com/oceanbase/odc/metadb/schedule/ScheduleTaskRepositoryTest.java @@ -17,7 +17,6 @@ import java.util.Collections; import java.util.Date; -import java.util.LinkedList; import java.util.List; import java.util.Optional; @@ -77,17 +76,6 @@ public void updateExecutor() { Assert.equals(executor, byId.get().getExecutor()); } - @Test - public void listByScheduleIdAndStatus() { - taskRepository.deleteAll(); - createScheduleTask(); - List statuses = new LinkedList<>(); - statuses.add(TaskStatus.RUNNING); - statuses.add(TaskStatus.PREPARING); - List byJobNameAndStatus = taskRepository.findByJobNameAndStatusIn("1", statuses); - Assert.equals(byJobNameAndStatus.size(), 1); - } - @Test public void findByIdIn() { ScheduleTaskEntity scheduleTask = createScheduleTask(); diff --git a/server/integration-test/src/test/java/com/oceanbase/odc/metadb/task/JobRepositoryTest.java b/server/integration-test/src/test/java/com/oceanbase/odc/metadb/task/JobRepositoryTest.java index 9df79a3519..fe987d212d 100644 --- a/server/integration-test/src/test/java/com/oceanbase/odc/metadb/task/JobRepositoryTest.java +++ b/server/integration-test/src/test/java/com/oceanbase/odc/metadb/task/JobRepositoryTest.java @@ -21,8 +21,8 @@ import org.springframework.beans.factory.annotation.Autowired; import com.oceanbase.odc.ServiceTestEnv; +import com.oceanbase.odc.service.task.base.databasechange.DatabaseChangeTask; import com.oceanbase.odc.service.task.enums.JobStatus; -import com.oceanbase.odc.service.task.runtime.DatabaseChangeTask; import com.oceanbase.odc.service.task.util.JobDateUtils; /** diff --git a/server/integration-test/src/test/java/com/oceanbase/odc/service/collaboration/ProjectServiceTest.java b/server/integration-test/src/test/java/com/oceanbase/odc/service/collaboration/ProjectServiceTest.java index 763a1c470a..75319561a1 100644 --- a/server/integration-test/src/test/java/com/oceanbase/odc/service/collaboration/ProjectServiceTest.java +++ b/server/integration-test/src/test/java/com/oceanbase/odc/service/collaboration/ProjectServiceTest.java @@ -15,8 +15,14 @@ */ package com.oceanbase.odc.service.collaboration; +import static org.mockito.ArgumentMatchers.anyCollection; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.Mockito.doNothing; + import java.util.ArrayList; +import java.util.Arrays; import java.util.Date; +import java.util.HashSet; import java.util.List; import java.util.UUID; @@ -36,6 +42,7 @@ import com.oceanbase.odc.core.shared.constant.ResourceRoleName; import com.oceanbase.odc.core.shared.constant.ResourceType; import com.oceanbase.odc.core.shared.exception.BadRequestException; +import com.oceanbase.odc.core.shared.exception.UnsupportedException; import com.oceanbase.odc.metadb.collaboration.ProjectEntity; import com.oceanbase.odc.metadb.collaboration.ProjectRepository; import com.oceanbase.odc.metadb.connection.DatabaseEntity; @@ -49,12 +56,14 @@ import com.oceanbase.odc.service.collaboration.project.model.Project.ProjectMember; import com.oceanbase.odc.service.collaboration.project.model.QueryProjectParams; import com.oceanbase.odc.service.collaboration.project.model.SetArchivedReq; +import com.oceanbase.odc.service.iam.ProjectPermissionValidator; import com.oceanbase.odc.service.iam.ResourceRoleService; import com.oceanbase.odc.service.iam.UserOrganizationService; import com.oceanbase.odc.service.iam.UserService; import com.oceanbase.odc.service.iam.auth.AuthenticationFacade; import com.oceanbase.odc.service.iam.model.User; import com.oceanbase.odc.service.iam.model.UserResourceRole; +import com.oceanbase.odc.service.schedule.ScheduleService; import com.oceanbase.odc.test.tool.TestRandom; /** @@ -89,6 +98,12 @@ public class ProjectServiceTest extends ServiceTestEnv { @MockBean private ResourceRoleRepository resourceRoleRepository; + @MockBean + private ProjectPermissionValidator projectPermissionValidator; + + @MockBean + private ScheduleService scheduleService; + @Before public void setUp() { Mockito.when(userService.nullSafeGet(Mockito.anyLong())).thenReturn(getUserEntity()); @@ -119,7 +134,8 @@ public void testCreateProject_Success() { public void testGetProject_Success() { Project saved = projectService.create(getProject()); Mockito.when( - resourceRoleService.listByResourceTypeAndId(Mockito.eq(ResourceType.ODC_PROJECT), Mockito.anyLong())) + resourceRoleService.listByResourceTypeAndResourceId(Mockito.eq(ResourceType.ODC_PROJECT), + Mockito.anyLong())) .thenReturn(listUserResourceRole(saved.getId())); Project actual = projectService.detail(saved.getId()); Assert.assertNotNull(actual); @@ -130,7 +146,8 @@ public void testGetProject_syncTimeIsNull() { Date syncTime = new Date(); Project saved = projectService.create(getProject()); Mockito.when( - resourceRoleService.listByResourceTypeAndId(Mockito.eq(ResourceType.ODC_PROJECT), Mockito.anyLong())) + resourceRoleService.listByResourceTypeAndResourceId(Mockito.eq(ResourceType.ODC_PROJECT), + Mockito.anyLong())) .thenReturn(listUserResourceRole(saved.getId())); createDatabase(saved.getId(), null); createDatabase(saved.getId(), syncTime); @@ -143,7 +160,8 @@ public void testGetProject_syncTimeNotNull() { Date syncTime = new Date(); Project saved = projectService.create(getProject()); Mockito.when( - resourceRoleService.listByResourceTypeAndId(Mockito.eq(ResourceType.ODC_PROJECT), Mockito.anyLong())) + resourceRoleService.listByResourceTypeAndResourceId(Mockito.eq(ResourceType.ODC_PROJECT), + Mockito.anyLong())) .thenReturn(listUserResourceRole(saved.getId())); createDatabase(saved.getId(), syncTime); createDatabase(saved.getId(), DateUtils.addDays(syncTime, 1)); @@ -164,8 +182,11 @@ public void testUpdateProject_Success() { public void testArchiveProject_Archived() throws InterruptedException { Project saved = projectService.create(getProject()); Mockito.when(resourceRoleService.saveAll(Mockito.any())).thenReturn(listUserResourceRole(saved.getId())); + doNothing().when(projectPermissionValidator).checkProjectRole(anyCollection(), anyList()); SetArchivedReq req = new SetArchivedReq(); req.setArchived(true); + Mockito.when(scheduleService.listUnfinishedSchedulesByProjectId(Pageable.unpaged(), saved.getId())) + .thenReturn(Page.empty()); Project archived = projectService.setArchived(saved.getId(), req); Assert.assertTrue(archived.getArchived()); } @@ -179,6 +200,27 @@ public void testArchiveProject_NotArchived() throws InterruptedException { projectService.setArchived(saved.getId(), req); } + @Test + public void testDeleteProject_ArchivedProject_Success() throws InterruptedException { + Project saved = projectService.create(getProject()); + Mockito.when(resourceRoleService.saveAll(Mockito.any())).thenReturn(listUserResourceRole(saved.getId())); + doNothing().when(projectPermissionValidator).checkProjectRole(anyCollection(), anyList()); + SetArchivedReq req = new SetArchivedReq(); + req.setArchived(true); + Mockito.when(scheduleService.listUnfinishedSchedulesByProjectId(Pageable.unpaged(), saved.getId())) + .thenReturn(Page.empty()); + projectService.setArchived(saved.getId(), req); + Assert.assertTrue(projectService.batchDelete(new HashSet<>(Arrays.asList(saved.getId())))); + } + + @Test(expected = UnsupportedException.class) + public void testDeleteProject_NotArchivedProject_Fail() { + Project saved = projectService.create(getProject()); + Mockito.when(resourceRoleService.saveAll(Mockito.any())).thenReturn(listUserResourceRole(saved.getId())); + doNothing().when(projectPermissionValidator).checkProjectRole(anyCollection(), anyList()); + projectService.batchDelete(new HashSet<>(Arrays.asList(saved.getId()))); + } + @Test public void test_listBasicInfoForApply() { ProjectEntity entity = projectRepository.save(getProjectEntity()); diff --git a/server/integration-test/src/test/java/com/oceanbase/odc/service/db/DBIdentitiesServiceTest.java b/server/integration-test/src/test/java/com/oceanbase/odc/service/db/DBIdentitiesServiceTest.java new file mode 100644 index 0000000000..84842489d0 --- /dev/null +++ b/server/integration-test/src/test/java/com/oceanbase/odc/service/db/DBIdentitiesServiceTest.java @@ -0,0 +1,126 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.db; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.jdbc.core.JdbcOperations; + +import com.oceanbase.odc.ServiceTestEnv; +import com.oceanbase.odc.TestConnectionUtil; +import com.oceanbase.odc.core.session.ConnectionSession; +import com.oceanbase.odc.core.session.ConnectionSessionConstants; +import com.oceanbase.odc.core.session.ConnectionSessionUtil; +import com.oceanbase.odc.core.shared.constant.ConnectType; +import com.oceanbase.odc.service.db.model.ObjectIdentity; +import com.oceanbase.odc.service.db.model.SchemaIdentities; +import com.oceanbase.tools.dbbrowser.model.DBObjectType; + +/** + * @description: + * @author: zijia.cj + * @date: 2024/9/10 09:20 + * @since: 4.3.3 + */ +public class DBIdentitiesServiceTest extends ServiceTestEnv { + @Autowired + private DBIdentitiesService dbIdentitiesService; + + private final static List TABLE_NAME_LIST = Arrays.asList("test_table_1", "test_table_2", "test_table_3"); + private final static List VIEW_NAME_LIST = Arrays.asList("test_view_1", "test_view_2", "test_view_3"); + + @Before + public void setUp() { + ConnectionSession session = TestConnectionUtil.getTestConnectionSession(ConnectType.OB_MYSQL); + StringBuilder tableBuilder = new StringBuilder(); + for (String name : TABLE_NAME_LIST) { + tableBuilder.append(String.format("CREATE TABLE IF NOT EXISTS %s (\n" + + " id BIGINT NOT NULL AUTO_INCREMENT,\n" + + " PRIMARY key (`id`)\n" + + ");", name)); + } + session.getSyncJdbcExecutor(ConnectionSessionConstants.CONSOLE_DS_KEY).execute(tableBuilder.toString()); + StringBuilder viewBuilder = new StringBuilder(); + for (int i = 0; i < VIEW_NAME_LIST.size(); i++) { + viewBuilder.append(String.format("CREATE VIEW %s AS SELECT * FROM %s;", VIEW_NAME_LIST.get(i), + TABLE_NAME_LIST.get(i))); + } + session.getSyncJdbcExecutor(ConnectionSessionConstants.CONSOLE_DS_KEY).execute(viewBuilder.toString()); + } + + @After + public void clear() { + ConnectionSession connectionSession = TestConnectionUtil.getTestConnectionSession(ConnectType.OB_MYSQL); + JdbcOperations jdbcOperations = + connectionSession.getSyncJdbcExecutor(ConnectionSessionConstants.CONSOLE_DS_KEY); + StringBuilder viewBuilder = new StringBuilder(); + for (String name : VIEW_NAME_LIST) { + viewBuilder.append(String.format("drop view %s;", name)); + } + jdbcOperations.execute(viewBuilder.toString()); + StringBuilder tableBuilder = new StringBuilder(); + for (String name : TABLE_NAME_LIST) { + tableBuilder.append(String.format("drop table %s;", name)); + } + jdbcOperations.execute(tableBuilder.toString()); + } + + @Test + public void list_whenTypesIsEmpty_getEmptyList() { + ConnectionSession connectionSession = TestConnectionUtil.getTestConnectionSession(ConnectType.OB_MYSQL); + List result = dbIdentitiesService.list(connectionSession, Collections.emptyList()); + Assert.assertTrue(result.isEmpty()); + } + + @Test + public void list_whenTypesOnlyContainsTable_success() { + ConnectionSession connectionSession = TestConnectionUtil.getTestConnectionSession(ConnectType.OB_MYSQL); + List types = Collections.singletonList(DBObjectType.TABLE); + List result = dbIdentitiesService.list(connectionSession, types); + Assert.assertNotNull(result); + Map> schema2Identities = result.stream().collect( + Collectors.toMap(SchemaIdentities::getSchemaName, SchemaIdentities::getIdentities)); + List nameList = schema2Identities.get( + ConnectionSessionUtil.getCurrentSchema(connectionSession)).stream() + .filter(x -> DBObjectType.TABLE.equals(x.getType())).map(ObjectIdentity::getName).collect( + Collectors.toList()); + Assert.assertTrue(nameList.containsAll(TABLE_NAME_LIST)); + } + + @Test + public void list_whenTypesOnlyContainsView_success() { + ConnectionSession connectionSession = TestConnectionUtil.getTestConnectionSession(ConnectType.OB_MYSQL); + List types = Collections.singletonList(DBObjectType.VIEW); + List result = dbIdentitiesService.list(connectionSession, types); + Assert.assertNotNull(result); + Map> schema2Identities = result.stream().collect( + Collectors.toMap(SchemaIdentities::getSchemaName, SchemaIdentities::getIdentities)); + List nameList = schema2Identities.get( + ConnectionSessionUtil.getCurrentSchema(connectionSession)).stream() + .filter(x -> DBObjectType.VIEW.equals(x.getType())).map(ObjectIdentity::getName).collect( + Collectors.toList()); + Assert.assertTrue(nameList.containsAll(VIEW_NAME_LIST)); + } +} diff --git a/server/integration-test/src/test/java/com/oceanbase/odc/service/db/DBPLModifyHelperTest.java b/server/integration-test/src/test/java/com/oceanbase/odc/service/db/DBPLModifyHelperTest.java new file mode 100644 index 0000000000..3e2994daf1 --- /dev/null +++ b/server/integration-test/src/test/java/com/oceanbase/odc/service/db/DBPLModifyHelperTest.java @@ -0,0 +1,286 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.db; + +import static org.hibernate.validator.internal.util.Contracts.assertNotNull; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThrows; + +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.mockito.Mockito; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.jdbc.BadSqlGrammarException; + +import com.oceanbase.odc.ServiceTestEnv; +import com.oceanbase.odc.TestConnectionUtil; +import com.oceanbase.odc.common.util.VersionUtils; +import com.oceanbase.odc.core.session.ConnectionSession; +import com.oceanbase.odc.core.session.ConnectionSessionConstants; +import com.oceanbase.odc.core.session.ConnectionSessionUtil; +import com.oceanbase.odc.core.shared.constant.ConnectType; +import com.oceanbase.odc.core.shared.constant.DialectType; +import com.oceanbase.odc.core.shared.exception.BadRequestException; +import com.oceanbase.odc.core.sql.execute.SyncJdbcExecutor; +import com.oceanbase.odc.core.sql.split.SqlCommentProcessor; +import com.oceanbase.odc.service.db.model.EditPLReq; +import com.oceanbase.odc.service.db.model.EditPLResp; +import com.oceanbase.odc.service.session.ConnectSessionService; +import com.oceanbase.tools.dbbrowser.model.DBObjectType; + +/** + * @description: + * @author: zijia.cj + * @date: 2024/10/18 10:53 + * @since: 4.3.3 + */ +public class DBPLModifyHelperTest extends ServiceTestEnv { + private static final String ODC_TEST_PROCEDURE = "ODC_TEST_PROCEDURE"; + private static final String ODC_TEST_FUNCTION = "ODC_TEST_FUNCTION"; + private static final String ODC_TEST_TRIGGER = "ODC_TEST_TRIGGER"; + private static final String ODC_TEST_TRIGGER_TABLE = "ODC_TEST_TRIGGER_TABLE"; + + @MockBean + private ConnectSessionService sessionService; + + @Autowired + private DBPLModifyHelper dbplModifyHelper; + + + @BeforeClass + public static void setUp() throws Exception { + ConnectionSession testConnectionSession = TestConnectionUtil.getTestConnectionSession(ConnectType.OB_MYSQL); + SyncJdbcExecutor syncJdbcExecutor = testConnectionSession.getSyncJdbcExecutor( + ConnectionSessionConstants.CONSOLE_DS_KEY); + // prepare test environment for procedure + executeDropPLSql(DBObjectType.PROCEDURE, ODC_TEST_PROCEDURE); + executeDropPLSql(DBObjectType.PROCEDURE, DBPLModifyHelper.ODC_TEMPORARY_PROCEDURE); + String createTestProcedure = "CREATE PROCEDURE " + ODC_TEST_PROCEDURE + "(IN num INT, OUT square INT)\n" + + "BEGIN\n" + + " SET square = num * num;\n" + + "END"; + syncJdbcExecutor.execute(createTestProcedure); + // prepare test environment for function + executeDropPLSql(DBObjectType.FUNCTION, ODC_TEST_FUNCTION); + executeDropPLSql(DBObjectType.FUNCTION, DBPLModifyHelper.ODC_TEMPORARY_FUNCTION); + String createTestFunction = "CREATE FUNCTION " + ODC_TEST_FUNCTION + "(num INT) \n" + + "RETURNS INT\n" + + "BEGIN\n" + + " RETURN num * num * num;\n" + + "END"; + syncJdbcExecutor.execute(createTestFunction); + // prepare test environment for trigger + executeDropPLSql(DBObjectType.TRIGGER, ODC_TEST_TRIGGER); + executeDropPLSql(DBObjectType.TRIGGER, DBPLModifyHelper.ODC_TEMPORARY_TRIGGER); + executeDropPLSql(DBObjectType.TABLE, ODC_TEST_TRIGGER_TABLE); + String createTestTable = "CREATE TABLE " + ODC_TEST_TRIGGER_TABLE + "(\n" + + " id INT AUTO_INCREMENT PRIMARY KEY,\n" + + " value INT\n" + + ");"; + syncJdbcExecutor.execute(createTestTable); + String createTestTrigger = "CREATE TRIGGER " + ODC_TEST_TRIGGER + "\n" + + "BEFORE INSERT ON " + ODC_TEST_TRIGGER_TABLE + "\n" + + "FOR EACH ROW\n" + + "BEGIN\n" + + " SET NEW.value = NEW.value * 1;\n" + + "END"; + syncJdbcExecutor.execute(createTestTrigger); + } + + + @AfterClass + public static void clear() throws Exception { + // clear test environment for procedure + executeDropPLSql(DBObjectType.PROCEDURE, ODC_TEST_PROCEDURE); + executeDropPLSql(DBObjectType.PROCEDURE, DBPLModifyHelper.ODC_TEMPORARY_PROCEDURE); + // clear test environment for function + executeDropPLSql(DBObjectType.FUNCTION, ODC_TEST_FUNCTION); + executeDropPLSql(DBObjectType.FUNCTION, DBPLModifyHelper.ODC_TEMPORARY_FUNCTION); + // clear test environment for trigger + executeDropPLSql(DBObjectType.TRIGGER, ODC_TEST_TRIGGER); + executeDropPLSql(DBObjectType.TRIGGER, DBPLModifyHelper.ODC_TEMPORARY_TRIGGER); + executeDropPLSql(DBObjectType.TABLE, ODC_TEST_TRIGGER_TABLE); + } + + @Before + public void mock() { + ConnectionSession testConnectionSession = TestConnectionUtil.getTestConnectionSession(ConnectType.OB_MYSQL); + SqlCommentProcessor sqlCommentProcessor = new SqlCommentProcessor(DialectType.OB_MYSQL, true, true, true); + testConnectionSession.setAttribute(ConnectionSessionConstants.SQL_COMMENT_PROCESSOR_KEY, sqlCommentProcessor); + String sessionId = testConnectionSession.getId(); + Mockito.when(sessionService.nullSafeGet(sessionId, true)).thenReturn(testConnectionSession); + Mockito.when(sessionService.nullSafeGet(sessionId)).thenReturn(testConnectionSession); + } + + @Test + public void editProcedureForOBMysql_normal_successResult() throws Exception { + ConnectionSession testConnectionSession = TestConnectionUtil.getTestConnectionSession(ConnectType.OB_MYSQL); + String editTestProcedure = "CREATE PROCEDURE " + ODC_TEST_PROCEDURE + "(IN num1 INT, OUT square1 INT)\n" + + "BEGIN\n" + + " SET square1 = num1 * num1;\n" + + "END"; + EditPLResp editPLResp = executeEditPL(testConnectionSession, editTestProcedure, ODC_TEST_PROCEDURE, + DBObjectType.PROCEDURE); + assertNotNull(editPLResp); + assertFalse(editPLResp.isApprovalRequired()); + assertNull(editPLResp.getErrorMessage()); + } + + + @Test + public void editProcedureForOBMysql_odcTempProcedureHaveExisted_failResult() throws Exception { + ConnectionSession testConnectionSession = TestConnectionUtil.getTestConnectionSession(ConnectType.OB_MYSQL); + SyncJdbcExecutor syncJdbcExecutor = testConnectionSession.getSyncJdbcExecutor( + ConnectionSessionConstants.CONSOLE_DS_KEY); + String createTempProcedure = "CREATE PROCEDURE " + DBPLModifyHelper.ODC_TEMPORARY_PROCEDURE + + "(IN num INT, OUT square INT)\n" + + "BEGIN\n" + + " SET square = num * num;\n" + + "END"; + syncJdbcExecutor.execute(createTempProcedure); + String editTestProcedure = "CREATE PROCEDURE " + ODC_TEST_PROCEDURE + "(IN num1 INT, OUT square1 INT)\n" + + "BEGIN\n" + + " SET square1 = num1 * num1;\n" + + "END"; + EditPLResp editPLResp = executeEditPL(testConnectionSession, editTestProcedure, ODC_TEST_PROCEDURE, + DBObjectType.PROCEDURE); + assertNotNull(editPLResp); + assertFalse(editPLResp.isApprovalRequired()); + assertNotNull(editPLResp.getErrorMessage()); + executeDropPLSql(DBObjectType.PROCEDURE, DBPLModifyHelper.ODC_TEMPORARY_PROCEDURE); + } + + @Test + public void editFunctionForOBMysql_normal_successResult() throws Exception { + ConnectionSession testConnectionSession = TestConnectionUtil.getTestConnectionSession(ConnectType.OB_MYSQL); + String editTestFunction = "CREATE FUNCTION " + ODC_TEST_FUNCTION + "(num1 INT) \n" + + "RETURNS INT\n" + + "BEGIN\n" + + " RETURN num1 * num1 * num1;\n" + + "END"; + EditPLResp editPLResp = executeEditPL(testConnectionSession, editTestFunction, ODC_TEST_FUNCTION, + DBObjectType.FUNCTION); + assertNotNull(editPLResp); + assertFalse(editPLResp.isApprovalRequired()); + assertNull(editPLResp.getErrorMessage()); + } + + @Test + public void editFunctionForOBMysql_odcTempFunctionHaveExisted_failResultResult() throws Exception { + ConnectionSession testConnectionSession = TestConnectionUtil.getTestConnectionSession(ConnectType.OB_MYSQL); + SyncJdbcExecutor syncJdbcExecutor = testConnectionSession.getSyncJdbcExecutor( + ConnectionSessionConstants.CONSOLE_DS_KEY); + String createODCTempFunction = + "CREATE FUNCTION " + DBPLModifyHelper.ODC_TEMPORARY_FUNCTION + "(num INT) \n" + + "RETURNS INT\n" + + "BEGIN\n" + + " RETURN num * num * num;\n" + + "END"; + syncJdbcExecutor.execute(createODCTempFunction); + String editTestFunction = "CREATE FUNCTION " + ODC_TEST_FUNCTION + "(num1 INT) \n" + + "RETURNS INT\n" + + "BEGIN\n" + + " RETURN num1 * num1 * num1;\n" + + "END"; + EditPLResp editPLResp = executeEditPL(testConnectionSession, editTestFunction, ODC_TEST_FUNCTION, + DBObjectType.FUNCTION); + assertNotNull(editPLResp); + assertFalse(editPLResp.isApprovalRequired()); + assertNotNull(editPLResp.getErrorMessage()); + executeDropPLSql(DBObjectType.FUNCTION, DBPLModifyHelper.ODC_TEMPORARY_FUNCTION); + } + + @Test + public void editTriggerForOBMysql_normal_successResult() throws Exception { + ConnectionSession testConnectionSession = TestConnectionUtil.getTestConnectionSession(ConnectType.OB_MYSQL); + String editTestTrigger = "CREATE TRIGGER " + ODC_TEST_TRIGGER + "\n" + + "BEFORE INSERT ON " + ODC_TEST_TRIGGER_TABLE + "\n" + + "FOR EACH ROW\n" + + "BEGIN\n" + + " SET NEW.value = NEW.value * 2;\n" + + "END"; + if (VersionUtils.isLessThan(ConnectionSessionUtil.getVersion(testConnectionSession), + DBPLModifyHelper.OB_VERSION_SUPPORT_MULTIPLE_SAME_TRIGGERS)) { + assertThrows(BadRequestException.class, + () -> executeEditPL(testConnectionSession, editTestTrigger, ODC_TEST_TRIGGER, + DBObjectType.TRIGGER)); + } else { + EditPLResp editPLResp = executeEditPL(testConnectionSession, editTestTrigger, ODC_TEST_TRIGGER, + DBObjectType.TRIGGER); + assertNotNull(editPLResp); + assertFalse(editPLResp.isApprovalRequired()); + assertNull(editPLResp.getErrorMessage()); + } + } + + @Test + public void editTriggerForOBMysql_odcTempTriggerHaveExisted_failResult() throws Exception { + ConnectionSession testConnectionSession = TestConnectionUtil.getTestConnectionSession(ConnectType.OB_MYSQL); + SyncJdbcExecutor syncJdbcExecutor = testConnectionSession.getSyncJdbcExecutor( + ConnectionSessionConstants.CONSOLE_DS_KEY); + String createODCTempTrigger = + "CREATE TRIGGER " + DBPLModifyHelper.ODC_TEMPORARY_TRIGGER + "\n" + + "BEFORE UPDATE ON " + ODC_TEST_TRIGGER_TABLE + "\n" + + "FOR EACH ROW\n" + + "BEGIN\n" + + " SET NEW.value = NEW.value * 2;\n" + + "END"; + syncJdbcExecutor.execute(createODCTempTrigger); + String editTestTrigger = + "CREATE TRIGGER " + ODC_TEST_TRIGGER + "\n" + + "BEFORE INSERT ON " + ODC_TEST_TRIGGER_TABLE + "\n" + + "FOR EACH ROW\n" + + "BEGIN\n" + + " SET NEW.value = NEW.value * 2;\n" + + "END"; + + if (VersionUtils.isLessThan(ConnectionSessionUtil.getVersion(testConnectionSession), + DBPLModifyHelper.OB_VERSION_SUPPORT_MULTIPLE_SAME_TRIGGERS)) { + assertThrows(BadRequestException.class, + () -> executeEditPL(testConnectionSession, editTestTrigger, ODC_TEST_TRIGGER, + DBObjectType.TRIGGER)); + } else { + assertThrows(BadSqlGrammarException.class, + () -> executeEditPL(testConnectionSession, editTestTrigger, ODC_TEST_TRIGGER, + DBObjectType.TRIGGER)); + } + executeDropPLSql(DBObjectType.TRIGGER, DBPLModifyHelper.ODC_TEMPORARY_TRIGGER); + executeDropPLSql(DBObjectType.TABLE, ODC_TEST_TRIGGER_TABLE); + } + + + private EditPLResp executeEditPL(ConnectionSession testConnectionSession, String editTestPL, String plName, + DBObjectType plType) + throws Exception { + EditPLReq editPLReq = new EditPLReq(); + editPLReq.setSql(editTestPL); + editPLReq.setObjectType(plType); + editPLReq.setObjectName(plName); + return dbplModifyHelper.editPL(testConnectionSession.getId(), editPLReq); + } + + private static void executeDropPLSql(DBObjectType plType, String plName) { + ConnectionSession testConnectionSession = TestConnectionUtil.getTestConnectionSession(ConnectType.OB_MYSQL); + SyncJdbcExecutor syncJdbcExecutor = testConnectionSession.getSyncJdbcExecutor( + ConnectionSessionConstants.CONSOLE_DS_KEY); + String dropTestProcedure = "DROP " + plType + " IF EXISTS " + plName; + syncJdbcExecutor.execute(dropTestProcedure); + } +} diff --git a/server/integration-test/src/test/java/com/oceanbase/odc/service/db/DBTableServiceTest.java b/server/integration-test/src/test/java/com/oceanbase/odc/service/db/DBTableServiceTest.java index 83c9da0c63..f0df673b27 100644 --- a/server/integration-test/src/test/java/com/oceanbase/odc/service/db/DBTableServiceTest.java +++ b/server/integration-test/src/test/java/com/oceanbase/odc/service/db/DBTableServiceTest.java @@ -25,6 +25,7 @@ import com.oceanbase.odc.core.session.ConnectionSession; import com.oceanbase.odc.core.shared.constant.ConnectType; import com.oceanbase.odc.core.shared.exception.NotFoundException; +import com.oceanbase.tools.dbbrowser.model.DBObjectType; import com.oceanbase.tools.dbbrowser.model.DBTable; /** @@ -47,7 +48,7 @@ public void getTable_not_exist_OB_MYSQL() { ConnectionSession session = TestConnectionUtil.getTestConnectionSession(ConnectType.OB_MYSQL); - DBTable table = dbTableService.getTable(session, "abc", "abc"); + DBTable table = dbTableService.getTable(session, "abc", "abc", DBObjectType.TABLE); } @Test @@ -57,6 +58,6 @@ public void getTable_not_exist_OB_ORACLE() { ConnectionSession session = TestConnectionUtil.getTestConnectionSession(ConnectType.OB_ORACLE); - DBTable table = dbTableService.getTable(session, "abc", "abc"); + DBTable table = dbTableService.getTable(session, "abc", "abc", DBObjectType.TABLE); } } diff --git a/server/integration-test/src/test/java/com/oceanbase/odc/service/db/TableServiceTest.java b/server/integration-test/src/test/java/com/oceanbase/odc/service/db/TableServiceTest.java new file mode 100644 index 0000000000..23ee2c0949 --- /dev/null +++ b/server/integration-test/src/test/java/com/oceanbase/odc/service/db/TableServiceTest.java @@ -0,0 +1,272 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.db; + +import static org.mockito.ArgumentMatchers.eq; + +import java.sql.SQLException; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.mockito.Mockito; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.jdbc.core.JdbcOperations; + +import com.oceanbase.odc.ServiceTestEnv; +import com.oceanbase.odc.TestConnectionUtil; +import com.oceanbase.odc.core.session.ConnectionSession; +import com.oceanbase.odc.core.session.ConnectionSessionConstants; +import com.oceanbase.odc.core.session.ConnectionSessionUtil; +import com.oceanbase.odc.core.shared.constant.ConnectType; +import com.oceanbase.odc.core.shared.constant.OrganizationType; +import com.oceanbase.odc.core.sql.execute.SyncJdbcExecutor; +import com.oceanbase.odc.metadb.dbobject.DBObjectEntity; +import com.oceanbase.odc.metadb.dbobject.DBObjectRepository; +import com.oceanbase.odc.service.connection.database.DatabaseService; +import com.oceanbase.odc.service.connection.database.model.Database; +import com.oceanbase.odc.service.connection.model.ConnectionConfig; +import com.oceanbase.odc.service.connection.table.TableService; +import com.oceanbase.odc.service.connection.table.model.QueryTableParams; +import com.oceanbase.odc.service.connection.table.model.Table; +import com.oceanbase.odc.service.iam.auth.AuthenticationFacade; +import com.oceanbase.odc.service.iam.model.User; +import com.oceanbase.odc.service.permission.DBResourcePermissionHelper; +import com.oceanbase.tools.dbbrowser.model.DBObjectType; + +/** + * @description: + * @author: zijia.cj + * @date: 2024/9/9 14:50 + * @since: 4.3.3 + */ +public class TableServiceTest extends ServiceTestEnv { + @Autowired + private TableService tableService; + @MockBean + private DBObjectRepository dbObjectRepository; + @MockBean + private AuthenticationFacade authenticationFacade; + @MockBean + private DatabaseService databaseService; + @MockBean + private DBResourcePermissionHelper dbResourcePermissionHelper; + + private static final String OB_MYSQL_TABLE_CREATE_TEMPLATE = "CREATE TABLE IF NOT EXISTS %s (\n" + + " ID BIGINT NOT NULL AUTO_INCREMENT,\n" + + " PRIMARY KEY (`ID`)\n" + + ")"; + + private static final String OB_ORACLE_TABLE_CREATE_TEMPLATE = "CREATE TABLE %s (\n" + + " ID NUMBER PRIMARY KEY\n" + + ")"; + private static final String TABLE_DROP_TEMPLATE = "DROP TABLE %s"; + private static final String VIEW_CREATE_TEMPLATE = "CREATE VIEW %s AS SELECT 1 AS dummy FROM dual"; + private static final String VIEW_DROP_TEMPLATE = "DROP VIEW %s"; + private final static List TABLE_NAME_LIST = Arrays.asList("TEST_TABLE_1", "TEST_TABLE_2", "TEST_TABLE_3"); + private final static List VIEW_NAME_LIST = Arrays.asList("TEST_VIEW_10", "TEST_VIEW_20", "TEST_VIEW_30"); + + private final static Long DATABASE_ID = 1L; + + private static final Long USER_ID = 1L; + + @BeforeClass + public static void setUp() { + clear(); + createTablesOrViewsByConnectType(ConnectType.OB_MYSQL, TABLE_NAME_LIST, OB_MYSQL_TABLE_CREATE_TEMPLATE); + createTablesOrViewsByConnectType(ConnectType.OB_ORACLE, TABLE_NAME_LIST, OB_ORACLE_TABLE_CREATE_TEMPLATE); + createTablesOrViewsByConnectType(ConnectType.MYSQL, TABLE_NAME_LIST, OB_MYSQL_TABLE_CREATE_TEMPLATE); + createTablesOrViewsByConnectType(ConnectType.ORACLE, TABLE_NAME_LIST, OB_ORACLE_TABLE_CREATE_TEMPLATE); + createTablesOrViewsByConnectType(ConnectType.OB_MYSQL, VIEW_NAME_LIST, VIEW_CREATE_TEMPLATE); + createTablesOrViewsByConnectType(ConnectType.OB_ORACLE, VIEW_NAME_LIST, VIEW_CREATE_TEMPLATE); + createTablesOrViewsByConnectType(ConnectType.MYSQL, VIEW_NAME_LIST, VIEW_CREATE_TEMPLATE); + createTablesOrViewsByConnectType(ConnectType.ORACLE, VIEW_NAME_LIST, VIEW_CREATE_TEMPLATE); + } + + @AfterClass + public static void clear() { + dropTablesOrViewsByConnectTypes(ConnectType.OB_MYSQL, VIEW_NAME_LIST, VIEW_DROP_TEMPLATE); + dropTablesOrViewsByConnectTypes(ConnectType.OB_ORACLE, VIEW_NAME_LIST, VIEW_DROP_TEMPLATE); + dropTablesOrViewsByConnectTypes(ConnectType.MYSQL, VIEW_NAME_LIST, VIEW_DROP_TEMPLATE); + dropTablesOrViewsByConnectTypes(ConnectType.ORACLE, VIEW_NAME_LIST, VIEW_DROP_TEMPLATE); + dropTablesOrViewsByConnectTypes(ConnectType.OB_MYSQL, TABLE_NAME_LIST, TABLE_DROP_TEMPLATE); + dropTablesOrViewsByConnectTypes(ConnectType.OB_ORACLE, TABLE_NAME_LIST, TABLE_DROP_TEMPLATE); + dropTablesOrViewsByConnectTypes(ConnectType.MYSQL, TABLE_NAME_LIST, TABLE_DROP_TEMPLATE); + dropTablesOrViewsByConnectTypes(ConnectType.ORACLE, TABLE_NAME_LIST, TABLE_DROP_TEMPLATE); + } + + @Test + public void list_whenConnectionTypeIsOBMysqlAndTypesIsEmpty_getEmptyList() + throws SQLException, InterruptedException { + QueryTableParams params = QueryTableParams.builder() + .databaseId(DATABASE_ID) + .types(Collections.emptyList()) + .includePermittedAction(false) + .build(); + List list = tableService.list(params); + Assert.assertTrue(list.isEmpty()); + } + + @Test + public void list_whenConnectionTypeIsOBMysqlAndOrganizationTypeIsIndividual_succeed() + throws SQLException, InterruptedException { + testByConnectionTypeInIndividualSpace(ConnectType.OB_MYSQL); + } + + @Test + public void list_whenConnectionTypeIsOBMysqlAndOrganizationTypeIsTeam_succeed() + throws SQLException, InterruptedException { + testByConnectionTypeInTeamSpace(ConnectType.OB_MYSQL); + } + + @Test + public void list_whenConnectionTypeIsMysqlAndOrganizationTypeIsIndividual_succeed() + throws SQLException, InterruptedException { + testByConnectionTypeInIndividualSpace(ConnectType.MYSQL); + } + + @Test + public void list_whenConnectionTypeIsMysqlAndOrganizationTypeIsTeam_succeed() + throws SQLException, InterruptedException { + testByConnectionTypeInTeamSpace(ConnectType.MYSQL); + } + + @Test + public void list_whenConnectionTypeIsOBOracleAndOrganizationTypeIsIndividual_succeed() + throws SQLException, InterruptedException { + testByConnectionTypeInIndividualSpace(ConnectType.OB_ORACLE); + } + + @Test + public void list_whenConnectionTypeIsOBOracleAndOrganizationTypeIsTeam_succeed() + throws SQLException, InterruptedException { + testByConnectionTypeInTeamSpace(ConnectType.OB_ORACLE); + } + + @Test + public void list_whenConnectionTypeIsOracleAndOrganizationTypeIsIndividual_succeed() + throws SQLException, InterruptedException { + testByConnectionTypeInIndividualSpace(ConnectType.ORACLE); + } + + @Test + public void list_whenConnectionTypeIsOracleAndOrganizationTypeIsTeam_succeed() + throws SQLException, InterruptedException { + testByConnectionTypeInTeamSpace(ConnectType.ORACLE); + } + + private List getTableEntities(DBObjectType dbObjectType, List names) { + return names.stream().map(name -> { + DBObjectEntity entity = new DBObjectEntity(); + entity.setName(name); + entity.setType(dbObjectType); + entity.setDatabaseId(DATABASE_ID); + return entity; + }).collect(java.util.stream.Collectors.toList()); + + } + + private User getIndivisualUser() { + User user = User.of(USER_ID); + user.setOrganizationType(OrganizationType.INDIVIDUAL); + return user; + } + + private User getTeamUser() { + User user = User.of(USER_ID); + user.setOrganizationType(OrganizationType.TEAM); + return user; + } + + private Database getDatabaseByConnectType(ConnectType connectType) { + Database database = new Database(); + database.setId(DATABASE_ID); + ConnectionSession session = TestConnectionUtil.getTestConnectionSession(connectType); + ConnectionConfig connectionConfig = (ConnectionConfig) ConnectionSessionUtil.getConnectionConfig(session); + String currentSchema = ConnectionSessionUtil.getCurrentSchema(session); + database.setName(currentSchema); + database.setDataSource(connectionConfig); + return database; + } + + private static void createTablesOrViewsByConnectType(ConnectType connectType, List tableNames, + String format) { + ConnectionSession session = TestConnectionUtil.getTestConnectionSession(connectType); + SyncJdbcExecutor syncJdbcExecutor = session.getSyncJdbcExecutor(ConnectionSessionConstants.CONSOLE_DS_KEY); + for (String name : tableNames) { + syncJdbcExecutor.execute(String.format(format, name)); + } + } + + private static void dropTablesOrViewsByConnectTypes(ConnectType connectType, List tableNames, + String format) { + ConnectionSession connectionSession = TestConnectionUtil.getTestConnectionSession(connectType); + JdbcOperations jdbcOperations = + connectionSession.getSyncJdbcExecutor(ConnectionSessionConstants.CONSOLE_DS_KEY); + for (String name : tableNames) { + try { + jdbcOperations.execute(String.format(format, name)); + } catch (Exception e) { + // ignore + } + } + } + + private void testByConnectionTypeInIndividualSpace(ConnectType connectType) + throws SQLException, InterruptedException { + Mockito.when(authenticationFacade.currentUser()).thenReturn(getIndivisualUser()); + testByConnectionType(connectType); + } + + private void testByConnectionTypeInTeamSpace(ConnectType connectType) throws SQLException, InterruptedException { + Mockito.when(authenticationFacade.currentUser()).thenReturn(getTeamUser()); + Mockito.when( + dbObjectRepository.findByDatabaseIdAndTypeOrderByNameAsc(Mockito.anyLong(), eq(DBObjectType.TABLE))) + .thenReturn(getTableEntities(DBObjectType.TABLE, TABLE_NAME_LIST)); + Mockito.when(dbObjectRepository.findByDatabaseIdAndTypeOrderByNameAsc(Mockito.anyLong(), eq(DBObjectType.VIEW))) + .thenReturn(getTableEntities(DBObjectType.VIEW, VIEW_NAME_LIST)); + Mockito.when(dbObjectRepository.findByDatabaseIdAndTypeOrderByNameAsc(Mockito.anyLong(), + eq(DBObjectType.EXTERNAL_TABLE))) + .thenReturn(Arrays.asList()); + testByConnectionType(connectType); + } + + private void testByConnectionType(ConnectType connectType) throws SQLException, InterruptedException { + Mockito.when(databaseService.detail(Mockito.any())).thenReturn(getDatabaseByConnectType(connectType)); + QueryTableParams params = QueryTableParams.builder() + .databaseId(DATABASE_ID) + .types(Arrays.asList(DBObjectType.TABLE, DBObjectType.VIEW, DBObjectType.EXTERNAL_TABLE)) + .includePermittedAction(false) + .build(); + List
list = tableService.list(params); + Assert.assertFalse(list.isEmpty()); + List tableList = list.stream().filter(table -> table.getType() == DBObjectType.TABLE).map( + Table::getName).map(String::toUpperCase).collect( + Collectors.toList()); + List viewList = list.stream().filter(table -> table.getType() == DBObjectType.VIEW).map(Table::getName) + .map(String::toUpperCase).collect( + Collectors.toList()); + Assert.assertTrue(tableList.containsAll(TABLE_NAME_LIST)); + Assert.assertTrue(viewList.containsAll(VIEW_NAME_LIST)); + } + +} diff --git a/server/integration-test/src/test/java/com/oceanbase/odc/service/flow/FlowInstanceServiceTest.java b/server/integration-test/src/test/java/com/oceanbase/odc/service/flow/FlowInstanceServiceTest.java index 4c6bf80d5d..8f82c4c474 100644 --- a/server/integration-test/src/test/java/com/oceanbase/odc/service/flow/FlowInstanceServiceTest.java +++ b/server/integration-test/src/test/java/com/oceanbase/odc/service/flow/FlowInstanceServiceTest.java @@ -54,12 +54,14 @@ import com.oceanbase.odc.ServiceTestEnv; import com.oceanbase.odc.common.event.EventPublisher; import com.oceanbase.odc.common.json.JsonUtils; +import com.oceanbase.odc.core.authority.SecurityManager; import com.oceanbase.odc.core.shared.constant.ConnectType; import com.oceanbase.odc.core.shared.constant.ConnectionVisibleScope; import com.oceanbase.odc.core.shared.constant.FlowStatus; import com.oceanbase.odc.core.shared.constant.ResourceRoleName; import com.oceanbase.odc.core.shared.constant.TaskErrorStrategy; import com.oceanbase.odc.core.shared.constant.TaskType; +import com.oceanbase.odc.core.shared.exception.AccessDeniedException; import com.oceanbase.odc.core.shared.exception.NotFoundException; import com.oceanbase.odc.core.shared.exception.OverLimitException; import com.oceanbase.odc.metadb.flow.FlowInstanceEntity; @@ -76,6 +78,7 @@ import com.oceanbase.odc.metadb.task.TaskRepository; import com.oceanbase.odc.plugin.task.api.datatransfer.model.DataTransferConfig; import com.oceanbase.odc.plugin.task.api.datatransfer.model.DataTransferObject; +import com.oceanbase.odc.service.collaboration.project.model.Project; import com.oceanbase.odc.service.connection.ConnectionService; import com.oceanbase.odc.service.connection.database.DatabaseService; import com.oceanbase.odc.service.connection.database.model.Database; @@ -179,6 +182,8 @@ public class FlowInstanceServiceTest extends ServiceTestEnv { private UserTaskInstanceCandidateRepository userTaskInstanceCandidateRepository; @MockBean private DBResourcePermissionHelper permissionHelper; + @Autowired + private SecurityManager securityManager; @Before public void setUp() { @@ -206,6 +211,9 @@ public void setUp() { UserEntity user = new UserEntity(); user.setEnabled(true); when(userService.nullSafeGet(anyLong())).thenReturn(user); + when(userService.getCurrentUserJoinedProjectIds()).thenReturn(Collections.singleton(1L)); + when(databaseService.listDatabasesByIds(Mockito.anyCollection())) + .thenReturn(Collections.singletonList(getDatabase())); } @Test @@ -400,9 +408,8 @@ public void detail_unauthorized_expThrown() { FlowInstanceEntity entity = optional.get(); entity.setCreatorId(-1); flowInstanceRepository.saveAndFlush(entity); - - thrown.expectMessage(String.format("ODC_FLOW_INSTANCE not found by id=%d", flowInstance.getId())); - thrown.expect(NotFoundException.class); + securityManager.login(null, null); + thrown.expect(AccessDeniedException.class); flowInstanceService.detail(flowInstance.getId()); } @@ -624,9 +631,16 @@ private Database getDatabase() { ConnectionConfig connectionConfig = new ConnectionConfig(); connectionConfig.setId(1L); database.setDataSource(connectionConfig); + database.setProject(getProject()); return database; } + private Project getProject() { + Project project = new Project(); + project.setId(1L); + return project; + } + private List getApprovalNodes() { List nodes = new ArrayList<>(); ApprovalNodeConfig first = new ApprovalNodeConfig(); diff --git a/server/integration-test/src/test/java/com/oceanbase/odc/service/flow/FlowResponseMapperFactoryTest.java b/server/integration-test/src/test/java/com/oceanbase/odc/service/flow/FlowResponseMapperFactoryTest.java index f44e2e90c9..7f2b058682 100644 --- a/server/integration-test/src/test/java/com/oceanbase/odc/service/flow/FlowResponseMapperFactoryTest.java +++ b/server/integration-test/src/test/java/com/oceanbase/odc/service/flow/FlowResponseMapperFactoryTest.java @@ -52,6 +52,7 @@ import com.oceanbase.odc.metadb.iam.UserEntity; import com.oceanbase.odc.metadb.task.TaskEntity; import com.oceanbase.odc.metadb.task.TaskRepository; +import com.oceanbase.odc.service.collaboration.project.model.Project; import com.oceanbase.odc.service.connection.database.DatabaseService; import com.oceanbase.odc.service.connection.database.model.Database; import com.oceanbase.odc.service.connection.model.ConnectionConfig; @@ -151,6 +152,7 @@ public void generateFlowInstanceDetailResp_entityInput_returnDesp() { id -> Collections.singletonList(TestRandom.nextObject(Date.class))) .getCandidatesByFlowInstanceId( id -> Collections.singleton(TestRandom.nextObject(UserEntity.class))) + .getProjectById(id -> TestRandom.nextObject(Project.class)) .build(); FlowInstanceDetailResp detailResp = mapper.map(instanceEntity); diff --git a/server/integration-test/src/test/java/com/oceanbase/odc/service/iam/ResourceRoleServiceTest.java b/server/integration-test/src/test/java/com/oceanbase/odc/service/iam/ResourceRoleServiceTest.java index 329ada996c..ca735cae75 100644 --- a/server/integration-test/src/test/java/com/oceanbase/odc/service/iam/ResourceRoleServiceTest.java +++ b/server/integration-test/src/test/java/com/oceanbase/odc/service/iam/ResourceRoleServiceTest.java @@ -81,7 +81,7 @@ public void testListByResourceId_Success() { .thenReturn(Optional.of(getResourceRoleEntity().get(0))); Mockito.when(authenticationFacade.currentOrganizationId()).thenReturn(1L); resourceRoleService.saveAll(Arrays.asList(getProjectOwner())); - int actual = resourceRoleService.listByResourceTypeAndId(ResourceType.ODC_PROJECT, 1L).size(); + int actual = resourceRoleService.listByResourceTypeAndResourceId(ResourceType.ODC_PROJECT, 1L).size(); Assert.assertEquals(1, actual); } diff --git a/server/integration-test/src/test/java/com/oceanbase/odc/service/session/ConnectConsoleServiceTest.java b/server/integration-test/src/test/java/com/oceanbase/odc/service/session/ConnectConsoleServiceTest.java index d0ec0ce5db..024eb52437 100644 --- a/server/integration-test/src/test/java/com/oceanbase/odc/service/session/ConnectConsoleServiceTest.java +++ b/server/integration-test/src/test/java/com/oceanbase/odc/service/session/ConnectConsoleServiceTest.java @@ -97,15 +97,6 @@ public void getAsyncResult_killSessionSql_successResult() throws Exception { Assert.assertFalse(resultList.isEmpty()); } - @Test - public void killSession_directLink() { - String sql = "kill session 12345"; - injectAsyncJdbcExecutor(JdbcGeneralResult.successResult(SqlTuple.newTuple(sql))); - List jdbcGeneralResults = defaultConnectSessionManage.executeKillSession( - sessionService.nullSafeGet(sessionid), Collections.singletonList(SqlTuple.newTuple(sql)), sql); - Assert.assertFalse(jdbcGeneralResults.isEmpty()); - } - @Test public void getAsyncResult_wrongKillSessionSql_failedResult() throws Exception { String sql = "kill session /*"; diff --git a/server/odc-service/src/test/java/com/oceanbase/odc/service/db/util/OBMysqlCallFunctionCallBackTest.java b/server/integration-test/src/test/java/com/oceanbase/odc/service/session/OBMysqlCallFunctionCallBackTest.java similarity index 67% rename from server/odc-service/src/test/java/com/oceanbase/odc/service/db/util/OBMysqlCallFunctionCallBackTest.java rename to server/integration-test/src/test/java/com/oceanbase/odc/service/session/OBMysqlCallFunctionCallBackTest.java index d89d190ff2..1a1dd301df 100644 --- a/server/odc-service/src/test/java/com/oceanbase/odc/service/db/util/OBMysqlCallFunctionCallBackTest.java +++ b/server/integration-test/src/test/java/com/oceanbase/odc/service/session/OBMysqlCallFunctionCallBackTest.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.db.util; +package com.oceanbase.odc.service.session; import java.io.IOException; import java.io.InputStream; @@ -30,19 +30,33 @@ import org.springframework.jdbc.core.ConnectionCallback; import org.springframework.jdbc.core.JdbcTemplate; +import com.oceanbase.odc.ServiceTestEnv; +import com.oceanbase.odc.TestConnectionUtil; +import com.oceanbase.odc.core.session.ConnectionSession; +import com.oceanbase.odc.core.shared.constant.ConnectType; +import com.oceanbase.odc.core.sql.execute.mapper.DefaultJdbcRowMapper; import com.oceanbase.odc.service.db.model.CallFunctionReq; import com.oceanbase.odc.service.db.model.CallFunctionResp; import com.oceanbase.odc.service.db.model.PLOutParam; +import com.oceanbase.odc.service.db.util.OBMysqlCallFunctionCallBack; import com.oceanbase.odc.test.database.TestDBConfigurations; import com.oceanbase.tools.dbbrowser.model.DBFunction; import com.oceanbase.tools.dbbrowser.model.DBPLParam; import com.oceanbase.tools.dbbrowser.model.DBPLParamMode; -public class OBMysqlCallFunctionCallBackTest { +/** + * @description: + * @author: zijia.cj + * @date: 2024/12/30 15:52 + * @since: 4.3.3 + */ +public class OBMysqlCallFunctionCallBackTest extends ServiceTestEnv { + + public static final String TEST_CASE_1 = "TEST_CASE_1"; + public static final String TEST_CASE_2 = "TEST_CASE_2"; + public static final String TEST_CASE_3 = "TEST_CASE_3"; + public static final String TEST_CASE_4 = "func_test_4"; - public static final String TEST_CASE_1 = "func_test"; - public static final String TEST_CASE_2 = "func_test_1"; - public static final String TEST_CASE_3 = "func_test_2"; @BeforeClass public static void setUp() throws IOException { @@ -58,6 +72,7 @@ public static void clear() { mysql.execute("DROP FUNCTION " + TEST_CASE_1); mysql.execute("DROP FUNCTION " + TEST_CASE_2); mysql.execute("DROP FUNCTION " + TEST_CASE_3); + mysql.execute("DROP FUNCTION " + TEST_CASE_4); } @Test @@ -83,7 +98,10 @@ public void doInConnection_normalFunction_callSucceed() { function.setReturnType("int"); callFunctionReq.setFunction(function); - ConnectionCallback callback = new OBMysqlCallFunctionCallBack(callFunctionReq, -1); + ConnectionSession session = TestConnectionUtil.getTestConnectionSession(ConnectType.OB_MYSQL); + DefaultJdbcRowMapper defaultJdbcRowMapper = new DefaultJdbcRowMapper(session); + ConnectionCallback callback = + new OBMysqlCallFunctionCallBack(callFunctionReq, -1, defaultJdbcRowMapper); JdbcTemplate jdbcTemplate = new JdbcTemplate(TestDBConfigurations.getInstance().getTestOBMysqlConfiguration().getDataSource()); CallFunctionResp actual = jdbcTemplate.execute(callback); @@ -128,7 +146,10 @@ public void doInConnection_nullParamExists_callSucceed() { function.setReturnType("int"); callFunctionReq.setFunction(function); - ConnectionCallback callback = new OBMysqlCallFunctionCallBack(callFunctionReq, -1); + ConnectionSession session = TestConnectionUtil.getTestConnectionSession(ConnectType.OB_MYSQL); + DefaultJdbcRowMapper defaultJdbcRowMapper = new DefaultJdbcRowMapper(session); + ConnectionCallback callback = + new OBMysqlCallFunctionCallBack(callFunctionReq, -1, defaultJdbcRowMapper); JdbcTemplate jdbcTemplate = new JdbcTemplate(TestDBConfigurations.getInstance().getTestOBMysqlConfiguration().getDataSource()); CallFunctionResp actual = jdbcTemplate.execute(callback); @@ -173,7 +194,10 @@ public void doInConnection_emptyParamExists_callSucceed() { function.setReturnType("int"); callFunctionReq.setFunction(function); - ConnectionCallback callback = new OBMysqlCallFunctionCallBack(callFunctionReq, -1); + ConnectionSession session = TestConnectionUtil.getTestConnectionSession(ConnectType.OB_MYSQL); + DefaultJdbcRowMapper defaultJdbcRowMapper = new DefaultJdbcRowMapper(session); + ConnectionCallback callback = + new OBMysqlCallFunctionCallBack(callFunctionReq, -1, defaultJdbcRowMapper); JdbcTemplate jdbcTemplate = new JdbcTemplate(TestDBConfigurations.getInstance().getTestOBMysqlConfiguration().getDataSource()); CallFunctionResp actual = jdbcTemplate.execute(callback); @@ -205,7 +229,10 @@ public void doInConnection_unusualParam_callSucceed() { function.setParams(list); callFunctionReq.setFunction(function); - ConnectionCallback callback = new OBMysqlCallFunctionCallBack(callFunctionReq, -1); + ConnectionSession session = TestConnectionUtil.getTestConnectionSession(ConnectType.OB_MYSQL); + DefaultJdbcRowMapper defaultJdbcRowMapper = new DefaultJdbcRowMapper(session); + ConnectionCallback callback = + new OBMysqlCallFunctionCallBack(callFunctionReq, -1, defaultJdbcRowMapper); JdbcTemplate jdbcTemplate = new JdbcTemplate(TestDBConfigurations.getInstance().getTestOBMysqlConfiguration().getDataSource()); CallFunctionResp actual = jdbcTemplate.execute(callback); @@ -220,6 +247,48 @@ public void doInConnection_unusualParam_callSucceed() { Assert.assertEquals(expect, actual); } + @Test + public void doInConnection_returnTypeIsYear_callSucceed() { + testCallFunctionWhenReturnIsYear("2024", "2024"); + testCallFunctionWhenReturnIsYear("0000", "0000"); + testCallFunctionWhenReturnIsYear("0", "2000"); + testCallFunctionWhenReturnIsYear("1", "2001"); + testCallFunctionWhenReturnIsYear("99", "1999"); + } + + private static void testCallFunctionWhenReturnIsYear(String input, String expectOutput) { + CallFunctionReq callFunctionReq = new CallFunctionReq(); + DBFunction function = new DBFunction(); + function.setFunName(TEST_CASE_4); + List list = new ArrayList<>(); + DBPLParam param = new DBPLParam(); + param.setParamName("p1"); + param.setDefaultValue(input); + param.setDataType("year"); + param.setParamMode(DBPLParamMode.IN); + list.add(param); + function.setParams(list); + function.setReturnType("year"); + callFunctionReq.setFunction(function); + + ConnectionSession session = TestConnectionUtil.getTestConnectionSession(ConnectType.OB_MYSQL); + DefaultJdbcRowMapper defaultJdbcRowMapper = new DefaultJdbcRowMapper(session); + ConnectionCallback callback = + new OBMysqlCallFunctionCallBack(callFunctionReq, -1, defaultJdbcRowMapper); + JdbcTemplate jdbcTemplate = + new JdbcTemplate(TestDBConfigurations.getInstance().getTestOBMysqlConfiguration().getDataSource()); + CallFunctionResp actual = jdbcTemplate.execute(callback); + + CallFunctionResp expect = new CallFunctionResp(); + PLOutParam plOutParam = new PLOutParam(); + plOutParam.setParamName(TEST_CASE_4); + plOutParam.setDataType("year"); + plOutParam.setValue(expectOutput); + expect.setReturnValue(plOutParam); + expect.setOutParams(null); + Assert.assertEquals(expect, actual); + } + private static List getContent() throws IOException { String delimiter = "\\$\\$\\s*"; try (InputStream input = OBMysqlCallFunctionCallBackTest.class.getClassLoader() @@ -230,5 +299,4 @@ private static List getContent() throws IOException { return new ArrayList<>(Arrays.asList(substitutor.replace(new String(buffer)).split(delimiter))); } } - } diff --git a/server/integration-test/src/test/java/com/oceanbase/odc/service/structurecompare/obmysql/DefaultDBStructureComparatorTest.java b/server/integration-test/src/test/java/com/oceanbase/odc/service/structurecompare/obmysql/DefaultDBStructureComparatorTest.java index 1ea3abf8dd..eca8748d1d 100644 --- a/server/integration-test/src/test/java/com/oceanbase/odc/service/structurecompare/obmysql/DefaultDBStructureComparatorTest.java +++ b/server/integration-test/src/test/java/com/oceanbase/odc/service/structurecompare/obmysql/DefaultDBStructureComparatorTest.java @@ -406,4 +406,22 @@ public void test_modifyPartitionType() { excepted.setChangeScript("-- Unsupported operation to modify table partition type\n"); Assert.assertEquals(excepted, actual); } + + @Test + public void test_bitColumnDefaultValue() { + List actuals = results.stream().filter( + item -> item.getDbObjectName().equals("bit_column_default_value")).collect(Collectors.toList()) + .get(0).getSubDBObjectComparisonResult().stream() + .filter(item -> item.getDbObjectType().equals(DBObjectType.COLUMN) + && item.getComparisonResult().equals(ComparisonResult.ONLY_IN_SOURCE)) + .collect( + Collectors.toList()); + + DBObjectComparisonResult expect = + new DBObjectComparisonResult(DBObjectType.COLUMN, "bit_data_1", sourceSchemaName, targetSchemaName); + expect.setComparisonResult(ComparisonResult.ONLY_IN_SOURCE); + expect.setChangeScript("ALTER TABLE `" + targetSchemaName + + "`.`bit_column_default_value` ADD COLUMN `bit_data_1` bit(1) DEFAULT b'0' NOT NULL;\n"); + Assert.assertEquals(expect, actuals.get(0)); + } } diff --git a/server/integration-test/src/test/java/com/oceanbase/odc/service/task/ExecutorIdentifierTest.java b/server/integration-test/src/test/java/com/oceanbase/odc/service/task/ExecutorIdentifierTest.java index aaef88bd1b..4adcc832cd 100644 --- a/server/integration-test/src/test/java/com/oceanbase/odc/service/task/ExecutorIdentifierTest.java +++ b/server/integration-test/src/test/java/com/oceanbase/odc/service/task/ExecutorIdentifierTest.java @@ -45,6 +45,26 @@ public void test_parser_successful() throws JobException { Assert.assertEquals(identifier.toString(), identifierString); } + @Test + public void test_parser_decode() { + + // old version1 + String str2 = "http://odc:8989/default/xxx:xxxx001"; + ExecutorIdentifier identifierOld = ExecutorIdentifierParser.parser(str2); + Assert.assertEquals(identifierOld.getNamespace(), "default"); + Assert.assertEquals(identifierOld.getExecutorName(), "xxx:xxxx001"); + // old version2 + String str3 = "http://odc:8989/xxx:xxxx001"; + ExecutorIdentifier identifierOld2 = ExecutorIdentifierParser.parser(str3); + Assert.assertNull(identifierOld2.getNamespace()); + Assert.assertEquals(identifierOld2.getExecutorName(), "xxx:xxxx001"); + // old version3 + String str4 = "http://odc:8989/"; + ExecutorIdentifier identifierOld3 = ExecutorIdentifierParser.parser(str4); + Assert.assertNull(identifierOld3.getNamespace()); + Assert.assertEquals(identifierOld3.getExecutorName(), ""); + } + @Test public void test_executorDefaultValue_successful() throws JobException { ExecutorIdentifier identifier = DefaultExecutorIdentifier.builder().namespace(null) diff --git a/server/integration-test/src/test/java/com/oceanbase/odc/service/task/JobSchedulerTest.java b/server/integration-test/src/test/java/com/oceanbase/odc/service/task/JobSchedulerTest.java index c34944f72e..e33c3201bc 100644 --- a/server/integration-test/src/test/java/com/oceanbase/odc/service/task/JobSchedulerTest.java +++ b/server/integration-test/src/test/java/com/oceanbase/odc/service/task/JobSchedulerTest.java @@ -25,12 +25,12 @@ import com.oceanbase.odc.common.event.LocalEventPublisher; import com.oceanbase.odc.metadb.task.JobEntity; +import com.oceanbase.odc.service.task.base.databasechange.DatabaseChangeTask; import com.oceanbase.odc.service.task.caller.JobContext; import com.oceanbase.odc.service.task.config.DefaultJobConfiguration; import com.oceanbase.odc.service.task.config.DefaultTaskFrameworkProperties; import com.oceanbase.odc.service.task.dispatch.JobDispatcher; import com.oceanbase.odc.service.task.exception.JobException; -import com.oceanbase.odc.service.task.runtime.DatabaseChangeTask; import com.oceanbase.odc.service.task.schedule.DefaultJobDefinition; import com.oceanbase.odc.service.task.schedule.JobScheduler; import com.oceanbase.odc.service.task.schedule.StdJobScheduler; diff --git a/server/integration-test/src/test/java/com/oceanbase/odc/service/task/NativeK8sClientTest.java b/server/integration-test/src/test/java/com/oceanbase/odc/service/task/NativeK8sClientTest.java index 5382cd6a3a..3649ec3b96 100644 --- a/server/integration-test/src/test/java/com/oceanbase/odc/service/task/NativeK8sClientTest.java +++ b/server/integration-test/src/test/java/com/oceanbase/odc/service/task/NativeK8sClientTest.java @@ -26,12 +26,13 @@ import org.junit.Ignore; import org.junit.Test; -import com.oceanbase.odc.service.task.caller.K8sJobClient; -import com.oceanbase.odc.service.task.caller.K8sJobResponse; -import com.oceanbase.odc.service.task.caller.NativeK8sJobClient; -import com.oceanbase.odc.service.task.caller.PodConfig; import com.oceanbase.odc.service.task.config.K8sProperties; import com.oceanbase.odc.service.task.exception.JobException; +import com.oceanbase.odc.service.task.resource.K8sPodResource; +import com.oceanbase.odc.service.task.resource.K8sResourceContext; +import com.oceanbase.odc.service.task.resource.PodConfig; +import com.oceanbase.odc.service.task.resource.client.K8sJobClient; +import com.oceanbase.odc.service.task.resource.client.NativeK8sJobClient; import com.oceanbase.odc.service.task.schedule.JobIdentity; import com.oceanbase.odc.service.task.util.JobUtils; import com.oceanbase.odc.test.database.TestProperties; @@ -63,10 +64,12 @@ public void test_createJob() throws JobException { String exceptedJobName = JobUtils.generateExecutorName(jobIdentity); List cmd = Arrays.asList("perl", "-Mbignum=bpi", "-wle", "print bpi(2000)"); PodConfig podParam = new PodConfig(); - String generateJobOfName = k8sClient.create("default", exceptedJobName, imageName, cmd, podParam); + K8sResourceContext context = + new K8sResourceContext(podParam, exceptedJobName, imageName, "group", "type", podParam); + K8sPodResource generateJobOfName = k8sClient.create(context); Assert.assertEquals(exceptedJobName, generateJobOfName); - Optional queryJobName = k8sClient.get("default", exceptedJobName); + Optional queryJobName = k8sClient.get("default", exceptedJobName); Assert.assertTrue(queryJobName.isPresent()); Assert.assertEquals(exceptedJobName, queryJobName.get()); diff --git a/server/integration-test/src/test/java/com/oceanbase/odc/service/task/ProcessModeTest.java b/server/integration-test/src/test/java/com/oceanbase/odc/service/task/ProcessModeTest.java index bb260ddfe9..1eba295e26 100644 --- a/server/integration-test/src/test/java/com/oceanbase/odc/service/task/ProcessModeTest.java +++ b/server/integration-test/src/test/java/com/oceanbase/odc/service/task/ProcessModeTest.java @@ -41,6 +41,7 @@ import com.oceanbase.odc.service.flow.task.model.DatabaseChangeParameters; import com.oceanbase.odc.service.objectstorage.cloud.model.ObjectStorageConfiguration; import com.oceanbase.odc.service.plugin.PluginProperties; +import com.oceanbase.odc.service.task.base.databasechange.DatabaseChangeTask; import com.oceanbase.odc.service.task.caller.ExecutorProcessBuilderFactory; import com.oceanbase.odc.service.task.caller.JobContext; import com.oceanbase.odc.service.task.caller.JobEnvironmentFactory; @@ -48,7 +49,6 @@ import com.oceanbase.odc.service.task.constants.JobEnvKeyConstants; import com.oceanbase.odc.service.task.constants.JobParametersKeyConstants; import com.oceanbase.odc.service.task.enums.TaskRunMode; -import com.oceanbase.odc.service.task.runtime.DatabaseChangeTask; import com.oceanbase.odc.service.task.schedule.DefaultJobContextBuilder; import com.oceanbase.odc.service.task.schedule.DefaultJobDefinition; import com.oceanbase.odc.service.task.schedule.JobDefinition; diff --git a/server/integration-test/src/test/java/com/oceanbase/odc/service/task/TaskApplicationTest.java b/server/integration-test/src/test/java/com/oceanbase/odc/service/task/TaskApplicationTest.java deleted file mode 100644 index f5f9e85026..0000000000 --- a/server/integration-test/src/test/java/com/oceanbase/odc/service/task/TaskApplicationTest.java +++ /dev/null @@ -1,120 +0,0 @@ -/* - * Copyright (c) 2023 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.oceanbase.odc.service.task; - -import java.util.HashMap; -import java.util.Map; - -import org.apache.commons.collections4.CollectionUtils; -import org.junit.Assert; -import org.junit.Ignore; -import org.junit.Test; - -import com.oceanbase.odc.TestConnectionUtil; -import com.oceanbase.odc.common.json.JsonUtils; -import com.oceanbase.odc.core.shared.PreConditions; -import com.oceanbase.odc.core.shared.constant.ConnectType; -import com.oceanbase.odc.core.shared.constant.ErrorCodes; -import com.oceanbase.odc.core.shared.constant.TaskErrorStrategy; -import com.oceanbase.odc.core.shared.constant.TaskType; -import com.oceanbase.odc.service.cloud.model.CloudProvider; -import com.oceanbase.odc.service.connection.model.ConnectionConfig; -import com.oceanbase.odc.service.flow.task.model.DatabaseChangeParameters; -import com.oceanbase.odc.service.objectstorage.cloud.model.ObjectStorageConfiguration; -import com.oceanbase.odc.service.task.caller.JobContext; -import com.oceanbase.odc.service.task.caller.JobEnvironmentFactory; -import com.oceanbase.odc.service.task.constants.JobParametersKeyConstants; -import com.oceanbase.odc.service.task.enums.JobStatus; -import com.oceanbase.odc.service.task.enums.TaskRunMode; -import com.oceanbase.odc.service.task.executor.TaskApplication; -import com.oceanbase.odc.service.task.executor.server.ThreadPoolTaskExecutor; -import com.oceanbase.odc.service.task.executor.task.Task; -import com.oceanbase.odc.service.task.runtime.DatabaseChangeTask; -import com.oceanbase.odc.service.task.schedule.DefaultJobContextBuilder; -import com.oceanbase.odc.service.task.schedule.DefaultJobDefinition; -import com.oceanbase.odc.service.task.schedule.JobDefinition; -import com.oceanbase.odc.service.task.schedule.JobIdentity; -import com.oceanbase.odc.service.task.util.JobUtils; - -/** - * @author yaobin - * @date 2023-12-14 - * @since 4.2.4 - */ -@Ignore("manual run this case") -public class TaskApplicationTest extends BaseJobTest { - - @Test - public void test_executeDatabaseChangeTask_run() { - Long exceptedTaskId = System.currentTimeMillis(); - JobIdentity jobIdentity = JobIdentity.of(exceptedTaskId); - - setJobContextInSystemProperty(jobIdentity); - startTaskApplication(); - assertRunningResult(jobIdentity); - } - - - private void setJobContextInSystemProperty(JobIdentity jobIdentity) { - JobDefinition jd = buildJobDefinition(); - JobContext jc = new DefaultJobContextBuilder().build(jobIdentity, jd); - Map envMap = new JobEnvironmentFactory().build(jc, TaskRunMode.PROCESS); - JobUtils.encryptEnvironments(envMap); - envMap.forEach(System::setProperty); - } - - private void assertRunningResult(JobIdentity ji) { - - try { - Thread.sleep(60 * 1000L); - Task task = ThreadPoolTaskExecutor.getInstance().getTask(ji); - Assert.assertSame(JobStatus.DONE, task.getStatus()); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - - } - - - private void startTaskApplication() { - new Thread(() -> new TaskApplication().run(null)).start(); - } - - private JobDefinition buildJobDefinition() { - Long exceptedTaskId = System.currentTimeMillis(); - DatabaseChangeParameters parameters = new DatabaseChangeParameters(); - parameters.setSqlContent(String.format("CREATE TABLE %s (id int(10))", "t_" + exceptedTaskId)); - parameters.setErrorStrategy(TaskErrorStrategy.ABORT.name()); - PreConditions.validArgumentState( - parameters.getSqlContent() != null || CollectionUtils.isNotEmpty(parameters.getSqlObjectIds()), - ErrorCodes.BadArgument, new Object[] {"sql"}, "input sql is empty"); - - ConnectionConfig config = TestConnectionUtil.getTestConnectionConfig(ConnectType.OB_MYSQL); - Map jobData = new HashMap<>(); - jobData.put(JobParametersKeyConstants.META_TASK_PARAMETER_JSON, JsonUtils.toJson(parameters)); - jobData.put(JobParametersKeyConstants.CONNECTION_CONFIG, JobUtils.toJson(config)); - jobData.put(JobParametersKeyConstants.FLOW_INSTANCE_ID, exceptedTaskId + ""); - jobData.put(JobParametersKeyConstants.CURRENT_SCHEMA, config.getDefaultSchema()); - jobData.put(JobParametersKeyConstants.TASK_EXECUTION_TIMEOUT_MILLIS, 30 * 60 * 1000 + ""); - ObjectStorageConfiguration storageConfig = new ObjectStorageConfiguration(); - storageConfig.setCloudProvider(CloudProvider.NONE); - - return DefaultJobDefinition.builder().jobClass(DatabaseChangeTask.class) - .jobType(TaskType.ASYNC.name()) - .jobParameters(jobData) - .build(); - } -} diff --git a/server/odc-service/src/test/resources/db/mysql_function_callback_test.sql b/server/integration-test/src/test/resources/db/mysql_function_callback_test.sql similarity index 54% rename from server/odc-service/src/test/resources/db/mysql_function_callback_test.sql rename to server/integration-test/src/test/resources/db/mysql_function_callback_test.sql index 975f9b4d89..cbe187af19 100644 --- a/server/odc-service/src/test/resources/db/mysql_function_callback_test.sql +++ b/server/integration-test/src/test/resources/db/mysql_function_callback_test.sql @@ -1,4 +1,4 @@ -create function ${const:com.oceanbase.odc.service.db.util.OBMysqlCallFunctionCallBackTest.TEST_CASE_1} ( +create function ${const:com.oceanbase.odc.service.session.OBMysqlCallFunctionCallBackTest.TEST_CASE_1} ( p1 int, p2 int) returns int begin @@ -6,7 +6,7 @@ begin end; $$ -create function ${const:com.oceanbase.odc.service.db.util.OBMysqlCallFunctionCallBackTest.TEST_CASE_2} ( +create function ${const:com.oceanbase.odc.service.session.OBMysqlCallFunctionCallBackTest.TEST_CASE_2} ( p0 int, p1 int, p2 varchar(20)) returns int @@ -21,9 +21,16 @@ begin end; $$ -create function ${const:com.oceanbase.odc.service.db.util.OBMysqlCallFunctionCallBackTest.TEST_CASE_3} ( +create function ${const:com.oceanbase.odc.service.session.OBMysqlCallFunctionCallBackTest.TEST_CASE_3} ( p0 varchar(20)) returns varchar(20) begin return p0; end; +$$ + +create function ${const:com.oceanbase.odc.service.session.OBMysqlCallFunctionCallBackTest.TEST_CASE_4} ( +p1 year) returns year +begin +return p1; +end; $$ \ No newline at end of file diff --git a/server/integration-test/src/test/resources/structurecompare/obmysql/source_drop.sql b/server/integration-test/src/test/resources/structurecompare/obmysql/source_drop.sql index 116e2a2deb..deb24ee8f6 100644 --- a/server/integration-test/src/test/resources/structurecompare/obmysql/source_drop.sql +++ b/server/integration-test/src/test/resources/structurecompare/obmysql/source_drop.sql @@ -12,5 +12,6 @@ drop table if exists `converse_to_partition_table`; drop table if exists `converse_to_non_partition_table`; drop table if exists `modify_partition_type`; drop table if exists `update_options`; +drop table if exists `bit_column_default_value`; diff --git a/server/integration-test/src/test/resources/structurecompare/obmysql/source_schema_ddl.sql b/server/integration-test/src/test/resources/structurecompare/obmysql/source_schema_ddl.sql index d1890582b9..2d18a64c17 100644 --- a/server/integration-test/src/test/resources/structurecompare/obmysql/source_schema_ddl.sql +++ b/server/integration-test/src/test/resources/structurecompare/obmysql/source_schema_ddl.sql @@ -92,4 +92,9 @@ PARTITION BY HASH(`id`) PARTITIONS 4; create table `update_options`( `c1` INT(11) NOT NULL, `c2` INT(11) NOT NULL -) CHARACTER SET = utf8mb4 COLLATE = utf8mb4_bin COMMENT = 'comment1'; \ No newline at end of file +) CHARACTER SET = utf8mb4 COLLATE = utf8mb4_bin COMMENT = 'comment1'; + +CREATE TABLE `bit_column_default_value` ( + `id` BIGINT AUTO_INCREMENT PRIMARY KEY COMMENT 'auto increment id', + `bit_data_1` BIT(1) NOT NULL DEFAULT b'0' +); \ No newline at end of file diff --git a/server/integration-test/src/test/resources/structurecompare/obmysql/target_drop.sql b/server/integration-test/src/test/resources/structurecompare/obmysql/target_drop.sql index 8da7a94dea..cb4c11d3a4 100644 --- a/server/integration-test/src/test/resources/structurecompare/obmysql/target_drop.sql +++ b/server/integration-test/src/test/resources/structurecompare/obmysql/target_drop.sql @@ -11,4 +11,5 @@ drop table if exists `update_partition`; drop table if exists `converse_to_partition_table`; drop table if exists `converse_to_non_partition_table`; drop table if exists `modify_partition_type`; -drop table if exists `update_options`; \ No newline at end of file +drop table if exists `update_options`; +drop table if exists `bit_column_default_value`; \ No newline at end of file diff --git a/server/integration-test/src/test/resources/structurecompare/obmysql/target_schema_ddl.sql b/server/integration-test/src/test/resources/structurecompare/obmysql/target_schema_ddl.sql index 71ff5cb491..ffe6a83ef6 100644 --- a/server/integration-test/src/test/resources/structurecompare/obmysql/target_schema_ddl.sql +++ b/server/integration-test/src/test/resources/structurecompare/obmysql/target_schema_ddl.sql @@ -95,4 +95,8 @@ PARTITION BY KEY(`id`) PARTITIONS 4; create table `update_options`( `c1` INT(11) NOT NULL, `c2` INT(11) NOT NULL -) CHARACTER SET = gb18030 COLLATE = gb18030_chinese_ci COMMENT = 'comment2'; \ No newline at end of file +) CHARACTER SET = gb18030 COLLATE = gb18030_chinese_ci COMMENT = 'comment2'; + +CREATE TABLE `bit_column_default_value` ( + `id` BIGINT AUTO_INCREMENT PRIMARY KEY COMMENT 'auto increment id' +); \ No newline at end of file diff --git a/server/modules/pom.xml b/server/modules/pom.xml index 8cb1f293c1..d4efe7d7c4 100644 --- a/server/modules/pom.xml +++ b/server/modules/pom.xml @@ -5,7 +5,7 @@ com.oceanbase odc-parent - 4.3.2-SNAPSHOT + 4.3.3-SNAPSHOT ../../pom.xml pom diff --git a/server/modules/sample-module/pom.xml b/server/modules/sample-module/pom.xml index ac0d00fb4c..d3d1ceb16c 100644 --- a/server/modules/sample-module/pom.xml +++ b/server/modules/sample-module/pom.xml @@ -5,7 +5,7 @@ module-parent com.oceanbase - 4.3.2-SNAPSHOT + 4.3.3-SNAPSHOT ../pom.xml sample-module diff --git a/server/odc-common/pom.xml b/server/odc-common/pom.xml index 4ddc210be7..4e4a9cf2fc 100644 --- a/server/odc-common/pom.xml +++ b/server/odc-common/pom.xml @@ -7,7 +7,7 @@ com.oceanbase odc-parent - 4.3.2-SNAPSHOT + 4.3.3-SNAPSHOT ../../pom.xml odc-common diff --git a/server/odc-common/src/main/java/com/oceanbase/odc/common/util/HostUtils.java b/server/odc-common/src/main/java/com/oceanbase/odc/common/util/HostUtils.java new file mode 100644 index 0000000000..e0826b8b21 --- /dev/null +++ b/server/odc-common/src/main/java/com/oceanbase/odc/common/util/HostUtils.java @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.common.util; + +import javax.validation.constraints.NotNull; + +import lombok.Data; +import lombok.extern.slf4j.Slf4j; + +/** + * @Author: Lebie + * @Date: 2025/1/15 13:47 + * @Description: [] + */ +@Slf4j +public class HostUtils { + + @NotNull + public static ServerAddress extractServerAddress(String ipAndPort) { + String trimmed = StringUtils.trim(ipAndPort); + if (StringUtils.isBlank(trimmed)) { + log.info("unable to extract server address, text is empty"); + throw new IllegalArgumentException("Empty server address!"); + } + String[] segments = StringUtils.split(trimmed, ":"); + if (segments.length != 2) { + log.info("unable to extract server address, segments={}", segments); + throw new IllegalArgumentException("Invalid server address!"); + } + if (StringUtils.isEmpty(segments[0]) || StringUtils.isEmpty(segments[1])) { + log.info("unable to extract server address, segments={}", segments); + throw new IllegalArgumentException("Invalid server address!"); + } + return new ServerAddress(segments[0], segments[1]); + } + + + + @Data + public static class ServerAddress { + String ipAddress; + String port; + + public ServerAddress(String ipAddress, String port) { + this.ipAddress = ipAddress; + this.port = port; + } + } +} diff --git a/server/odc-common/src/main/java/com/oceanbase/odc/common/util/MapUtils.java b/server/odc-common/src/main/java/com/oceanbase/odc/common/util/MapUtils.java index 854450d897..d10e8d044a 100644 --- a/server/odc-common/src/main/java/com/oceanbase/odc/common/util/MapUtils.java +++ b/server/odc-common/src/main/java/com/oceanbase/odc/common/util/MapUtils.java @@ -18,6 +18,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.Map; +import java.util.function.BiFunction; import java.util.stream.IntStream; import com.google.common.base.Splitter; @@ -128,4 +129,34 @@ public static int size(final Map map) { return org.apache.commons.collections4.MapUtils.size(map); } + /** + * compare map, null map and empty map consider as equals + * + * @param src + * @param target + * @param key of map + * @param value of map + * @param equalFunction function with argument value to compare if value instance is equals + * @return true is map is equals + */ + public static boolean isEqual(Map src, Map target, + BiFunction equalFunction) { + int currentSize = size(src); + int targetSize = size(target); + // map size not equals + if (currentSize != targetSize) { + return false; + } + if (currentSize == 0) { + return true; + } + // check key map + for (Map.Entry entry : src.entrySet()) { + if (!equalFunction.apply(entry.getValue(), target.get(entry.getKey()))) { + return false; + } + } + return true; + } + } diff --git a/server/odc-common/src/main/java/com/oceanbase/odc/common/util/StringUtils.java b/server/odc-common/src/main/java/com/oceanbase/odc/common/util/StringUtils.java index 9b8ef520b6..8c3eb20a96 100644 --- a/server/odc-common/src/main/java/com/oceanbase/odc/common/util/StringUtils.java +++ b/server/odc-common/src/main/java/com/oceanbase/odc/common/util/StringUtils.java @@ -21,6 +21,7 @@ import java.util.Locale; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.UUID; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -58,6 +59,7 @@ public abstract class StringUtils extends org.apache.commons.lang3.StringUtils { private static final Pattern PORT_PATTERN = Pattern.compile("^(([1-9]\\d{0,3}|[1-5]\\d{4}|6[0-4]\\d{3}|65[0-4]\\d{2}|655[0-2]\\d|6553" + "[0-5]))$"); + private static final char[] REGULAR_CHARS = {'.', '*', '+', '?', '^', '$', '[', ']', '(', ')', '{', '}', '|', '\\'}; private static final String ORACLE_ESCAPE_KEYWORD = "ESCAPE '\\'"; private static final String DEFAULT_VARIABLE_PREFIX = "${"; private static final String DEFAULT_VARIABLE_SUFFIX = "}"; @@ -66,6 +68,22 @@ public abstract class StringUtils extends org.apache.commons.lang3.StringUtils { private StringUtils() {} + public static Optional escapeRegex(String str) { + return Optional.ofNullable(str).map(s -> { + if (s.isEmpty()) { + return ""; + } + StringBuilder escapedString = new StringBuilder(); + for (char c : s.toCharArray()) { + if (isRegularChar(c)) { + escapedString.append('\\'); + } + escapedString.append(c); + } + return escapedString.toString(); + }); + } + public static Boolean checkMysqlIdentifierQuoted(final String str) { if (StringUtils.isBlank(str)) { return false; @@ -173,7 +191,7 @@ public static void quoteColumnDefaultValuesForMySQL(DBTable table) { table.getColumns().forEach(column -> { String defaultValue = column.getDefaultValue(); if (StringUtils.isNotEmpty(defaultValue)) { - if (!isDefaultValueBuiltInFunction(column)) { + if (!isDefaultValueBuiltInFunction(column) && !DataTypeUtil.isBitType(column.getTypeName())) { column.setDefaultValue("'".concat(defaultValue.replace("'", "''")).concat("'")); } } @@ -471,4 +489,13 @@ public static String getTranslatableKey(@NonNull String str) { } return str.substring(DEFAULT_VARIABLE_PREFIX.length(), str.length() - DEFAULT_VARIABLE_SUFFIX.length()); } + + private static boolean isRegularChar(char c) { + for (char special : REGULAR_CHARS) { + if (c == special) { + return true; + } + } + return false; + } } diff --git a/server/odc-common/src/test/java/com/oceanbase/odc/common/util/HostUtilsTest.java b/server/odc-common/src/test/java/com/oceanbase/odc/common/util/HostUtilsTest.java new file mode 100644 index 0000000000..14589c6d03 --- /dev/null +++ b/server/odc-common/src/test/java/com/oceanbase/odc/common/util/HostUtilsTest.java @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.common.util; + +import org.junit.Test; + +import com.oceanbase.odc.common.util.HostUtils.ServerAddress; + +import junit.framework.Assert; + +public class HostUtilsTest { + @Test + public void testExtractServerAddress_ValidExpression() { + String ipAndPort = "1.1.1.1:1234"; + ServerAddress actual = HostUtils.extractServerAddress(ipAndPort); + Assert.assertEquals("1.1.1.1", actual.getIpAddress()); + Assert.assertEquals("1234", actual.getPort()); + } + + @Test(expected = IllegalArgumentException.class) + public void testExtractServerAddress_InvalidExpression() { + String ipAndPort = "1.1.1.1"; + HostUtils.extractServerAddress(ipAndPort); + } +} diff --git a/server/odc-common/src/test/java/com/oceanbase/odc/common/util/MapUtilsTest.java b/server/odc-common/src/test/java/com/oceanbase/odc/common/util/MapUtilsTest.java index ad9b4981f6..fc90bcfad5 100644 --- a/server/odc-common/src/test/java/com/oceanbase/odc/common/util/MapUtilsTest.java +++ b/server/odc-common/src/test/java/com/oceanbase/odc/common/util/MapUtilsTest.java @@ -17,6 +17,7 @@ import java.util.HashMap; import java.util.Map; +import java.util.TreeMap; import org.junit.Assert; import org.junit.Test; @@ -97,4 +98,49 @@ public void formatKvString_ValueContainsSeparator_IllegalArgumentException() { map.put("a", "2,"); MapUtils.formatKvString(map); } + + @Test + public void isEquals_emptyAndNull_equals() { + // empty and null is equals + Assert.assertTrue(MapUtils.isEqual(new HashMap(), null, String::equals)); + Assert.assertTrue(MapUtils.isEqual(null, null, String::equals)); + } + + @Test + public void isEquals_differentMapSize_notEquals() { + // size not equals + Assert.assertFalse(MapUtils.isEqual(new HashMap(), new TreeMap() { + { + put("key1", "value1"); + } + }, String::equals)); + } + + @Test + public void isEquals_sameValue_equals() { + // value equals + Assert.assertTrue(MapUtils.isEqual(new HashMap() { + { + put("key1", "value1"); + } + }, new TreeMap() { + { + put("key1", "value1"); + } + }, String::equals)); + } + + @Test + public void isEquals_differentValue_notEquals() { + // value not equals + Assert.assertFalse(MapUtils.isEqual(new HashMap() { + { + put("key1", "value1"); + } + }, new TreeMap() { + { + put("key1", null); + } + }, String::equals)); + } } diff --git a/server/odc-common/src/test/java/com/oceanbase/odc/common/util/StringUtilsTest.java b/server/odc-common/src/test/java/com/oceanbase/odc/common/util/StringUtilsTest.java index 6501c5857d..2b6c1f4bfb 100644 --- a/server/odc-common/src/test/java/com/oceanbase/odc/common/util/StringUtilsTest.java +++ b/server/odc-common/src/test/java/com/oceanbase/odc/common/util/StringUtilsTest.java @@ -15,6 +15,8 @@ */ package com.oceanbase.odc.common.util; +import java.util.Optional; + import org.junit.Assert; import org.junit.Test; @@ -398,4 +400,31 @@ public void startWithIgnoreSpaceAndNewLines_targetsNonStartingWithPrefixWithMaxM Assert.assertTrue(StringUtils.startsWithIgnoreSpaceAndNewLines(target, prefix, true, 2)); } + @Test + public void escapeRegex_null_isNotPresent() { + Assert.assertFalse(StringUtils.escapeRegex(null).isPresent()); + } + + @Test + public void escapeRegex_stringContainsRegularChar_escapeSuccess() { + String regex = ".*+?^$[](){}|\\"; + String expect = "\\.\\*\\+\\?\\^\\$\\[\\]\\(\\)\\{\\}\\|\\\\"; + Optional escapedRegex = StringUtils.escapeRegex(regex); + Assert.assertTrue(escapedRegex.isPresent()); + Assert.assertEquals(expect, escapedRegex.get()); + } + + @Test + public void replaceFirst_replaceOriginalNameContainingRegularCharsInPlEdit_success() { + String pl = + "create function `FuN_test@111--#%&*()_+` (`p1` int(12)) returns int(11) begin declare v1 int; set v1 = p1 + 1; return v1; end;"; + String originalName = "FuN_test@111--#%&*()_+"; + String replaceName = "replace_name"; + String expect = + "create function `replace_name` (`p1` int(12)) returns int(11) begin declare v1 int; set v1 = p1 + 1; return v1; end;"; + Optional escapedRegex = StringUtils.escapeRegex(originalName); + String actual = pl.replaceFirst(escapedRegex.get(), replaceName); + Assert.assertEquals(expect, actual); + + } } diff --git a/server/odc-core/pom.xml b/server/odc-core/pom.xml index 85ab44fab0..36a3bb0e07 100644 --- a/server/odc-core/pom.xml +++ b/server/odc-core/pom.xml @@ -5,7 +5,7 @@ com.oceanbase odc-parent - 4.3.2-SNAPSHOT + 4.3.3-SNAPSHOT ../../pom.xml odc-core diff --git a/server/odc-core/src/main/java/com/oceanbase/odc/core/alarm/AlarmUtils.java b/server/odc-core/src/main/java/com/oceanbase/odc/core/alarm/AlarmUtils.java index 0b19bb6a8b..236187fa8d 100644 --- a/server/odc-core/src/main/java/com/oceanbase/odc/core/alarm/AlarmUtils.java +++ b/server/odc-core/src/main/java/com/oceanbase/odc/core/alarm/AlarmUtils.java @@ -30,6 +30,7 @@ public final class AlarmUtils { public static final String TENANT_NAME = "Tenant"; public static final String ORGANIZATION_NAME = "OrganizationId"; public static final String MESSAGE_NAME = "Message"; + public static final String ODC_RESOURCE = "OdcResource"; public static final String ALARM_TARGET_NAME = "AlarmTarget"; public static final String ALARM_FIRE_ERROR_NAME = "AlarmFireError"; public static final String FAILED_REASON_NAME = "FailedReason"; @@ -37,6 +38,8 @@ public final class AlarmUtils { * TaskFramework alarm message names */ public static final String TASK_JOB_ID_NAME = "JobId"; + public static final String RESOURCE_ID_NAME = "ResourceID"; + public static final String RESOURCE_TYPE = "ResourceType"; public static final String TASK_TYPE_NAME = "TaskType"; public static final String SCHEDULE_ID_NAME = "ScheduleId"; public static final Collection TASK_FRAMEWORK_ALARM_DIGEST_NAMES = diff --git a/server/odc-core/src/main/java/com/oceanbase/odc/core/authority/DefaultLoginSecurityManager.java b/server/odc-core/src/main/java/com/oceanbase/odc/core/authority/DefaultLoginSecurityManager.java index 636fd9a4bc..2d0bec901e 100644 --- a/server/odc-core/src/main/java/com/oceanbase/odc/core/authority/DefaultLoginSecurityManager.java +++ b/server/odc-core/src/main/java/com/oceanbase/odc/core/authority/DefaultLoginSecurityManager.java @@ -216,4 +216,10 @@ public Permission getPermissionByResourceRoles(SecurityResource resource, Collec return this.permissionProvider.getPermissionByResourceRoles(resource, resourceRoles); } + @Override + public Permission getPermissionByActionsAndResourceRoles(SecurityResource resource, Collection actions, + Collection resourceRoles) { + return this.permissionProvider.getPermissionByActionsAndResourceRoles(resource, actions, resourceRoles); + } + } diff --git a/server/odc-core/src/main/java/com/oceanbase/odc/core/authority/interceptor/MethodAuthorizedInterceptor.java b/server/odc-core/src/main/java/com/oceanbase/odc/core/authority/interceptor/MethodAuthorizedInterceptor.java index 92e3a636ee..83dc340a39 100644 --- a/server/odc-core/src/main/java/com/oceanbase/odc/core/authority/interceptor/MethodAuthorizedInterceptor.java +++ b/server/odc-core/src/main/java/com/oceanbase/odc/core/authority/interceptor/MethodAuthorizedInterceptor.java @@ -110,19 +110,26 @@ private Permission beforeAuthentication(PreAuthenticate annotation, Object targe } } List actions = Arrays.asList(annotation.actions()); - List roles = Arrays.asList(annotation.hasAnyResourceRole()); - String role = annotation.hasResourceRole(); + List resourceRoles = Arrays.asList(annotation.hasAnyResourceRole()); + String resourceRole = annotation.hasResourceRole(); + + if (CollectionUtils.isNotEmpty(actions) + && (CollectionUtils.isNotEmpty(resourceRoles) || StringUtils.isNotBlank(resourceRole))) { + return getSecurityManager().getPermissionByActionsAndResourceRoles( + new DefaultSecurityResource(resourceId, resourceType), actions, resourceRoles); + } + if (CollectionUtils.isNotEmpty(actions)) { return getSecurityManager().getPermissionByActions(new DefaultSecurityResource(resourceId, resourceType), actions); - } else if (CollectionUtils.isNotEmpty(roles)) { + } else if (CollectionUtils.isNotEmpty(resourceRoles)) { return getSecurityManager().getPermissionByResourceRoles( new DefaultSecurityResource(resourceId, resourceType), - roles); - } else if (StringUtils.isNotBlank(role)) { + resourceRoles); + } else if (StringUtils.isNotBlank(resourceRole)) { return getSecurityManager().getPermissionByResourceRoles( new DefaultSecurityResource(resourceId, resourceType), - Collections.singleton(role)); + Collections.singleton(resourceRole)); } else { throw new NullPointerException("The actions and hasAnyRole are both empty"); } diff --git a/server/odc-core/src/main/java/com/oceanbase/odc/core/authority/permission/ComposedPermission.java b/server/odc-core/src/main/java/com/oceanbase/odc/core/authority/permission/ComposedPermission.java new file mode 100644 index 0000000000..eb60b38435 --- /dev/null +++ b/server/odc-core/src/main/java/com/oceanbase/odc/core/authority/permission/ComposedPermission.java @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.core.authority.permission; + +import java.util.List; + +import org.springframework.util.CollectionUtils; + +import lombok.Getter; +import lombok.NonNull; + +/** + * @Author: Lebie + * @Date: 2024/11/11 15:11 + * @Description: [] + */ +@Getter +public class ComposedPermission implements Permission { + private final List permissions; + + public ComposedPermission(@NonNull List permissions) { + this.permissions = permissions; + } + + @Override + public boolean implies(Permission permission) { + if (!(permission instanceof ComposedPermission)) { + return false; + } + ComposedPermission composedPermission = (ComposedPermission) permission; + if (CollectionUtils.isEmpty(composedPermission.getPermissions())) { + return true; + } + if (CollectionUtils.isEmpty(this.permissions)) { + return false; + } + for (Permission thatPermission : composedPermission.getPermissions()) { + for (Permission thisPermission : this.permissions) { + if (thisPermission.implies(thatPermission)) { + return true; + } + } + } + return false; + } +} diff --git a/server/odc-core/src/main/java/com/oceanbase/odc/core/authority/permission/PermissionProvider.java b/server/odc-core/src/main/java/com/oceanbase/odc/core/authority/permission/PermissionProvider.java index 5ec7097dcf..7593166b41 100644 --- a/server/odc-core/src/main/java/com/oceanbase/odc/core/authority/permission/PermissionProvider.java +++ b/server/odc-core/src/main/java/com/oceanbase/odc/core/authority/permission/PermissionProvider.java @@ -48,4 +48,13 @@ public interface PermissionProvider { */ Permission getPermissionByResourceRoles(SecurityResource resource, Collection resourceRoles); + /** + * @param resource {@link SecurityResource} + * @param actions action collection + * @param resourceRoles, see {@link ResourceRoleName} enums + * @return permission collection + */ + Permission getPermissionByActionsAndResourceRoles(SecurityResource resource, Collection actions, + Collection resourceRoles); + } diff --git a/server/odc-core/src/main/java/com/oceanbase/odc/core/authority/permission/PrivateConnectionPermission.java b/server/odc-core/src/main/java/com/oceanbase/odc/core/authority/permission/PrivateConnectionPermission.java deleted file mode 100644 index c0bc79cafc..0000000000 --- a/server/odc-core/src/main/java/com/oceanbase/odc/core/authority/permission/PrivateConnectionPermission.java +++ /dev/null @@ -1,113 +0,0 @@ -/* - * Copyright (c) 2023 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.oceanbase.odc.core.authority.permission; - -import java.util.Collections; -import java.util.LinkedList; -import java.util.List; -import java.util.Set; - -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.Validate; - -import com.oceanbase.odc.core.shared.constant.ResourceType; - -/** - * Permission for private Connection object - * - * @author yh263208 - * @date 2021-09-06 19:23 - * @since ODC_release_3.2.0 - * @see Permission - * @see ResourcePermission - */ -public class PrivateConnectionPermission extends ResourcePermission { - private static final long serialVersionUID = 7923423426638008112L; - - public static final String CONNECTION_USE = "use"; - - public static final int CONNECT = 0x20; - public static final int USE = ResourcePermission.READ | CONNECT; - public static final int ALL = ResourcePermission.ALL | CONNECT; - - public PrivateConnectionPermission(String resourceId, String action) { - super(resourceId, ResourceType.ODC_PRIVATE_CONNECTION.name(), action); - } - - @Override - protected int getMaskFromAction(String action) { - int mask = super.getMaskFromAction(action); - if (action == null || StringUtils.isBlank(action)) { - return mask; - } - String newAction = action.replaceAll(" |\r|\n|\f|\t", ""); - if ("*".equals(action)) { - return ALL; - } - String[] actionList = newAction.split(","); - for (String actionItem : actionList) { - if (CONNECTION_USE.equalsIgnoreCase(actionItem)) { - mask |= USE; - } else if ("*".equals(actionItem)) { - return ALL; - } - } - return mask; - } - - public static Set getAllActions() { - Set returnVal = ResourcePermission.getAllActions(); - returnVal.addAll(Collections.singletonList(CONNECTION_USE)); - return returnVal; - } - - public static String getActions(int mask) { - List actionList = getActionList(mask); - return String.join(",", actionList); - } - - public static List getActionList(int mask) { - List actionList = new LinkedList<>(); - if ((mask & ALL) == ALL) { - actionList.add("*"); - return actionList; - } - List actions = ResourcePermission.getActionList(mask); - if (CollectionUtils.containsAny(actions, "*")) { - actionList.addAll(ResourcePermission.getAllActions()); - } else { - actionList.addAll(actions); - } - if ((mask & USE) == USE) { - actionList.add(CONNECTION_USE); - } - return actionList; - } - - @Override - protected void initPermissionMask(int mask) { - Validate.isTrue((mask & ALL) == mask, "Mask value is illegal"); - this.mask = mask; - } - - @Override - public String toString() { - return ResourceType.ODC_PRIVATE_CONNECTION.getLocalizedMessage() - + ":" + this.resourceId + ": " + getActions(this.mask); - } - -} diff --git a/server/odc-core/src/main/java/com/oceanbase/odc/core/authority/permission/ProjectPermission.java b/server/odc-core/src/main/java/com/oceanbase/odc/core/authority/permission/ProjectPermission.java new file mode 100644 index 0000000000..dd45260e77 --- /dev/null +++ b/server/odc-core/src/main/java/com/oceanbase/odc/core/authority/permission/ProjectPermission.java @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.core.authority.permission; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +import org.apache.commons.lang3.Validate; + +import com.oceanbase.odc.common.util.StringUtils; +import com.oceanbase.odc.core.authority.model.SecurityResource; + +import lombok.Getter; +import lombok.NonNull; + +/** + * @Author: Lebie + * @Date: 2024/11/11 19:26 + * @Description: [] + */ +@Getter +public class ProjectPermission extends ResourcePermission { + private final String resourceId; + private final String resourceType; + private final List actions; + + public ProjectPermission(@NonNull SecurityResource resource, String action) { + super(resource.resourceId(), resource.resourceType(), action); + this.resourceId = resource.resourceId(); + this.resourceType = resource.resourceType(); + Validate.notNull(resourceId, "ResourceId can not be null"); + Validate.notNull(resourceType, "ResourceType can not be null"); + Validate.notEmpty(action); + this.actions = Arrays.asList(StringUtils.split(action, ",")).stream().map(e -> e.trim().toUpperCase()) + .collect(Collectors.toList()); + } + + public ProjectPermission(@NonNull SecurityResource resource, List actions) { + super(resource.resourceId(), resource.resourceType(), actions.stream().collect(Collectors.joining(","))); + this.resourceId = resource.resourceId(); + this.resourceType = resource.resourceType(); + Validate.notNull(resourceId, "ResourceId can not be null"); + Validate.notNull(resourceType, "ResourceType can not be null"); + Validate.notEmpty(actions); + this.actions = actions; + } + + + @Override + public boolean implies(Permission permission) { + if (this == permission) { + return true; + } + if (!(permission instanceof ProjectPermission)) { + return false; + } + ProjectPermission that = (ProjectPermission) permission; + return (this.resourceId.equals(that.resourceId) || "*".equals(this.resourceId)) + && (this.resourceType.equals(that.resourceType) || "*".equals(this.resourceType)) + && !Collections.disjoint(this.actions, that.getActions()); + } +} diff --git a/server/odc-core/src/main/java/com/oceanbase/odc/core/datasource/SingleConnectionDataSource.java b/server/odc-core/src/main/java/com/oceanbase/odc/core/datasource/SingleConnectionDataSource.java index 59da5ee38d..a58943a422 100644 --- a/server/odc-core/src/main/java/com/oceanbase/odc/core/datasource/SingleConnectionDataSource.java +++ b/server/odc-core/src/main/java/com/oceanbase/odc/core/datasource/SingleConnectionDataSource.java @@ -21,14 +21,19 @@ import java.lang.reflect.Proxy; import java.sql.Connection; import java.sql.SQLException; +import java.sql.Statement; import java.util.LinkedList; import java.util.List; import java.util.Objects; import java.util.Optional; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; +import com.oceanbase.odc.common.concurrent.ExecutorUtils; import com.oceanbase.odc.common.event.AbstractEvent; import com.oceanbase.odc.common.event.EventPublisher; import com.oceanbase.odc.core.datasource.event.ConnectionResetEvent; @@ -66,15 +71,42 @@ public class SingleConnectionDataSource extends BaseClassBasedDataSource impleme protected volatile Connection connection; private final List initializerList = new LinkedList<>(); private volatile Lock lock; + private final boolean keepAlive; + private long keepAliveIntervalMillis = 5 * 60 * 1000; + @Getter + @Setter + private String keepAliveSql = "SELECT 1 FROM DUAL"; + @Getter + private final int maxFailedKeepAliveAttempts = 5; + private AtomicInteger failedKeepAliveAttempts = new AtomicInteger(0); + private ScheduledExecutorService keepAliveScheduler; @Setter private long timeOutMillis = 10 * 1000; public SingleConnectionDataSource() { - this(false); + this(false, false); + } + + public SingleConnectionDataSource(boolean autoReconnect, boolean keepAlive) { + this.autoReconnect = autoReconnect; + this.keepAlive = keepAlive; + initKeepAliveScheduler(); } - public SingleConnectionDataSource(boolean autoReconnect) { + public SingleConnectionDataSource(boolean autoReconnect, boolean keepAlive, long keepAliveIntervalMillis) { this.autoReconnect = autoReconnect; + this.keepAlive = keepAlive; + this.keepAliveIntervalMillis = keepAliveIntervalMillis; + initKeepAliveScheduler(); + } + + public SingleConnectionDataSource(boolean autoReconnect, boolean keepAlive, long keepAliveIntervalMillis, + String keepAliveSql) { + this.autoReconnect = autoReconnect; + this.keepAlive = keepAlive; + this.keepAliveIntervalMillis = keepAliveIntervalMillis; + this.keepAliveSql = keepAliveSql; + initKeepAliveScheduler(); } @Override @@ -122,9 +154,10 @@ public Connection getConnection(String username, String password) throws SQLExce */ public synchronized void resetConnection() throws SQLException { log.info("The connection will be reset soon"); - close(); + closeConnection(); this.connection = null; this.lock = null; + this.failedKeepAliveAttempts.set(0); try (Connection conn = innerCreateConnection()) { onConnectionReset(conn); } @@ -136,13 +169,8 @@ public void addInitializer(@NonNull ConnectionInitializer connectionInitializer) @Override public void close() { - if (!Objects.isNull(this.connection)) { - try { - this.connection.close(); - } catch (Throwable throwable) { - log.error("Failed to close the connection", throwable); - } - } + shutdownKeepAliveScheduler(); + closeConnection(); } protected void prepareConnection(Connection con) throws SQLException { @@ -152,6 +180,17 @@ protected void prepareConnection(Connection con) throws SQLException { } } + + private void closeConnection() { + if (!Objects.isNull(this.connection)) { + try { + this.connection.close(); + } catch (Throwable throwable) { + log.error("Failed to close the connection", throwable); + } + } + } + private boolean tryLock(Lock lock) { try { boolean locked = lock.tryLock(timeOutMillis, TimeUnit.MILLISECONDS); @@ -235,6 +274,41 @@ private synchronized Connection innerCreateConnection() throws SQLException { } } + + private void initKeepAliveScheduler() { + if (!keepAlive || Objects.nonNull(keepAliveScheduler)) { + return; + } + failedKeepAliveAttempts.set(0); + keepAliveScheduler = Executors.newScheduledThreadPool(1); + keepAliveScheduler.scheduleWithFixedDelay(() -> { + if (failedKeepAliveAttempts.get() > maxFailedKeepAliveAttempts) { + return; + } + try (Connection conn = getConnection()) { + try (Statement statement = conn.createStatement()) { + statement.execute(keepAliveSql); + failedKeepAliveAttempts.set(0); + log.debug("Keep connection alive success"); + } + } catch (Exception e) { + log.warn("Failed to keep connection alive", e); + failedKeepAliveAttempts.incrementAndGet(); + } + }, this.keepAliveIntervalMillis, this.keepAliveIntervalMillis, TimeUnit.MILLISECONDS); + } + + + private void shutdownKeepAliveScheduler() { + try { + if (Objects.nonNull(keepAliveScheduler) && !keepAliveScheduler.isTerminated()) { + ExecutorUtils.gracefulShutdown(keepAliveScheduler, "connection-keep-alive-executor", 5L); + } + } catch (Exception ex) { + log.warn("Failed to shutdown keep alive scheduler", ex); + } + } + /** * Invocation handler that suppresses close calls on JDBC Connections. * diff --git a/server/odc-core/src/main/java/com/oceanbase/odc/core/session/ConnectionSessionConstants.java b/server/odc-core/src/main/java/com/oceanbase/odc/core/session/ConnectionSessionConstants.java index aae3896b6e..639f144bc2 100644 --- a/server/odc-core/src/main/java/com/oceanbase/odc/core/session/ConnectionSessionConstants.java +++ b/server/odc-core/src/main/java/com/oceanbase/odc/core/session/ConnectionSessionConstants.java @@ -74,6 +74,11 @@ public class ConnectionSessionConstants { * form of attributes, this is the key */ public static final String OB_VERSION = "OB_VERSION"; + /** + * The odp version needs to be stored in the database session in the form of attributes, this is the + * key + */ + public static final String ODP_VERSION = "ODP_VERSION"; /** * The connection account type current database session needs to be stored in the database session * in the form of attributes, this is the key diff --git a/server/odc-core/src/main/java/com/oceanbase/odc/core/session/ConnectionSessionUtil.java b/server/odc-core/src/main/java/com/oceanbase/odc/core/session/ConnectionSessionUtil.java index 9aa1ccbc4c..6d3136d8af 100644 --- a/server/odc-core/src/main/java/com/oceanbase/odc/core/session/ConnectionSessionUtil.java +++ b/server/odc-core/src/main/java/com/oceanbase/odc/core/session/ConnectionSessionUtil.java @@ -44,6 +44,7 @@ import com.oceanbase.odc.common.lang.Pair; import com.oceanbase.odc.common.util.HashUtils; import com.oceanbase.odc.common.util.SystemUtils; +import com.oceanbase.odc.common.util.VersionUtils; import com.oceanbase.odc.core.datasource.CloneableDataSourceFactory; import com.oceanbase.odc.core.shared.Verify; import com.oceanbase.odc.core.shared.constant.DialectType; @@ -72,6 +73,8 @@ @Slf4j public class ConnectionSessionUtil { + private static final String ODP_SPECIFIED_ROUTINE_ENABLED_VERSION_NUMBER = "3.1.11"; + public static void logSocketInfo(Connection connection, String scenario) { if (!(connection instanceof OceanBaseConnection)) { log.debug("skip log connection socket info due not an OceanBaseConnection, className={}, scenario={}", @@ -616,6 +619,25 @@ public static String getUniqueIdentifier(@NonNull ConnectionSession connectionSe return HashUtils.md5(connectionSession.getId()).replace("-", ""); } + /** + * Get the OBProxy version number. If an exception occurs or the version does not support, return + * null. + * + * @param connectionSession + * @return + */ + public static String getObProxyVersion(@NonNull ConnectionSession connectionSession) { + return (String) connectionSession.getAttribute(ConnectionSessionConstants.ODP_VERSION); + } + + public static boolean isSupportObProxyRoute(String obProxyVersion) { + if (obProxyVersion == null + || VersionUtils.isLessThan(obProxyVersion, ODP_SPECIFIED_ROUTINE_ENABLED_VERSION_NUMBER)) { + return false; + } + return true; + } + private static String getOrCreateFullPathAppendingSuffixToDataPath(@NonNull String suffix) throws IOException { String dataDir = SystemUtils.getEnvOrProperty("file.storage.dir"); if (dataDir == null) { diff --git a/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/PreConditions.java b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/PreConditions.java index 30282f7a3b..6a88790ecc 100644 --- a/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/PreConditions.java +++ b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/PreConditions.java @@ -269,16 +269,6 @@ public static boolean validFileSuffix(String fileName, List safeSuffixLi "file suffix is illegal"); } - public static void validNoDuplicatedAlterSchedule( - String parameterName, Object parameterValue, BooleanSupplier duplicatedChecker) { - if (duplicatedChecker.getAsBoolean()) { - throw new BadRequestException(ErrorCodes.AlterScheduleExists, - new Object[] {parameterName, parameterValue}, - String.format("alter task already exists by %s=%s", parameterName, - parameterValue.toString())); - } - } - public static void validSingleton(Collection collection, String parameterName) { notEmpty(collection, parameterName); if (collection.size() != 1) { diff --git a/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/AuditEventAction.java b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/AuditEventAction.java index 32bcc74d2d..9ceccde9f4 100644 --- a/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/AuditEventAction.java +++ b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/AuditEventAction.java @@ -349,6 +349,10 @@ public enum AuditEventAction implements Translatable { CREATE_PROJECT, + ARCHIVE_PROJECT, + + DELETE_PROJECT, + CREATE_ENVIRONMENT, UPDATE_ENVIRONMENT, diff --git a/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/ConnectType.java b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/ConnectType.java index 5c336e4345..6d2572833b 100644 --- a/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/ConnectType.java +++ b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/ConnectType.java @@ -35,6 +35,10 @@ public enum ConnectType { // reserved for future version ODP_SHARDING_OB_ORACLE(DialectType.OB_ORACLE), ORACLE(DialectType.ORACLE), + OSS(DialectType.FILE_SYSTEM), + OBS(DialectType.FILE_SYSTEM), + COS(DialectType.FILE_SYSTEM), + S3A(DialectType.FILE_SYSTEM), UNKNOWN(DialectType.UNKNOWN), ; diff --git a/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/DialectType.java b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/DialectType.java index 97b33431f8..b83c369e90 100644 --- a/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/DialectType.java +++ b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/DialectType.java @@ -29,6 +29,7 @@ public enum DialectType { ODP_SHARDING_OB_MYSQL, DORIS, POSTGRESQL, + FILE_SYSTEM, UNKNOWN, ; diff --git a/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/ErrorCodes.java b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/ErrorCodes.java index 8fa9eefb6c..08940280af 100644 --- a/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/ErrorCodes.java +++ b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/ErrorCodes.java @@ -170,6 +170,10 @@ public enum ErrorCodes implements ErrorCode { // Schedule AlterScheduleExists, InvalidCronExpression, + ScheduleIntervalTooShort, + UpdateNotAllowed, + PauseNotAllowed, + DeleteNotAllowed, // Partition plan PartitionPlanNoDropPreviewSqlGenerated, @@ -312,7 +316,13 @@ public enum ErrorCodes implements ErrorCode { * workspace */ WorkspaceDatabaseUserTypeMustBeAdmin, - ; + /** + * oss + */ + BucketNotExist, + InvalidAccessKeyId, + SignatureDoesNotMatch, + UnsupportedSyncTableStructure; @Override public String code() { diff --git a/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/FlowStatus.java b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/FlowStatus.java index 8469af1795..6504a046e0 100644 --- a/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/FlowStatus.java +++ b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/FlowStatus.java @@ -15,6 +15,10 @@ */ package com.oceanbase.odc.core.shared.constant; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + /** * @author wenniu.ly * @date 2022/2/9 @@ -87,6 +91,10 @@ public enum FlowStatus { */ COMPLETED, - PRE_CHECK_FAILED, + PRE_CHECK_FAILED; + public static List listUnfinishedStatus() { + return Collections.unmodifiableList( + Arrays.asList(CREATED, APPROVING, WAIT_FOR_EXECUTION, WAIT_FOR_CONFIRM, EXECUTING, ROLLBACKING)); + } } diff --git a/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/TaskStatus.java b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/TaskStatus.java index c2290e57e2..6f86cdd5f1 100644 --- a/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/TaskStatus.java +++ b/server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/TaskStatus.java @@ -36,6 +36,10 @@ public static List getProcessingStatus() { return Arrays.asList(PREPARING, RUNNING); } + public boolean isProcessing() { + return getProcessingStatus().contains(this); + } + public boolean isTerminated() { return TaskStatus.CANCELED == this || TaskStatus.FAILED == this || TaskStatus.DONE == this; } diff --git a/server/odc-core/src/main/java/com/oceanbase/odc/core/sql/execute/SessionOperations.java b/server/odc-core/src/main/java/com/oceanbase/odc/core/sql/execute/SessionOperations.java index 65fc5c0a7f..b3a54fd175 100644 --- a/server/odc-core/src/main/java/com/oceanbase/odc/core/sql/execute/SessionOperations.java +++ b/server/odc-core/src/main/java/com/oceanbase/odc/core/sql/execute/SessionOperations.java @@ -16,6 +16,11 @@ package com.oceanbase.odc.core.sql.execute; import java.sql.Connection; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import javax.validation.constraints.NotEmpty; import lombok.NonNull; @@ -30,4 +35,29 @@ public interface SessionOperations { void killQuery(@NonNull Connection connection, @NonNull String connectionId); + String getKillQuerySql(@NonNull String connectionId); + + String getKillSessionSql(@NonNull String connectionId); + + /** + * Get kill query SQL by connectionId + * + * @param connectionIds + * @return the map of connectionId to kill query SQL + */ + default Map getKillQuerySqls(@NotEmpty Set connectionIds) { + return connectionIds.stream().collect(Collectors.toMap(id -> id, this::getKillQuerySql)); + } + + + /** + * Get kill session SQL by connectionId + * + * @param connectionIds + * @return the map of connectionId to kill session SQL + */ + default Map getKillSessionSqls(@NotEmpty Set connectionIds) { + return connectionIds.stream().collect(Collectors.toMap(id -> id, this::getKillSessionSql)); + } + } diff --git a/server/odc-core/src/main/java/com/oceanbase/odc/core/sql/util/OBUtils.java b/server/odc-core/src/main/java/com/oceanbase/odc/core/sql/util/OBUtils.java index 78bcfd7d59..433f11aea0 100644 --- a/server/odc-core/src/main/java/com/oceanbase/odc/core/sql/util/OBUtils.java +++ b/server/odc-core/src/main/java/com/oceanbase/odc/core/sql/util/OBUtils.java @@ -308,6 +308,20 @@ public static String getObVersion(Connection connection) { } } + public static String getODPVersion(Connection connection) { + try (Statement statement = connection.createStatement()) { + try (ResultSet resultSet = statement.executeQuery("select proxy_version()")) { + if (resultSet.next()) { + return resultSet.getString("proxy_version()"); + } else { + return null; + } + } + } catch (Exception e) { + return null; + } + } + private static String buildSqlAudit(ConnectType connectType, String version) { StringBuilder str = new StringBuilder(); if (connectType.getDialectType().isOracle()) { diff --git a/server/odc-core/src/main/resources/i18n/BusinessMessages.properties b/server/odc-core/src/main/resources/i18n/BusinessMessages.properties index ccc8a223ae..c43c144c4b 100644 --- a/server/odc-core/src/main/resources/i18n/BusinessMessages.properties +++ b/server/odc-core/src/main/resources/i18n/BusinessMessages.properties @@ -224,7 +224,7 @@ com.oceanbase.odc.AuditEventAction.CREATE_ALTER_SCHEDULE_TASK=Create alter sched com.oceanbase.odc.AuditEventAction.CREATE_ONLINE_SCHEMA_CHANGE_TASK=Create online schema change task com.oceanbase.odc.AuditEventAction.CREATE_APPLY_PROJECT_PERMISSION_TASK=Create apply project permission task com.oceanbase.odc.AuditEventAction.CREATE_APPLY_DATABASE_PERMISSION_TASK=Create apply database permission task -com.oceanbase.odc.AuditEventAction.CREATE_APPLY_TABLE_PERMISSION_TASK=Create apply table permission task +com.oceanbase.odc.AuditEventAction.CREATE_APPLY_TABLE_PERMISSION_TASK=Create table/view permission application com.oceanbase.odc.AuditEventAction.STOP_ASYNC_TASK=Stop database change task com.oceanbase.odc.AuditEventAction.STOP_MULTIPLE_ASYNC_TASK=Stop batch database change task com.oceanbase.odc.AuditEventAction.STOP_MOCKDATA_TASK=Stop mock data task @@ -238,7 +238,7 @@ com.oceanbase.odc.AuditEventAction.STOP_ALTER_SCHEDULE_TASK=Stop alter schedule com.oceanbase.odc.AuditEventAction.STOP_ONLINE_SCHEMA_CHANGE_TASK=Stop online schema change task com.oceanbase.odc.AuditEventAction.STOP_APPLY_PROJECT_PERMISSION_TASK=Stop apply project permission task com.oceanbase.odc.AuditEventAction.STOP_APPLY_DATABASE_PERMISSION_TASK=Stop apply database permission task -com.oceanbase.odc.AuditEventAction.STOP_APPLY_TABLE_PERMISSION_TASK=Stop apply table permission task +com.oceanbase.odc.AuditEventAction.STOP_APPLY_TABLE_PERMISSION_TASK=Cancel table/view permission application com.oceanbase.odc.AuditEventAction.EXECUTE_ASYNC_TASK=Execute database change task com.oceanbase.odc.AuditEventAction.EXECUTE_MULTIPLE_ASYNC_TASK=Execute batch database change task com.oceanbase.odc.AuditEventAction.EXECUTE_MOCKDATA_TASK=Execute mock data task @@ -264,7 +264,7 @@ com.oceanbase.odc.AuditEventAction.APPROVE_ALTER_SCHEDULE_TASK=Approve alter sch com.oceanbase.odc.AuditEventAction.APPROVE_ONLINE_SCHEMA_CHANGE_TASK=Approve online schema change task com.oceanbase.odc.AuditEventAction.APPROVE_APPLY_PROJECT_PERMISSION_TASK=Approve apply project permission task com.oceanbase.odc.AuditEventAction.APPROVE_APPLY_DATABASE_PERMISSION_TASK=Approve apply database permission task -com.oceanbase.odc.AuditEventAction.APPROVE_APPLY_TABLE_PERMISSION_TASK=Approve apply table permission task +com.oceanbase.odc.AuditEventAction.APPROVE_APPLY_TABLE_PERMISSION_TASK=Approve table/view permission application com.oceanbase.odc.AuditEventAction.REJECT_ASYNC_TASK=Reject database change task com.oceanbase.odc.AuditEventAction.REJECT_MULTIPLE_ASYNC_TASK=Reject batch database change task com.oceanbase.odc.AuditEventAction.REJECT_MOCKDATA_TASK=Reject mock data task @@ -279,7 +279,7 @@ com.oceanbase.odc.AuditEventAction.REJECT_ALTER_SCHEDULE_TASK=Reject alter sched com.oceanbase.odc.AuditEventAction.REJECT_ONLINE_SCHEMA_CHANGE_TASK=Reject online schema change task com.oceanbase.odc.AuditEventAction.REJECT_APPLY_PROJECT_PERMISSION_TASK=Reject apply project permission task com.oceanbase.odc.AuditEventAction.REJECT_APPLY_DATABASE_PERMISSION_TASK=Reject apply database permission task -com.oceanbase.odc.AuditEventAction.REJECT_APPLY_TABLE_PERMISSION_TASK=Reject apply table permission task +com.oceanbase.odc.AuditEventAction.REJECT_APPLY_TABLE_PERMISSION_TASK=Reject table/view permission application com.oceanbase.odc.AuditEventAction.CREATE_DATA_MASKING_RULE=Create data masking rule com.oceanbase.odc.AuditEventAction.UPDATE_DATA_MASKING_RULE=Update data masking rule com.oceanbase.odc.AuditEventAction.ENABLE_DATA_MASKING_RULE=Enable data masking rule @@ -298,6 +298,8 @@ com.oceanbase.odc.AuditEventAction.CREATE_DATASOURCE=Create datasource com.oceanbase.odc.AuditEventAction.DELETE_DATASOURCE=delete datasource com.oceanbase.odc.AuditEventAction.UPDATE_DATASOURCE=Update datasource com.oceanbase.odc.AuditEventAction.CREATE_PROJECT=Create project +com.oceanbase.odc.AuditEventAction.ARCHIVE_PROJECT=Archive project +com.oceanbase.odc.AuditEventAction.DELETE_PROJECT=Delete project com.oceanbase.odc.AuditEventAction.UPDATE_SQL_SECURITY_RULE=Modify SQL security rule com.oceanbase.odc.AuditEventAction.GRANT_DATABASE_PERMISSION=Grant database permission com.oceanbase.odc.AuditEventAction.REVOKE_DATABASE_PERMISSION=Revoke database permission @@ -352,7 +354,7 @@ com.oceanbase.odc.AuditEventType.ALTER_SCHEDULE=Alter schedule com.oceanbase.odc.AuditEventType.ONLINE_SCHEMA_CHANGE=Online schema change com.oceanbase.odc.AuditEventType.APPLY_PROJECT_PERMISSION=Apply project permission com.oceanbase.odc.AuditEventType.APPLY_DATABASE_PERMISSION=Apply database permission -com.oceanbase.odc.AuditEventType.APPLY_TABLE_PERMISSION=Apply table permission +com.oceanbase.odc.AuditEventType.APPLY_TABLE_PERMISSION=Apply for table/view permissions com.oceanbase.odc.AuditEventType.DATA_MASKING_RULE=Data masking rule com.oceanbase.odc.AuditEventType.DATA_MASKING_POLICY=Data masking policy com.oceanbase.odc.AuditEventType.PERMISSION_APPLY=Permission apply @@ -506,28 +508,28 @@ com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.external-sql-inte # # sql-check module # -com.oceanbase.odc.CheckViolation.LocalizedMessage=There may be a problem with statement at line {0}, col {1}, detail: {2} -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.syntax-error.message=There is a syntax error in the statement, details: {0} +com.oceanbase.odc.CheckViolation.LocalizedMessage=There may be a problem with statement at line {0}, col {1}, detail: {2}. +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.syntax-error.message=There is a syntax error in the statement, details: {0}. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.syntax-error.name=Syntax Error -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.syntax-error.description=The statement to be checked has a syntax error +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.syntax-error.description=The statement to be checked has a syntax error. -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.index-column-calculation.message=Calculation on the column condition, which may cause the index to fail +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.index-column-calculation.message=Calculation on the column condition, which may cause the index to fail. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.index-column-calculation.name=Calculation on the column condition -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.index-column-calculation.description=Calculation on the column condition, which may cause the index to fail +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.index-column-calculation.description=Calculation on the column condition, which may cause the index to fail. -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.index-column-fuzzy-match.message=There is a left fuzzy match in the LIKE operation, which may cause the index to fail +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.index-column-fuzzy-match.message=There is a left fuzzy match in the LIKE operation, which may cause the index to fail. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.index-column-fuzzy-match.name=Left fuzzy match in the LIKE operation -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.index-column-fuzzy-match.description=There is a left fuzzy match in the LIKE operation, which may cause the index to fail +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.index-column-fuzzy-match.description=There is a left fuzzy match in the LIKE operation, which may cause the index to fail. -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.index-column-implicit-conversion.message=Implicit type conversion on the column condition, which may cause the index to fail +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.index-column-implicit-conversion.message=Implicit type conversion on the column condition, which may cause the index to fail. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.index-column-implicit-conversion.name=Implicit type conversion -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.index-column-implicit-conversion.description=Implicit type conversion on the column condition, which may cause the index to fail +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.index-column-implicit-conversion.description=Implicit type conversion on the column condition, which may cause the index to fail. -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-in-expr.message=Too many IN values to match may reduce the query efficiency, it is recommended not to exceed {0}, the actual is {1} +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-in-expr.message=Too many IN values to match may reduce the query efficiency, it is recommended not to exceed {0}, the actual is {1}. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-in-expr.name=Too many IN values to match -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-in-expr.description=Too many IN values to match may reduce the query efficiency +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-in-expr.description=Too many IN values to match may reduce the query efficiency. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-in-expr.max-in-expr-count=Maximum number of IN expressions -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-in-expr.max-in-expr-count.description=The maximum number of expressions in the IN operation in the conditional predicate +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-in-expr.max-in-expr-count.description=The maximum number of expressions in the IN operation in the conditional predicate. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-sql-affected-rows.message=The number of rows affected by this SQL statement exceeds the limit. Maximum Allowed Affected Rows: {0}; Estimated Affected Rows: {1}. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-sql-affected-rows.name=Estimated SQL Affected Rows @@ -536,112 +538,112 @@ com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-sql-affect com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-sql-affected-rows.allowed-max-sql-affected-count.description=Maximum Allowed SQL Affected Rows com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.estimate-sql-affected-rows-failed.message=The number of rows affected by this SQL statement cannot be determined or failed to be determined. -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.estimate-sql-affected-rows-failed.name=Unable to determine or failed to determine the number of rows affected by the SQL statement. +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.estimate-sql-affected-rows-failed.name=Unable to determine or failed to determine the number of rows affected by the SQL statement com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.estimate-sql-affected-rows-failed.description=Unable to determine or failed to determine the number of rows affected by the SQL statement. -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-table-join.message=Too many table objects JOIN may lead to an unoptimized execution plan. It is recommended not to exceed {0}, the actual is {1} +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-table-join.message=Too many table objects JOIN may lead to an unoptimized execution plan. It is recommended not to exceed {0}, the actual is {1}. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-table-join.name=Too many table objects JOIN -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-table-join.description=Too many table objects JOIN may lead to an unoptimized execution plan +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-table-join.description=Too many table objects JOIN may lead to an unoptimized execution plan. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-table-join.max-join-table-count=Max join table count com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-table-join.max-join-table-count.description=Maximum number of table joins -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-not-null-exists-not-in.message=NOT NULL is required in the NOT IN condition to avoid falling into nested loops +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-not-null-exists-not-in.message=NOT NULL is required in the NOT IN condition to avoid falling into nested loops. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-not-null-exists-not-in.name=NOT NULL is required in the NOT IN condition -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-not-null-exists-not-in.description=NOT NULL is required in the NOT IN condition to avoid falling into nested loops +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-not-null-exists-not-in.description=NOT NULL is required in the NOT IN condition to avoid falling into nested loops. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-valid-where-clause.message=Potential risk of constant true/false WHERE conditions in UPDATE/DELETE statements com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-valid-where-clause.name=No valid WHERE condition in UPDATE/DELETE statement com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-valid-where-clause.description=Potential risk of constant true/false WHERE conditions in UPDATE/DELETE statements -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-where-clause-exists.message=No WHERE condition in UPDATE/DELETE statement poses potential risk +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-where-clause-exists.message=No WHERE condition in UPDATE/DELETE statement poses potential risk. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-where-clause-exists.name=No WHERE condition in UPDATE/DELETE statement -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-where-clause-exists.description=No WHERE condition in UPDATE/DELETE statement poses potential risk +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-where-clause-exists.description=No WHERE condition in UPDATE/DELETE statement poses potential risk. -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-specific-column-exists.message=INSERT/REPLACE statements without specific columns may cause implicit conversion errors +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-specific-column-exists.message=INSERT/REPLACE statements without specific columns may cause implicit conversion errors. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-specific-column-exists.name=INSERT/REPLACE statement does not specify column names -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-specific-column-exists.description=INSERT/REPLACE statements without specific columns may cause implicit conversion errors +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-specific-column-exists.description=INSERT/REPLACE statements without specific columns may cause implicit conversion errors. -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-index-key-exists.message=Operation on table object {0} with index does not use index, may cause full table scan, resulting in performance degradation +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-index-key-exists.message=Operation on table object {0} with index does not use index, may cause full table scan, resulting in performance degradation. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-index-key-exists.name=No index exists -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-index-key-exists.description=Operation on table object with index does not use index, may cause full table scan, resulting in performance degradation +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-index-key-exists.description=Operation on table object with index does not use index, may cause full table scan, resulting in performance degradation. -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-partition-key-exists.message=Operation on partitioned table {0} does not use a partition key, resulting in partition pruning not possible +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-partition-key-exists.message=Operation on partitioned table {0} does not use a partition key, resulting in partition pruning not possible. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-partition-key-exists.name=No partition exists -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-partition-key-exists.description=Operation on partitioned table does not use a partition key, resulting in partition pruning not possible +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-partition-key-exists.description=Operation on partitioned table does not use a partition key, resulting in partition pruning not possible. -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-index-keys.message=There are too many indexes on the table object {0}, which may cause performance degradation, it is recommended not to exceed {1} +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-index-keys.message=There are too many indexes on the table object {0}, which may cause performance degradation, it is recommended not to exceed {1}. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-index-keys.name=Single table contains too many indexes -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-index-keys.description=Using too many indexes may cause performance degradation +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-index-keys.description=Using too many indexes may cause performance degradation. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-index-keys.max-index-count=Max index count -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-index-keys.max-index-count.description=The maximum number of indexes that can appear in a table definition +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-index-keys.max-index-count.description=The maximum number of indexes that can appear in a table definition. -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.prefer-local-index.message=It is recommended to use local indexes +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.prefer-local-index.message=It is recommended to use local indexes. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.prefer-local-index.name=Perfer local indexes -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.prefer-local-index.description=It is recommended to use local indexes +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.prefer-local-index.description=It is recommended to use local indexes. -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-columns.message=Table object {0} has too many columns, recommended no more than {1}, actual {2} +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-columns.message=Table object {0} has too many columns, recommended no more than {1}, actual {2}. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-columns.name=Single table contains too many columns -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-columns.description=Single table contains too many columns +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-columns.description=Single table contains too many columns. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-columns.max-column-definition-count=Max column definition count -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-columns.max-column-definition-count.description=The maximum number of columns that can appear in a table definition +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-columns.max-column-definition-count.description=The maximum number of columns that can appear in a table definition. -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-long-char-length.message=The length of the char type in the table definition is too long, it is recommended not to exceed {0}, the actual value is {1} +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-long-char-length.message=The length of the char type in the table definition is too long, it is recommended not to exceed {0}, the actual value is {1}. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-long-char-length.name=Limit the length of char type fields -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-long-char-length.description=The length of the char type field should not be too long +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-long-char-length.description=The length of the char type field should not be too long. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-long-char-length.max-char-length=char maximum allowed length com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-long-char-length.max-char-length.description=char maximum allowed length -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-unique-index-naming.message=The named {0} format of the unique index does not conform to the specification, pattern is {1} +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-unique-index-naming.message=The named {0} format of the unique index does not conform to the specification, pattern is {1}. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-unique-index-naming.name=Restrict unique index name format com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-unique-index-naming.description=The standardized naming of unique indexes helps to improve the efficiency of database development, default is uk_${table-name}_${column-name-1}_${column-name-2}_... com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-unique-index-naming.name-pattern=Unique index name regular expression com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-unique-index-naming.name-pattern.description=Unique index name regular expression -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.foreign-constraint-exists.message=Tables cannot use foreign keys +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.foreign-constraint-exists.message=Tables cannot use foreign keys. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.foreign-constraint-exists.name=Tables cannot use foreign keys -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.foreign-constraint-exists.description=Tables cannot use foreign keys +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.foreign-constraint-exists.description=Tables cannot use foreign keys. -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-primary-key-exists.message=The table must have a primary key +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-primary-key-exists.message=The table must have a primary key. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-primary-key-exists.name=The table must have a primary key -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-primary-key-exists.description=The table must have a primary key +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-primary-key-exists.description=The table must have a primary key. -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-table-comment-exists.message=The creation statement of the table object {0} must contain comments +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-table-comment-exists.message=The creation statement of the table object {0} must contain comments. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-table-comment-exists.name=Table must be commented -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-table-comment-exists.description=Table object existence annotation can help to quickly understand the business background of the table +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-table-comment-exists.description=Table object existence annotation can help to quickly understand the business background of the table. -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.table-name-in-black-list.message=Table object cannot be named with {0}, the name is blacklisted at {1} +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.table-name-in-black-list.message=Table object cannot be named with {0}, the name is blacklisted at {1}. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.table-name-in-black-list.name=Table names cannot be in the blacklist -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.table-name-in-black-list.description=The table name when creating the table, and the table name in the blacklist cannot be used when modifying the table name +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.table-name-in-black-list.description=The table name when creating the table, and the table name in the blacklist cannot be used when modifying the table name. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.table-name-in-black-list.black-list=Black list com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.table-name-in-black-list.black-list.description=Table Name blacklist -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-charset.message=The character set {0} used by the table object is not in the allowed range. Allowed character sets include: {1} +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-charset.message=The character set {0} used by the table object is not in the allowed range. Allowed character sets include: {1}. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-charset.name=Limit Table Character Set -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-charset.description=The charset used by the table object can only be selected from the options given in the whitelist +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-charset.description=The charset used by the table object can only be selected from the options given in the whitelist. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-charset.allowed-charsets=Allowed character sets com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-charset.allowed-charsets.description=Allowed character sets -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-collation.message=The collation {0} used by the table object is not in the allowed range. Allowed collations include: {1} +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-collation.message=The collation {0} used by the table object is not in the allowed range. Allowed collations include: {1}. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-collation.name=Restricts the collation of table objects -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-collation.description=The collation used by the table object can only be selected from the options given in the whitelist +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-collation.description=The collation used by the table object can only be selected from the options given in the whitelist. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-collation.allowed-collations=Allowed collations com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-collation.allowed-collations.description=Allowed collations -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-column-refs-in-index.message=There are {0} column references in the index out of a maximum of {1} column references +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-column-refs-in-index.message=There are {0} column references in the index out of a maximum of {1} column references. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-column-refs-in-index.name=Limit the number of columns included in a single index -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-column-refs-in-index.description=The number of column references in a single index definition should not be excessive +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-column-refs-in-index.description=The number of column references in a single index definition should not be excessive. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-column-refs-in-index.max-column-ref-count=Maximum number of column references com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-column-refs-in-index.max-column-ref-count.description=Maximum number of column references -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-datatypes.message=The primary key constraint involves a column whose type {0} is not allowed, allowed types include: {1} +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-datatypes.message=The primary key constraint involves a column whose type {0} is not allowed, allowed types include: {1}. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-datatypes.name=Restrict primary key data type -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-datatypes.description=The columns involved in the primary key of the table must be of the data type defined in the rule +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-datatypes.description=The columns involved in the primary key of the table must be of the data type defined in the rule. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-datatypes.allowed-datatypes=Types allowed as primary keys com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-datatypes.allowed-datatypes.description=Types allowed as primary keys -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-column-refs-in-primary-key.message=There are {0} column references in the primary key out of a maximum of {1} column references +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-column-refs-in-primary-key.message=There are {0} column references in the primary key out of a maximum of {1} column references. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-column-refs-in-primary-key.name=Limit the number of columns included in primary key -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-column-refs-in-primary-key.description=The number of column references in primary key definition should not be excessive +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-column-refs-in-primary-key.description=The number of column references in primary key definition should not be excessive. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-column-refs-in-primary-key.max-column-ref-count=Maximum number of column references com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-column-refs-in-primary-key.max-column-ref-count.description=Maximum number of column references @@ -649,51 +651,51 @@ com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-auto-in com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-auto-increment.name=Restrict primary key columns to auto-increment com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-auto-increment.description=Restrict primary key columns to auto-increment -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-index-naming.message=The named {0} format of the index does not conform to the specification, pattern is {1} +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-index-naming.message=The named {0} format of the index does not conform to the specification, pattern is {1}. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-index-naming.name=Restrict index name format com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-index-naming.description=The standardized naming of indexes helps to improve the efficiency of database development, default is idx_${table-name}_${column-name-1}_${column-name-2}_... com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-index-naming.name-pattern=Index name regular expression com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-index-naming.name-pattern.description=Index name regular expression -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-index-name-exists.message=The definition of an index or constraint is not named +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-index-name-exists.message=The definition of an index or constraint is not named. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-index-name-exists.name=Index or constraint needs to set name -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-index-name-exists.description=An index or constraint requires an explicitly defined name, otherwise the database names it automatically +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-index-name-exists.description=An index or constraint requires an explicitly defined name, otherwise the database names it automatically. -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.zerofill-exists.message=Numeric types use the ZEROFILL attribute +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.zerofill-exists.message=Numeric types use the ZEROFILL attribute. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.zerofill-exists.name=Numeric types cannot use the ZEROFILL attribute -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.zerofill-exists.description=Numeric types cannot use the ZEROFILL attribute +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.zerofill-exists.description=Numeric types cannot use the ZEROFILL attribute. -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-charset-exists.message=Cannot set character set on column +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-charset-exists.message=Cannot set character set on column. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-charset-exists.name=Cannot set character set on column -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-charset-exists.description=Cannot set character set on column +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-charset-exists.description=Cannot set character set on column. -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-collation-exists.message=Cannot set collation on column +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-collation-exists.message=Cannot set collation on column. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-collation-exists.name=Cannot set collation on column -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-collation-exists.description=Cannot set collation on column +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-collation-exists.description=Cannot set collation on column. -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-is-nullable.message=Columns of type {0} are not allowed to be null, and the data types of columns that are allowed to be null is: {1} +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-is-nullable.message=Columns of type {0} are not allowed to be null, and the data types of columns that are allowed to be null is: {1}. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-is-nullable.name=Restrict column not nullable (NOT NULL) com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-is-nullable.description=Restrict columns to be non-nullable com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-is-nullable.nullable-datatype-list=Nullable column data types com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-is-nullable.nullable-datatype-list.description=Nullable column data types -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-default-value-exists.message=Column with datatype {0} is not allowed without default value, column types without default are allowed: {1} +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-default-value-exists.message=Column with datatype {0} is not allowed without default value, column types without default are allowed: {1}. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-default-value-exists.name=Column has a default value -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-default-value-exists.description=Column has a default value +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-default-value-exists.description=Column has a default value. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-default-value-exists.no-default-value-datatype-list=Types without default values are allowed -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-default-value-exists.no-default-value-datatype-list.description=Types without default values are allowed +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-default-value-exists.no-default-value-datatype-list.description=Types without default values are allowed. -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-column-comment-exists.message=Column {0} has no column comments +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-column-comment-exists.message=Column {0} has no column comments. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-column-comment-exists.name=Column to be commented -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-column-comment-exists.description=Each column in the table definition needs to be commented +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-column-comment-exists.description=Each column in the table definition needs to be commented. -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-name-in-black-list.message=Column cannot be named with {0}, the name is blacklisted at {1} +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-name-in-black-list.message=Column cannot be named with {0}, the name is blacklisted at {1}. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-name-in-black-list.name=Column names cannot be in the blacklist -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-name-in-black-list.description=The column name when creating the table, and the column name in the blacklist cannot be used when modifying the column name +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-name-in-black-list.description=The column name when creating the table, and the column name in the blacklist cannot be used when modifying the column name. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-name-in-black-list.black-list=Black list com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-name-in-black-list.black-list.description=Column name blacklist -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-column-name-case.message=The case setting of the column name {0} does not conform to the specification +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-column-name-case.message=The case setting of the column name {0} does not conform to the specification. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-column-name-case.name=Restrict column name case com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-column-name-case.description=Restrict column name case com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-column-name-case.is-uppercase=Whether to limit uppercase @@ -701,7 +703,7 @@ com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-column-nam com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-column-name-case.is-lowercase=Whether to limit lowercase com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-column-name-case.is-lowercase.description=Whether to limit lowercase -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-name-case.message=The case setting of the table name {0} does not conform to the specification +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-name-case.message=The case setting of the table name {0} does not conform to the specification. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-name-case.name=Restrict table name case com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-name-case.description=Restrict table name case com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-name-case.is-uppercase=Whether to limit uppercase @@ -709,82 +711,90 @@ com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-name com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-name-case.is-lowercase=Whether to limit lowercase com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-name-case.is-lowercase.description=Whether to limit lowercase -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-auto-increment.message=The initial value of the auto-increment column of the restricted table is {0}, but the actual value is {1} +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-auto-increment.message=The initial value of the auto-increment column of the restricted table is {0}, but the actual value is {1}. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-auto-increment.name=Restrict auto-increment initial value for table creation com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-auto-increment.description=Restrict auto-increment initial value for table creation com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-auto-increment.init-value=Table auto-increment initial value com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-auto-increment.init-value.description=Table auto-increment initial value -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.select-star-exists.message=It is not recommended to use * to project all columns in the SELECT statement +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.select-star-exists.message=It is not recommended to use * to project all columns in the SELECT statement. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.select-star-exists.name=SELECT statement is deprecated * -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.select-star-exists.description=SELECT statement is deprecated * +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.select-star-exists.description=SELECT statement is deprecated *. -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.missing-required-columns.message=Table is missing required column: {0} +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.missing-required-columns.message=Table is missing required column: {0}. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.missing-required-columns.name=Table is missing required column -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.missing-required-columns.description=Table is missing required column +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.missing-required-columns.description=Table is missing required column. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.missing-required-columns.column-names=Collection of column names com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.missing-required-columns.column-names.description=Collection of column names -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-auto-increment-unsigned.message=Auto-increment column {0} is better to use unsigned type +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-auto-increment-unsigned.message=Auto-increment column {0} is better to use unsigned type. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-auto-increment-unsigned.name=Restrict auto-increment columns to use UNSIGNED type -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-auto-increment-unsigned.description=The unsigned type does not store negative numbers, and the value range stored in the same type is doubled, which can prevent the auto-increment column from exceeding the upper limit +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-auto-increment-unsigned.description=The unsigned type does not store negative numbers, and the value range stored in the same type is doubled, which can prevent the auto-increment column from exceeding the upper limit. -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-alter-statement.message=There are {0} modify statements for the same table {1}, exceeding the maximum limit of {2}, it is recommended to combine them into one ALTER statement +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-alter-statement.message=There are {0} modify statements for the same table {1}, exceeding the maximum limit of {2}, it is recommended to combine them into one ALTER statement. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-alter-statement.name=Limit the number of ALTERs executed on the same table at one time -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-alter-statement.description=Avoid the consumption caused by multiple table rebuilds and the impact on online business +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-alter-statement.description=Avoid the consumption caused by multiple table rebuilds and the impact on online business. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-alter-statement.max-alter-count=Maximum number of ALTERs -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-alter-statement.max-alter-count.description=The maximum number of ALTERs allowed for the same table +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-alter-statement.max-alter-count.description=The maximum number of ALTERs allowed for the same table. -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.not-null-column-without-default-value.message=Columns are marked as NOT NULL but do not have a default value set, which may result in insert errors +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.not-null-column-without-default-value.message=Columns are marked as NOT NULL but do not have a default value set, which may result in insert errors. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.not-null-column-without-default-value.name=When the field constraint is NOT NULL, it must have a default value -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.not-null-column-without-default-value.description=When the field constraint is NOT NULL, it must have a default value +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.not-null-column-without-default-value.description=When the field constraint is NOT NULL, it must have a default value. -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-naming.message=The named {0} format of the primary key does not conform to the specification, pattern is {1} +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-naming.message=The named {0} format of the primary key does not conform to the specification, pattern is {1}. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-naming.name=Restrict primary key name format com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-naming.description=The standardized naming of PK helps to improve the efficiency of database development, default is pk_${table-name}_${column-name-1}_${column-name-2}_... com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-naming.name-pattern=PK name regular expression com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-naming.name-pattern.description=PK name regular expression -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.prohibited-datatype-exists.message=Datatype {0} is forbidden, forbidden datatypes: {1} +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.prohibited-datatype-exists.message=Datatype {0} is forbidden, forbidden datatypes: {1}. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.prohibited-datatype-exists.name=Certain data types are prohibited -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.prohibited-datatype-exists.description=Certain data types are prohibited +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.prohibited-datatype-exists.description=Certain data types are prohibited. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.prohibited-datatype-exists.datatype-names=Prohibited data types com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.prohibited-datatype-exists.datatype-names.description=Prohibited data types -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-index-datatypes.message=Index reference column {0} is of datatype {1}, which is not allowed to be indexed, allowed types: {2} +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-index-datatypes.message=Index reference column {0} is of datatype {1}, which is not allowed to be indexed, allowed types: {2}. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-index-datatypes.name=Restricted Index Data Types -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-index-datatypes.description=Only columns of a specific data type can be referenced by the index, avoiding a large number of resources caused by incorrect indexing and serious performance problems +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-index-datatypes.description=Only columns of a specific data type can be referenced by the index, avoiding a large number of resources caused by incorrect indexing and serious performance problems. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-index-datatypes.allowed-datatypes=Allowed data types com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-index-datatypes.allowed-datatypes.description=Data types allowed in indexes -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-drop-object-types.message=The database object type {0} involved in the object delete statement is not within the allowed range, the object types allowed to be deleted include: {1} +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-drop-object-types.message=The database object type {0} involved in the object delete statement is not within the allowed range, the object types allowed to be deleted include: {1}. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-drop-object-types.name=Limit the types of objects that can be deleted -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-drop-object-types.description=Deletion of database objects outside the allowed range is not allowed +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-drop-object-types.description=Deletion of database objects outside the allowed range is not allowed. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-drop-object-types.allowed-object-types=Database object type that allowed to be deleted -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-drop-object-types.allowed-object-types.description=The types of database objects that are allowed to be deleted +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-drop-object-types.allowed-object-types.description=The types of database objects that are allowed to be deleted. -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-primary-key-name-exists.message=Primary key constraint/index are not named +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-primary-key-name-exists.message=Primary key constraint/index are not named. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-primary-key-name-exists.name=Primary key constraint/index need to set names -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-primary-key-name-exists.description=Primary key constraint/index need to be explicitly named, otherwise the database will automatically name them +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-primary-key-name-exists.description=Primary key constraint/index need to be explicitly named, otherwise the database will automatically name them. -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-auto-increment-datatypes.message=Column {0} is marked as auto-increment. The column type is {1}, which is not within the allowed type range: {2}. This may lead to unexpected situations such as numeric overflow +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-auto-increment-datatypes.message=Column {0} is marked as auto-increment. The column type is {1}, which is not within the allowed type range: {2}. This may lead to unexpected situations such as numeric overflow. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-auto-increment-datatypes.name=Limit optional data types for auto-increment columns -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-auto-increment-datatypes.description=The data type of columns marked as auto-increment should be carefully selected to avoid unexpected situations such as numerical overflow +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-auto-increment-datatypes.description=The data type of columns marked as auto-increment should be carefully selected to avoid unexpected situations such as numerical overflow. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-auto-increment-datatypes.allowed-datatypes=Allowed data types com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-auto-increment-datatypes.allowed-datatypes.description=A collection of data types that allow columns to be marked as auto-increment com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.object-name-using-reserved-words.message={0} is a reserved word and should not be used as an object name. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.object-name-using-reserved-words.name=Reserved words should not be used as object names -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.object-name-using-reserved-words.description=Reserved words should not be used as object names +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.object-name-using-reserved-words.description=Reserved words should not be used as object names. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.offline-schema-change-exists.message={0} is an offline structure change statement, which may take a long time. It is recommended to use lock-free structure change to execute this statement. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.offline-schema-change-exists.name=Offline structure change statement exists com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.offline-schema-change-exists.description=The offline structure change statement may involve full data modification, which will take a long time and may affect online business. It is recommended to use lock-free structure change to perform this operation. -com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.truncate-table-exists.message=The truncate statement will clear the table data, please use it with caution +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.truncate-table-exists.message=The truncate statement will clear the table data, please use it with caution. com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.truncate-table-exists.name=Truncate statement exists com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.truncate-table-exists.description=The truncate statement will clear table data and is very dangerous in a production environment. Please use it with caution. +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.create-table-like-exists.message=It is not recommended to use CREATE LIKE statement to create a table. +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.create-table-like-exists.name=CREATE LIKE statement exists +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.create-table-like-exists.description=It is not recommended to use CREATE LIKE statement to create a table. + +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.create-table-as-exists.message=It is not recommended to use CREATE AS statement to create a table. +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.create-table-as-exists.name=CREATE AS statement exists +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.create-table-as-exists.description=It is not recommended to use CREATE AS statement to create a table. + # below masking algorithm built-in com.oceanbase.odc.builtin-resource.masking-algorithm.mask-all.name=Mask all (system default) com.oceanbase.odc.builtin-resource.masking-algorithm.personal-name-chinese.name=Personal name (Chinese character) diff --git a/server/odc-core/src/main/resources/i18n/BusinessMessages_zh_CN.properties b/server/odc-core/src/main/resources/i18n/BusinessMessages_zh_CN.properties index 0dc888df39..dbe6d0d79b 100644 --- a/server/odc-core/src/main/resources/i18n/BusinessMessages_zh_CN.properties +++ b/server/odc-core/src/main/resources/i18n/BusinessMessages_zh_CN.properties @@ -223,7 +223,7 @@ com.oceanbase.odc.AuditEventAction.CREATE_ALTER_SCHEDULE_TASK=创建修改调度 com.oceanbase.odc.AuditEventAction.CREATE_ONLINE_SCHEMA_CHANGE_TASK=创建无锁结构变更任务 com.oceanbase.odc.AuditEventAction.CREATE_APPLY_PROJECT_PERMISSION_TASK=创建申请项目权限任务 com.oceanbase.odc.AuditEventAction.CREATE_APPLY_DATABASE_PERMISSION_TASK=创建申请数据库权限任务 -com.oceanbase.odc.AuditEventAction.CREATE_APPLY_TABLE_PERMISSION_TASK=创建申请数据表权限任务 +com.oceanbase.odc.AuditEventAction.CREATE_APPLY_TABLE_PERMISSION_TASK=创建申请数据表/视图权限任务 com.oceanbase.odc.AuditEventAction.STOP_ASYNC_TASK=停止数据库变更任务 com.oceanbase.odc.AuditEventAction.STOP_MULTIPLE_ASYNC_TASK=停止多库变更任务 com.oceanbase.odc.AuditEventAction.STOP_MOCKDATA_TASK=停止模拟数据任务 @@ -237,7 +237,7 @@ com.oceanbase.odc.AuditEventAction.STOP_ALTER_SCHEDULE_TASK=停止修改调度 com.oceanbase.odc.AuditEventAction.STOP_ONLINE_SCHEMA_CHANGE_TASK=停止无锁结构变更任务 com.oceanbase.odc.AuditEventAction.STOP_APPLY_PROJECT_PERMISSION_TASK=停止申请项目权限任务 com.oceanbase.odc.AuditEventAction.STOP_APPLY_DATABASE_PERMISSION_TASK=停止申请数据库权限任务 -com.oceanbase.odc.AuditEventAction.STOP_APPLY_TABLE_PERMISSION_TASK=停止申请数据表权限任务 +com.oceanbase.odc.AuditEventAction.STOP_APPLY_TABLE_PERMISSION_TASK=停止申请数据表/视图权限任务 com.oceanbase.odc.AuditEventAction.EXECUTE_ASYNC_TASK=执行数据库变更任务 com.oceanbase.odc.AuditEventAction.EXECUTE_MULTIPLE_ASYNC_TASK=执行多库变更任务 com.oceanbase.odc.AuditEventAction.EXECUTE_MOCKDATA_TASK=执行模拟数据任务 @@ -263,7 +263,7 @@ com.oceanbase.odc.AuditEventAction.APPROVE_ALTER_SCHEDULE_TASK=同意修改调 com.oceanbase.odc.AuditEventAction.APPROVE_ONLINE_SCHEMA_CHANGE_TASK=同意无锁结构变更任务 com.oceanbase.odc.AuditEventAction.APPROVE_APPLY_PROJECT_PERMISSION_TASK=同意申请项目权限任务 com.oceanbase.odc.AuditEventAction.APPROVE_APPLY_DATABASE_PERMISSION_TASK=同意申请数据库权限任务 -com.oceanbase.odc.AuditEventAction.APPROVE_APPLY_TABLE_PERMISSION_TASK=同意申请数据表权限任务 +com.oceanbase.odc.AuditEventAction.APPROVE_APPLY_TABLE_PERMISSION_TASK=同意申请数据表/视图权限任务 com.oceanbase.odc.AuditEventAction.REJECT_ASYNC_TASK=拒绝数据库变更任务 com.oceanbase.odc.AuditEventAction.REJECT_MULTIPLE_ASYNC_TASK=拒绝多库变更任务 com.oceanbase.odc.AuditEventAction.REJECT_MOCKDATA_TASK=拒绝模拟数据任务 @@ -278,7 +278,7 @@ com.oceanbase.odc.AuditEventAction.REJECT_ALTER_SCHEDULE_TASK=拒绝修改调度 com.oceanbase.odc.AuditEventAction.REJECT_ONLINE_SCHEMA_CHANGE_TASK=拒绝无锁结构变更任务 com.oceanbase.odc.AuditEventAction.REJECT_APPLY_PROJECT_PERMISSION_TASK=拒绝申请项目权限任务 com.oceanbase.odc.AuditEventAction.REJECT_APPLY_DATABASE_PERMISSION_TASK=拒绝申请数据库权限任务 -com.oceanbase.odc.AuditEventAction.REJECT_APPLY_TABLE_PERMISSION_TASK=拒绝申请数据表权限任务 +com.oceanbase.odc.AuditEventAction.REJECT_APPLY_TABLE_PERMISSION_TASK=拒绝申请数据表/视图权限任务 com.oceanbase.odc.AuditEventAction.CREATE_DATA_MASKING_RULE=创建脱敏规则 com.oceanbase.odc.AuditEventAction.UPDATE_DATA_MASKING_RULE=更新脱敏规则 com.oceanbase.odc.AuditEventAction.ENABLE_DATA_MASKING_RULE=启用脱敏规则 @@ -297,6 +297,8 @@ com.oceanbase.odc.AuditEventAction.CREATE_DATASOURCE=创建数据源 com.oceanbase.odc.AuditEventAction.DELETE_DATASOURCE=删除数据源 com.oceanbase.odc.AuditEventAction.UPDATE_DATASOURCE=更新数据源 com.oceanbase.odc.AuditEventAction.CREATE_PROJECT=创建项目 +com.oceanbase.odc.AuditEventAction.ARCHIVE_PROJECT=归档项目 +com.oceanbase.odc.AuditEventAction.DELETE_PROJECT=删除项目 com.oceanbase.odc.AuditEventAction.UPDATE_SQL_SECURITY_RULE=修改 SQL 安全规则 com.oceanbase.odc.AuditEventAction.GRANT_DATABASE_PERMISSION=新增库权限 com.oceanbase.odc.AuditEventAction.REVOKE_DATABASE_PERMISSION=回收库权限 @@ -357,7 +359,7 @@ com.oceanbase.odc.AuditEventType.ALTER_SCHEDULE=修改调度 com.oceanbase.odc.AuditEventType.ONLINE_SCHEMA_CHANGE=无锁结构变更 com.oceanbase.odc.AuditEventType.APPLY_PROJECT_PERMISSION=申请项目权限 com.oceanbase.odc.AuditEventType.APPLY_DATABASE_PERMISSION=申请数据库权限 -com.oceanbase.odc.AuditEventType.APPLY_TABLE_PERMISSION=申请数据表权限 +com.oceanbase.odc.AuditEventType.APPLY_TABLE_PERMISSION=申请数据表/视图权限 com.oceanbase.odc.AuditEventType.DATA_MASKING_RULE=脱敏规则 com.oceanbase.odc.AuditEventType.DATA_MASKING_POLICY=脱敏策略 com.oceanbase.odc.AuditEventType.PERMISSION_APPLY=权限申请 @@ -724,6 +726,14 @@ com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.truncate-table-exis com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.truncate-table-exists.name=存在 truncate 语句 com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.truncate-table-exists.description=truncate 语句将会清空表数据，在生产环境中十分危险，请谨慎使用 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.create-table-like-exists.message=不建议使用 CREATE LIKE 语句建表 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.create-table-like-exists.name=存在 CREATE LIKE 语句 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.create-table-like-exists.description=不建议使用 CREATE LIKE 语句建表 + +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.create-table-as-exists.message=不建议使用 CREATE AS 语句建表 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.create-table-as-exists.name=存在 CREATE AS 语句 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.create-table-as-exists.description=不建议使用 CREATE AS 语句建表 + # below masking algorithm built-in com.oceanbase.odc.builtin-resource.masking-algorithm.mask-all.name=全部遮掩（系统默认） com.oceanbase.odc.builtin-resource.masking-algorithm.personal-name-chinese.name=个人姓名（汉字类型） @@ -791,7 +801,7 @@ com.oceanbase.odc.TaskType.PRE_CHECK=预检查 com.oceanbase.odc.TaskType.EXPORT_RESULT_SET=导出结果集 com.oceanbase.odc.TaskType.APPLY_PROJECT_PERMISSION=申请项目权限 com.oceanbase.odc.TaskType.APPLY_DATABASE_PERMISSION=申请数据库权限 -com.oceanbase.odc.TaskType.APPLY_TABLE_PERMISSION=申请数据表权限 +com.oceanbase.odc.TaskType.APPLY_TABLE_PERMISSION=申请数据表/视图权限 com.oceanbase.odc.TaskType.SQL_PLAN=SQL 计划 com.oceanbase.odc.TaskType.DATA_ARCHIVE=数据归档 com.oceanbase.odc.TaskType.DATA_DELETE=数据清理 @@ -822,7 +832,7 @@ com.oceanbase.odc.notification.channel-test-message=【ODC】消息通道验证 # com.oceanbase.odc.builtin-resource.permission-apply.project.description=申请项目【{0}】的【{1}】权限 com.oceanbase.odc.builtin-resource.permission-apply.database.description=申请数据库的【{0}】权限 -com.oceanbase.odc.builtin-resource.permission-apply.table.description=申请数据表的【{0}】权限 +com.oceanbase.odc.builtin-resource.permission-apply.table.description=申请数据表/视图的【{0}】权限 # # Multiple Async diff --git a/server/odc-core/src/main/resources/i18n/BusinessMessages_zh_TW.properties b/server/odc-core/src/main/resources/i18n/BusinessMessages_zh_TW.properties index 6d3b3ae0a9..6f3b3a207e 100644 --- a/server/odc-core/src/main/resources/i18n/BusinessMessages_zh_TW.properties +++ b/server/odc-core/src/main/resources/i18n/BusinessMessages_zh_TW.properties @@ -224,7 +224,7 @@ com.oceanbase.odc.AuditEventAction.CREATE_ALTER_SCHEDULE_TASK=創建修改調度 com.oceanbase.odc.AuditEventAction.CREATE_ONLINE_SCHEMA_CHANGE_TASK=創建無鎖結構變更任務 com.oceanbase.odc.AuditEventAction.CREATE_APPLY_PROJECT_PERMISSION_TASK=創建申請項目權限任務 com.oceanbase.odc.AuditEventAction.CREATE_APPLY_DATABASE_PERMISSION_TASK=創建申請數據庫權限任務 -com.oceanbase.odc.AuditEventAction.CREATE_APPLY_TABLE_PERMISSION_TASK=創建申請數據表權限任務 +com.oceanbase.odc.AuditEventAction.CREATE_APPLY_TABLE_PERMISSION_TASK=創建申請數據表/視圖權限任務 com.oceanbase.odc.AuditEventAction.STOP_ASYNC_TASK=停止數據庫變更任務 com.oceanbase.odc.AuditEventAction.STOP_MULTIPLE_ASYNC_TASK=停止多庫變更任務 com.oceanbase.odc.AuditEventAction.STOP_MOCKDATA_TASK=停止模擬數據任務 @@ -238,7 +238,7 @@ com.oceanbase.odc.AuditEventAction.STOP_ALTER_SCHEDULE_TASK=停止修改調度 com.oceanbase.odc.AuditEventAction.STOP_ONLINE_SCHEMA_CHANGE_TASK=停止無鎖結構變更任務 com.oceanbase.odc.AuditEventAction.STOP_APPLY_PROJECT_PERMISSION_TASK=停止申請項目權限任務 com.oceanbase.odc.AuditEventAction.STOP_APPLY_DATABASE_PERMISSION_TASK=停止申請數據庫權限任務 -com.oceanbase.odc.AuditEventAction.STOP_APPLY_TABLE_PERMISSION_TASK=停止申請數據表權限任務 +com.oceanbase.odc.AuditEventAction.STOP_APPLY_TABLE_PERMISSION_TASK=停止申請數據表/視圖權限任務 com.oceanbase.odc.AuditEventAction.EXECUTE_ASYNC_TASK=執行數據庫變更任務 com.oceanbase.odc.AuditEventAction.EXECUTE_MULTIPLE_ASYNC_TASK=執行多庫變更任務 com.oceanbase.odc.AuditEventAction.EXECUTE_MOCKDATA_TASK=執行模擬數據任務 @@ -264,7 +264,7 @@ com.oceanbase.odc.AuditEventAction.APPROVE_ALTER_SCHEDULE_TASK=同意修改調 com.oceanbase.odc.AuditEventAction.APPROVE_ONLINE_SCHEMA_CHANGE_TASK=同意無鎖結構變更任務 com.oceanbase.odc.AuditEventAction.APPROVE_APPLY_PROJECT_PERMISSION_TASK=同意申請項目權限任務 com.oceanbase.odc.AuditEventAction.APPROVE_APPLY_DATABASE_PERMISSION_TASK=同意申請數據庫權限任務 -com.oceanbase.odc.AuditEventAction.APPROVE_APPLY_TABLE_PERMISSION_TASK=同意申請數據表權限任務 +com.oceanbase.odc.AuditEventAction.APPROVE_APPLY_TABLE_PERMISSION_TASK=同意申請數據表/視圖權限任務 com.oceanbase.odc.AuditEventAction.REJECT_ASYNC_TASK=拒絕數據庫變更任務 com.oceanbase.odc.AuditEventAction.REJECT_MULTIPLE_ASYNC_TASK=拒絕多庫變更任務 com.oceanbase.odc.AuditEventAction.REJECT_MOCKDATA_TASK=拒絕模擬數據任務 @@ -279,7 +279,7 @@ com.oceanbase.odc.AuditEventAction.REJECT_ALTER_SCHEDULE_TASK=拒絕修改調度 com.oceanbase.odc.AuditEventAction.REJECT_ONLINE_SCHEMA_CHANGE_TASK=拒絕無鎖結構變更任務 com.oceanbase.odc.AuditEventAction.REJECT_APPLY_PROJECT_PERMISSION_TASK=拒絕申請項目權限任務 com.oceanbase.odc.AuditEventAction.REJECT_APPLY_DATABASE_PERMISSION_TASK=拒絕申請數據庫權限任務 -com.oceanbase.odc.AuditEventAction.REJECT_APPLY_TABLE_PERMISSION_TASK=拒絕申請數據表權限任務 +com.oceanbase.odc.AuditEventAction.REJECT_APPLY_TABLE_PERMISSION_TASK=拒絕申請數據表/視圖權限任務 com.oceanbase.odc.AuditEventAction.CREATE_DATA_MASKING_RULE=創建脫敏規則 com.oceanbase.odc.AuditEventAction.UPDATE_DATA_MASKING_RULE=更新脫敏規則 com.oceanbase.odc.AuditEventAction.ENABLE_DATA_MASKING_RULE=啓用脫敏規則 @@ -298,6 +298,8 @@ com.oceanbase.odc.AuditEventAction.CREATE_DATASOURCE=創建數據庫源 com.oceanbase.odc.AuditEventAction.DELETE_DATASOURCE=删除數據庫源 com.oceanbase.odc.AuditEventAction.UPDATE_DATASOURCE=更新數據庫源 com.oceanbase.odc.AuditEventAction.CREATE_PROJECT=創建项目 +com.oceanbase.odc.AuditEventAction.ARCHIVE_PROJECT=歸檔項目 +com.oceanbase.odc.AuditEventAction.DELETE_PROJECT=刪除項目 com.oceanbase.odc.AuditEventAction.UPDATE_SQL_SECURITY_RULE=修改 SQL 安全規則 com.oceanbase.odc.AuditEventAction.GRANT_DATABASE_PERMISSION=新增庫權限 com.oceanbase.odc.AuditEventAction.REVOKE_DATABASE_PERMISSION=回收庫權限 @@ -357,7 +359,7 @@ com.oceanbase.odc.AuditEventType.STRUCTURE_COMPARISON=結構比對 com.oceanbase.odc.AuditEventType.ONLINE_SCHEMA_CHANGE=無鎖結構變更 com.oceanbase.odc.AuditEventType.APPLY_PROJECT_PERMISSION=申請項目權限 com.oceanbase.odc.AuditEventType.APPLY_DATABASE_PERMISSION=申請數據庫權限 -com.oceanbase.odc.AuditEventType.APPLY_TABLE_PERMISSION=申請數據表權限 +com.oceanbase.odc.AuditEventType.APPLY_TABLE_PERMISSION=申請數據表/視圖權限 com.oceanbase.odc.AuditEventType.DATA_MASKING_RULE=脫敏規則 com.oceanbase.odc.AuditEventType.DATA_MASKING_POLICY=脫敏策略 com.oceanbase.odc.AuditEventType.PERMISSION_APPLY=權限申請 @@ -789,6 +791,14 @@ com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.truncate-table-exis com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.truncate-table-exists.name=存在 truncate 語句 com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.truncate-table-exists.description=truncate 語句將會清空表格數據，在生產環境中十分危險，請謹慎使用 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.create-table-like-exists.message=不建議使用 CREATE LIKE 語句建表 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.create-table-like-exists.name=存在 CREATE LIKE 語句 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.create-table-like-exists.description=不建議使用 CREATE LIKE 語句建表 + +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.create-table-as-exists.message=不建議使用 CREATE AS 語句建表 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.create-table-as-exists.name=存在 CREATE AS 語句 +com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.create-table-as-exists.description=不建議使用 CREATE AS 語句建表 + # below masking algorithm built-in com.oceanbase.odc.builtin-resource.masking-algorithm.mask-all.name=全部遮掩（系統默認） com.oceanbase.odc.builtin-resource.masking-algorithm.personal-name-chinese.name=個人姓名（漢字類型） @@ -861,7 +871,7 @@ com.oceanbase.odc.TaskType.PRE_CHECK=預檢查 com.oceanbase.odc.TaskType.EXPORT_RESULT_SET=導出結果集 com.oceanbase.odc.TaskType.APPLY_PROJECT_PERMISSION=申請項目權限 com.oceanbase.odc.TaskType.APPLY_DATABASE_PERMISSION=申請數據庫權限 -com.oceanbase.odc.TaskType.APPLY_TABLE_PERMISSION=申請數據表權限 +com.oceanbase.odc.TaskType.APPLY_TABLE_PERMISSION=申請數據表/視圖權限 com.oceanbase.odc.TaskType.SQL_PLAN=SQL 計劃 com.oceanbase.odc.TaskType.DATA_ARCHIVE=數據歸檔 com.oceanbase.odc.TaskType.DATA_DELETE=數據清理 @@ -892,7 +902,7 @@ com.oceanbase.odc.notification.channel-test-message=【ODC】消息通道驗證 # com.oceanbase.odc.builtin-resource.permission-apply.project.description=申請項目【{0}】的【{1}】權限 com.oceanbase.odc.builtin-resource.permission-apply.database.description=申請數據庫的【{0}】權限 -com.oceanbase.odc.builtin-resource.permission-apply.table.description=申請數據表的【{0}】權限 +com.oceanbase.odc.builtin-resource.permission-apply.table.description=申請數據表/視圖的【{0}】權限 # # Multiple Async diff --git a/server/odc-core/src/main/resources/i18n/ErrorMessages.properties b/server/odc-core/src/main/resources/i18n/ErrorMessages.properties index 85f4974381..3b4c0109b5 100644 --- a/server/odc-core/src/main/resources/i18n/ErrorMessages.properties +++ b/server/odc-core/src/main/resources/i18n/ErrorMessages.properties @@ -200,4 +200,12 @@ com.oceanbase.odc.ErrorCodes.TimeDataTypePrecisionMismatch=The time type precisi com.oceanbase.odc.ErrorCodes.DatabaseAccessDenied=Database access is denied because the current user does not have {0} permissions for the database. com.oceanbase.odc.ErrorCodes.ObQueryProfileNotSupported=Query profile is only available for OceanBase Database with versions equal to or higher than {0}. com.oceanbase.odc.ErrorCodes.WorksheetEditVersionConflict=Someone has just modified this file. Please refresh the page to get the latest version and continue editing. -com.oceanbase.odc.ErrorCodes.WorkspaceDatabaseUserTypeMustBeAdmin=The database user type used in the workspace must be a super account. \ No newline at end of file +com.oceanbase.odc.ErrorCodes.WorkspaceDatabaseUserTypeMustBeAdmin=The database user type used in the workspace must be a super account. +com.oceanbase.odc.ErrorCodes.BucketNotExist=Bucket does not exist +com.oceanbase.odc.ErrorCodes.InvalidAccessKeyId=Invalid access key id +com.oceanbase.odc.ErrorCodes.SignatureDoesNotMatch=Invalid access key secret +com.oceanbase.odc.ErrorCodes.UnsupportedSyncTableStructure=Sync table structure is not supported for {0} to {1} +com.oceanbase.odc.ErrorCodes.ScheduleIntervalTooShort=The execution interval is configured too short, please reconfigure. The minimum interval is: {0} seconds. +com.oceanbase.odc.ErrorCodes.UpdateNotAllowed=Editing is not allowed in the current state. Please try again after disabling. +com.oceanbase.odc.ErrorCodes.PauseNotAllowed=Disabling is not allowed in the current state, please check if there are any records in execution. +com.oceanbase.odc.ErrorCodes.DeleteNotAllowed=Deletion is not allowed in the current state. diff --git a/server/odc-core/src/main/resources/i18n/ErrorMessages_zh_CN.properties b/server/odc-core/src/main/resources/i18n/ErrorMessages_zh_CN.properties index c4ea9bdadd..ec15c6c22e 100644 --- a/server/odc-core/src/main/resources/i18n/ErrorMessages_zh_CN.properties +++ b/server/odc-core/src/main/resources/i18n/ErrorMessages_zh_CN.properties @@ -199,4 +199,14 @@ com.oceanbase.odc.ErrorCodes.TimeDataTypePrecisionMismatch=时间类型精度不 com.oceanbase.odc.ErrorCodes.DatabaseAccessDenied=数据库访问被拒绝，因为当前用户没有数据库的{0}权限 com.oceanbase.odc.ErrorCodes.ObQueryProfileNotSupported=执行详情仅在高于或等于 {0} 的 OceanBase 版本可用 com.oceanbase.odc.ErrorCodes.WorksheetEditVersionConflict=有人刚刚修改了这个工作簿，请刷新页面以获取最新版本并重新编辑 -com.oceanbase.odc.ErrorCodes.WorkspaceDatabaseUserTypeMustBeAdmin=工作空间中使用的数据库用户类型必须为超级账号 \ No newline at end of file +com.oceanbase.odc.ErrorCodes.WorkspaceDatabaseUserTypeMustBeAdmin=工作空间中使用的数据库用户类型必须为超级账号 +com.oceanbase.odc.ErrorCodes.UpdateNotAllowed=当前状态下不允许编辑，请禁用后重试 +com.oceanbase.odc.ErrorCodes.PauseNotAllowed=当前状态下不允许禁用，请检查是否存在执行中的记录 +com.oceanbase.odc.ErrorCodes.DeleteNotAllowed=当前状态下不允许删除 + + +com.oceanbase.odc.ErrorCodes.BucketNotExist=桶不存在 +com.oceanbase.odc.ErrorCodes.InvalidAccessKeyId=无效的 AccessKeyId +com.oceanbase.odc.ErrorCodes.SignatureDoesNotMatch=无效的 AccessKeySecret +com.oceanbase.odc.ErrorCodes.UnsupportedSyncTableStructure=结构同步暂不支持 {0} 到 {1} +com.oceanbase.odc.ErrorCodes.ScheduleIntervalTooShort=执行间隔配置过短，请重新配置。最小间隔为：{0} 秒 \ No newline at end of file diff --git a/server/odc-core/src/main/resources/i18n/ErrorMessages_zh_TW.properties b/server/odc-core/src/main/resources/i18n/ErrorMessages_zh_TW.properties index 5e8ed7b042..d587dfe771 100644 --- a/server/odc-core/src/main/resources/i18n/ErrorMessages_zh_TW.properties +++ b/server/odc-core/src/main/resources/i18n/ErrorMessages_zh_TW.properties @@ -199,4 +199,12 @@ com.oceanbase.odc.ErrorCodes.TimeDataTypePrecisionMismatch=時間類型精度不 com.oceanbase.odc.ErrorCodes.DatabaseAccessDenied=資料庫存取被拒絕，因為目前使用者沒有資料庫的{0}權限 com.oceanbase.odc.ErrorCodes.ObQueryProfileNotSupported=執行詳情僅在高於或等於 {0} 的 OceanBase 版本可用 com.oceanbase.odc.ErrorCodes.WorksheetEditVersionConflict=有人剛剛修改了這個工作簿，請重新整理頁面以取得最新版本並繼續編輯 -com.oceanbase.odc.ErrorCodes.WorkspaceDatabaseUserTypeMustBeAdmin=工作空間中使用的資料庫用戶類型必須為超級帳號 \ No newline at end of file +com.oceanbase.odc.ErrorCodes.WorkspaceDatabaseUserTypeMustBeAdmin=工作空間中使用的資料庫用戶類型必須為超級帳號 +com.oceanbase.odc.ErrorCodes.UpdateNotAllowed=當前狀態下不允許編輯，請在禁用後重試 +com.oceanbase.odc.ErrorCodes.PauseNotAllowed=當前狀態下不允許禁用，請檢查是否存在執行中的記錄 +com.oceanbase.odc.ErrorCodes.DeleteNotAllowed=在目前狀態下不允許删除 + + + +com.oceanbase.odc.ErrorCodes.UnsupportedSyncTableStructure=結構同步暫不支持 {0} 到 {1} +com.oceanbase.odc.ErrorCodes.ScheduleIntervalTooShort=執行間隔設定過短，請重新設定。最小間隔應為：{0} 秒 \ No newline at end of file diff --git a/server/odc-core/src/test/java/com/oceanbase/odc/core/authority/ComposedPermissionTest.java b/server/odc-core/src/test/java/com/oceanbase/odc/core/authority/ComposedPermissionTest.java new file mode 100644 index 0000000000..907b5990bf --- /dev/null +++ b/server/odc-core/src/test/java/com/oceanbase/odc/core/authority/ComposedPermissionTest.java @@ -0,0 +1,122 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.core.authority; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import org.junit.Assert; +import org.junit.Test; + +import com.oceanbase.odc.core.authority.model.DefaultSecurityResource; +import com.oceanbase.odc.core.authority.permission.ComposedPermission; +import com.oceanbase.odc.core.authority.permission.Permission; +import com.oceanbase.odc.core.authority.permission.ProjectPermission; +import com.oceanbase.odc.core.authority.permission.ResourceRoleBasedPermission; +import com.oceanbase.odc.core.shared.constant.ResourceRoleName; + +/** + * @Author: Lebie + * @Date: 2024/11/13 17:45 + * @Description: [] + */ +public class ComposedPermissionTest { + @Test + public void implies_HasResourcePermission_impliesTrue() { + ComposedPermission thisPermission = new ComposedPermission(Arrays.asList(getResourcePermission("*", "DBA"))); + ComposedPermission thatPermission = new ComposedPermission( + Arrays.asList(getResourceRolePermission("1", Arrays.asList(ResourceRoleName.DBA)), + getResourcePermission("1", "DBA"))); + Assert.assertTrue(thisPermission.implies(thatPermission)); + } + + @Test + public void implies_HasResourceRolePermission_impliesTrue() { + ComposedPermission thisPermission = new ComposedPermission( + Arrays.asList(getResourceRolePermission("1", Arrays.asList(ResourceRoleName.DBA)))); + ComposedPermission thatPermission = new ComposedPermission( + Arrays.asList(getResourceRolePermission("1", Arrays.asList(ResourceRoleName.DBA)), + getResourcePermission("1", "DBA"))); + Assert.assertTrue(thisPermission.implies(thatPermission)); + } + + @Test + public void implies_HasBothPermissions_impliesTrue() { + ComposedPermission thisPermission = new ComposedPermission(Arrays.asList(getResourcePermission("*", "DBA"), + getResourceRolePermission("1", Arrays.asList(ResourceRoleName.DBA)))); + ComposedPermission thatPermission = new ComposedPermission( + Arrays.asList(getResourceRolePermission("1", Arrays.asList(ResourceRoleName.DBA)), + getResourcePermission("1", "DBA"))); + Assert.assertTrue(thisPermission.implies(thatPermission)); + } + + @Test + public void implies_HasWrongAndRightPermissions_impliesTrue() { + ComposedPermission thisPermission = new ComposedPermission(Arrays.asList(getResourcePermission("*", "OWNER"), + getResourceRolePermission("1", Arrays.asList(ResourceRoleName.DBA)))); + ComposedPermission thatPermission = new ComposedPermission( + Arrays.asList(getResourceRolePermission("1", Arrays.asList(ResourceRoleName.DBA)), + getResourcePermission("1", "DBA"))); + Assert.assertTrue(thisPermission.implies(thatPermission)); + } + + @Test + public void implies_ImpliesNoPermission_impliesFalse() { + ComposedPermission thisPermission = new ComposedPermission(Arrays.asList(getResourcePermission("*", "OWNER"))); + ComposedPermission thatPermission = new ComposedPermission(Collections.emptyList()); + Assert.assertTrue(thisPermission.implies(thatPermission)); + } + + + @Test + public void implies_HasWrongResourceRolePermission_impliesFalse() { + ComposedPermission thisPermission = new ComposedPermission( + Arrays.asList(getResourceRolePermission("1", Arrays.asList(ResourceRoleName.OWNER)))); + ComposedPermission thatPermission = new ComposedPermission( + Arrays.asList(getResourceRolePermission("1", Arrays.asList(ResourceRoleName.DBA)), + getResourcePermission("1", "DBA"))); + Assert.assertFalse(thisPermission.implies(thatPermission)); + } + + @Test + public void implies_HasWrongResourcePermission_impliesFalse() { + ComposedPermission thisPermission = new ComposedPermission( + Arrays.asList(getResourcePermission("*", "OWNER"))); + ComposedPermission thatPermission = new ComposedPermission( + Arrays.asList(getResourceRolePermission("1", Arrays.asList(ResourceRoleName.DBA)), + getResourcePermission("1", "DBA"))); + Assert.assertFalse(thisPermission.implies(thatPermission)); + } + + @Test + public void implies_HasNoPermission_impliesFalse() { + ComposedPermission thisPermission = new ComposedPermission(Collections.emptyList()); + ComposedPermission thatPermission = new ComposedPermission( + Arrays.asList(getResourceRolePermission("1", Arrays.asList(ResourceRoleName.DBA)), + getResourcePermission("1", "DBA"))); + Assert.assertFalse(thisPermission.implies(thatPermission)); + } + + private Permission getResourceRolePermission(String resourceId, List resourceRoles) { + return new ResourceRoleBasedPermission(new DefaultSecurityResource(resourceId, "ODC_PROJECT"), + resourceRoles); + } + + private Permission getResourcePermission(String resourceId, String actions) { + return new ProjectPermission(new DefaultSecurityResource(resourceId, "ODC_PROJECT"), actions); + } +} diff --git a/server/odc-core/src/test/java/com/oceanbase/odc/core/authority/tool/TestPermissionProvider.java b/server/odc-core/src/test/java/com/oceanbase/odc/core/authority/tool/TestPermissionProvider.java index 39b556a89d..75b009ac7b 100644 --- a/server/odc-core/src/test/java/com/oceanbase/odc/core/authority/tool/TestPermissionProvider.java +++ b/server/odc-core/src/test/java/com/oceanbase/odc/core/authority/tool/TestPermissionProvider.java @@ -15,14 +15,15 @@ */ package com.oceanbase.odc.core.authority.tool; +import java.util.Arrays; import java.util.Collection; import com.oceanbase.odc.core.authority.model.SecurityResource; +import com.oceanbase.odc.core.authority.permission.ComposedPermission; import com.oceanbase.odc.core.authority.permission.ConnectionPermission; import com.oceanbase.odc.core.authority.permission.DatabasePermission; import com.oceanbase.odc.core.authority.permission.Permission; import com.oceanbase.odc.core.authority.permission.PermissionProvider; -import com.oceanbase.odc.core.authority.permission.PrivateConnectionPermission; import com.oceanbase.odc.core.authority.permission.ResourcePermission; import com.oceanbase.odc.core.authority.permission.ResourceRoleBasedPermission; import com.oceanbase.odc.core.shared.constant.ResourceType; @@ -41,8 +42,6 @@ public class TestPermissionProvider implements PermissionProvider { public Permission getPermissionByActions(SecurityResource resource, Collection actions) { if (ResourceType.ODC_CONNECTION.name().equals(resource.resourceType())) { return new ConnectionPermission(resource.resourceId(), String.join(",", actions)); - } else if (ResourceType.ODC_PRIVATE_CONNECTION.name().equals(resource.resourceType())) { - return new PrivateConnectionPermission(resource.resourceId(), String.join(",", actions)); } else if (ResourceType.ODC_DATABASE.name().equals(resource.resourceType())) { return new DatabasePermission(resource.resourceId(), String.join(",", actions)); } @@ -54,5 +53,12 @@ public Permission getPermissionByResourceRoles(SecurityResource resource, Collec return new ResourceRoleBasedPermission(resource, String.join(",", resourceRoles)); } + @Override + public Permission getPermissionByActionsAndResourceRoles(SecurityResource resource, Collection actions, + Collection resourceRoles) { + return new ComposedPermission(Arrays.asList(getPermissionByActions(resource, actions), + getPermissionByResourceRoles(resource, resourceRoles))); + } + } diff --git a/server/odc-core/src/test/java/com/oceanbase/odc/core/datasource/SingleConnectionDataSourceTest.java b/server/odc-core/src/test/java/com/oceanbase/odc/core/datasource/SingleConnectionDataSourceTest.java index 1fe5d3fb70..8e265bc1ee 100644 --- a/server/odc-core/src/test/java/com/oceanbase/odc/core/datasource/SingleConnectionDataSourceTest.java +++ b/server/odc-core/src/test/java/com/oceanbase/odc/core/datasource/SingleConnectionDataSourceTest.java @@ -15,6 +15,15 @@ */ package com.oceanbase.odc.core.datasource; +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; @@ -140,6 +149,47 @@ public void testGetConnectionLock() throws Exception { Assert.assertTrue(exceptions.get(0) instanceof ConflictException); } + @Test + public void testKeepAlive_Enabled() throws SQLException, InterruptedException { + Connection mockConnection = mock(Connection.class); + Statement mockStatement = mock(Statement.class); + when(mockConnection.createStatement()).thenReturn(mockStatement); + doNothing().when(mockConnection).close(); + try (SingleConnectionDataSource spiedDataSource = spy(new SingleConnectionDataSource(false, true, 100) { + @Override + public Connection getConnection() { + return mockConnection; + } + })) { + doReturn(mockConnection).when(spiedDataSource).getConnection(); + + Thread.sleep(1000); + + verify(mockStatement, atLeastOnce()).execute(spiedDataSource.getKeepAliveSql()); + } + } + + @Test + public void testKeepAlive_Disabled() throws SQLException, InterruptedException { + Connection mockConnection = mock(Connection.class); + Statement mockStatement = mock(Statement.class); + when(mockConnection.createStatement()).thenReturn(mockStatement); + doNothing().when(mockConnection).close(); + try (SingleConnectionDataSource spiedDataSource = spy(new SingleConnectionDataSource(false, false, 100) { + @Override + public Connection getConnection() { + return mockConnection; + } + })) { + doReturn(mockConnection).when(spiedDataSource).getConnection(); + + Thread.sleep(1000); + + verify(mockStatement, never()).execute(spiedDataSource.getKeepAliveSql()); + } + + } + private void checkConnection(SingleConnectionDataSource dataSource) throws SQLException { try (Connection connection = dataSource.getConnection()) { try (Statement statement = connection.createStatement()) { diff --git a/server/odc-core/src/test/java/com/oceanbase/odc/core/sql/execute/TestSessionOperations.java b/server/odc-core/src/test/java/com/oceanbase/odc/core/sql/execute/TestSessionOperations.java index 023a4ee67f..68c73a3721 100644 --- a/server/odc-core/src/test/java/com/oceanbase/odc/core/sql/execute/TestSessionOperations.java +++ b/server/odc-core/src/test/java/com/oceanbase/odc/core/sql/execute/TestSessionOperations.java @@ -50,4 +50,13 @@ public void killQuery(@NonNull Connection connection, @NonNull String connection } } + @Override + public String getKillQuerySql(@NonNull String connectionId) { + return "KILL QUERY " + connectionId; + } + + @Override + public String getKillSessionSql(@NonNull String connectionId) { + return "KILL " + connectionId; + } } diff --git a/server/odc-migrate/pom.xml b/server/odc-migrate/pom.xml index 34470f27de..9b79fcade3 100644 --- a/server/odc-migrate/pom.xml +++ b/server/odc-migrate/pom.xml @@ -5,7 +5,7 @@ com.oceanbase odc-parent - 4.3.2-SNAPSHOT + 4.3.3-SNAPSHOT ../../pom.xml odc-migrate diff --git a/server/odc-migrate/src/main/resources/init-config/default-audit-event-meta.yml b/server/odc-migrate/src/main/resources/init-config/default-audit-event-meta.yml index a5e85ac1de..dac3e86d48 100644 --- a/server/odc-migrate/src/main/resources/init-config/default-audit-event-meta.yml +++ b/server/odc-migrate/src/main/resources/init-config/default-audit-event-meta.yml @@ -555,3 +555,19 @@ in_connection: 0 enabled: 1 id: 100 + +- type: "PROJECT_MANAGEMENT" + action: "ARCHIVE_PROJECT" + method_signature: "com.oceanbase.odc.server.web.controller.v2.ProjectController.setArchived" + sid_extract_expression: "" + in_connection: 0 + enabled: 1 + id: 101 + +- type: "PROJECT_MANAGEMENT" + action: "DELETE_PROJECT" + method_signature: "com.oceanbase.odc.server.web.controller.v2.ProjectController.batchDelete" + sid_extract_expression: "" + in_connection: 0 + enabled: 1 + id: 102 diff --git a/server/odc-migrate/src/main/resources/init-config/init/regulation-rule-applying.yaml b/server/odc-migrate/src/main/resources/init-config/init/regulation-rule-applying.yaml index 26050692c1..1db76776b7 100644 --- a/server/odc-migrate/src/main/resources/init-config/init/regulation-rule-applying.yaml +++ b/server/odc-migrate/src/main/resources/init-config/init/regulation-rule-applying.yaml @@ -6,6 +6,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -16,6 +17,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -26,6 +28,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.max-return-rows.metadata.name}":1000}' @@ -36,6 +39,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.allow-sql-types.metadata.name}":["SELECT", "EXPLAIN", "SHOW", "HELP", "START_TRANS", "COMMIT", "ROLLBACK", "SORT", "DESC"]}' @@ -46,6 +50,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.max-execute-sqls.metadata.name}":1000}' @@ -56,6 +61,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -66,6 +72,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -76,6 +83,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -86,6 +94,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -97,6 +106,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -107,6 +117,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -117,6 +128,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.max-return-rows.metadata.name}":100000}' @@ -127,6 +139,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.allow-sql-types.metadata.name}":["UPDATE", "DELETE", "INSERT", "SELECT", "CREATE", "DROP", "ALTER", "REPLACE", "SET", "USE_DB", "EXPLAIN", "SHOW", "HELP", "START_TRANS", "COMMIT", "ROLLBACK", "SORT", "DESC", "TRUNCATE", "OTHERS"]}' @@ -137,6 +150,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.max-execute-sqls.metadata.name}":1000}' @@ -147,6 +161,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -157,6 +172,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -167,6 +183,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -178,6 +195,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -188,6 +206,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -198,6 +217,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.max-return-rows.metadata.name}":100000}' @@ -208,6 +228,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.allow-sql-types.metadata.name}":["UPDATE", "DELETE", "INSERT", "SELECT", "SET", "REPLACE", "EXPLAIN", "SHOW", "HELP", "START_TRANS", "COMMIT", "ROLLBACK", "SORT", "DESC", "TRUNCATE"]}' @@ -218,6 +239,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.max-execute-sqls.metadata.name}":1000}' @@ -228,6 +250,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -238,6 +261,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -248,6 +272,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -259,6 +284,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -269,6 +295,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -279,6 +306,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.max-return-rows.metadata.name}":1000}' @@ -289,6 +317,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.allow-sql-types.metadata.name}":["SELECT", "EXPLAIN", "SHOW", "HELP", "START_TRANS", "COMMIT", "ROLLBACK", "SORT", "DESC"]}' @@ -299,6 +328,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.max-execute-sqls.metadata.name}":1000}' @@ -309,6 +339,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -319,6 +350,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -329,6 +361,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -340,6 +373,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -350,6 +384,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -360,6 +395,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -370,6 +406,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' propertiesJson: '{}' - enabled: 1 level: 0 @@ -382,6 +419,26 @@ - 'ORACLE' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' +- enabled: 1 + level: 1 + rulesetName: ${com.oceanbase.odc.builtin-resource.regulation.ruleset.default-sit-ruleset.name} + ruleName: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.create-table-like-exists.name} + appliedDialectTypes: + - 'OB_MYSQL' + - 'MYSQL' + - 'ODP_SHARDING_OB_MYSQL' + propertiesJson: '{}' +- enabled: 1 + level: 1 + rulesetName: ${com.oceanbase.odc.builtin-resource.regulation.ruleset.default-sit-ruleset.name} + ruleName: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.create-table-as-exists.name} + appliedDialectTypes: + - 'OB_MYSQL' + - 'OB_ORACLE' + - 'ORACLE' + - 'MYSQL' + - 'ODP_SHARDING_OB_MYSQL' + propertiesJson: '{}' - enabled: 1 level: 0 rulesetName: ${com.oceanbase.odc.builtin-resource.regulation.ruleset.default-sit-ruleset.name} @@ -389,6 +446,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -399,6 +457,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -409,6 +468,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -419,6 +479,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -429,6 +490,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -439,6 +501,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-in-expr.max-in-expr-count}":200}' @@ -449,6 +512,8 @@ appliedDialectTypes: - 'OB_MYSQL' - 'MYSQL' + - 'OB_ORACLE' + - 'ORACLE' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-sql-affected-rows.allowed-max-sql-affected-count}":1000}' - enabled: 1 level: 0 @@ -465,6 +530,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-table-join.max-join-table-count}":10}' @@ -475,6 +541,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-index-keys.max-index-count}":10}' @@ -485,6 +552,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-column-refs-in-index.max-column-ref-count}":100}' @@ -495,6 +563,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' - enabled: 1 @@ -504,6 +573,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-columns.max-column-definition-count}":100}' @@ -514,6 +584,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-long-char-length.max-char-length}":1000}' @@ -524,6 +595,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -534,6 +606,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -544,6 +617,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -554,6 +628,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.table-name-in-black-list.black-list}":[]}' @@ -582,6 +657,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-datatypes.allowed-datatypes}":["int", "varchar2", "varchar", "number", "float", "bigint"]}' @@ -601,6 +677,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-column-refs-in-primary-key.max-column-ref-count}":100}' @@ -611,6 +688,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-index-naming.name-pattern}":null}' @@ -621,6 +699,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-unique-index-naming.name-pattern}":null}' @@ -631,6 +710,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -641,6 +721,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -660,6 +741,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -670,6 +752,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -680,6 +763,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-is-nullable.nullable-datatype-list}":[]}' @@ -690,6 +774,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-default-value-exists.no-default-value-datatype-list}":[]}' @@ -700,6 +785,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -710,6 +796,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-name-in-black-list.black-list}":[]}' @@ -719,6 +806,7 @@ ruleName: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-column-name-case.name} appliedDialectTypes: - 'OB_ORACLE' + - 'ORACLE' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-column-name-case.is-uppercase}":false, "${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-column-name-case.is-lowercase}":false}' - enabled: 1 level: 0 @@ -726,6 +814,7 @@ ruleName: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-name-case.name} appliedDialectTypes: - 'OB_ORACLE' + - 'ORACLE' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-name-case.is-uppercase}":false, "${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-name-case.is-lowercase}":false}' - enabled: 1 level: 0 @@ -743,6 +832,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -753,6 +843,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.missing-required-columns.column-names}":[]}' @@ -772,6 +863,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-alter-statement.max-alter-count}":10}' @@ -782,6 +874,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -792,6 +885,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-naming.name-pattern}":null}' @@ -802,6 +896,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.prohibited-datatype-exists.datatype-names}":[]}' @@ -812,6 +907,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-index-datatypes.allowed-datatypes}":["int", "varchar2", "number", "float", "bigint"]}' @@ -831,6 +927,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-drop-object-types.allowed-object-types}":[]}' @@ -841,6 +938,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -851,6 +949,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -861,6 +960,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -871,6 +971,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' propertiesJson: '{}' - enabled: 1 level: 0 @@ -883,6 +984,26 @@ - 'ORACLE' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' +- enabled: 1 + level: 1 + rulesetName: ${com.oceanbase.odc.builtin-resource.regulation.ruleset.default-dev-ruleset.name} + ruleName: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.create-table-like-exists.name} + appliedDialectTypes: + - 'OB_MYSQL' + - 'MYSQL' + - 'ODP_SHARDING_OB_MYSQL' + propertiesJson: '{}' +- enabled: 1 + level: 1 + rulesetName: ${com.oceanbase.odc.builtin-resource.regulation.ruleset.default-dev-ruleset.name} + ruleName: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.create-table-as-exists.name} + appliedDialectTypes: + - 'OB_MYSQL' + - 'OB_ORACLE' + - 'ORACLE' + - 'MYSQL' + - 'ODP_SHARDING_OB_MYSQL' + propertiesJson: '{}' - enabled: 1 level: 0 rulesetName: ${com.oceanbase.odc.builtin-resource.regulation.ruleset.default-dev-ruleset.name} @@ -890,6 +1011,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -900,6 +1022,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -910,6 +1033,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -920,6 +1044,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -930,6 +1055,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -940,6 +1066,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-in-expr.max-in-expr-count}":200}' @@ -950,6 +1077,8 @@ appliedDialectTypes: - 'OB_MYSQL' - 'MYSQL' + - 'OB_ORACLE' + - 'ORACLE' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-sql-affected-rows.allowed-max-sql-affected-count}":10000}' - enabled: 1 level: 0 @@ -966,6 +1095,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-table-join.max-join-table-count}":10}' @@ -976,6 +1106,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-index-keys.max-index-count}":10}' @@ -986,6 +1117,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-column-refs-in-index.max-column-ref-count}":100}' @@ -996,6 +1128,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' - enabled: 1 @@ -1005,6 +1138,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-columns.max-column-definition-count}":100}' @@ -1015,6 +1149,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-long-char-length.max-char-length}":1000}' @@ -1025,6 +1160,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -1035,6 +1171,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -1045,6 +1182,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -1055,6 +1193,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.table-name-in-black-list.black-list}":[]}' @@ -1083,6 +1222,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-datatypes.allowed-datatypes}":["int", "varchar2", "varchar", "number", "float", "bigint"]}' @@ -1102,6 +1242,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-column-refs-in-primary-key.max-column-ref-count}":100}' @@ -1112,6 +1253,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-index-naming.name-pattern}":null}' @@ -1122,6 +1264,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-unique-index-naming.name-pattern}":null}' @@ -1132,6 +1275,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -1142,6 +1286,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -1161,6 +1306,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -1171,6 +1317,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -1181,6 +1328,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-is-nullable.nullable-datatype-list}":[]}' @@ -1191,6 +1339,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-default-value-exists.no-default-value-datatype-list}":[]}' @@ -1201,6 +1350,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -1211,6 +1361,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-name-in-black-list.black-list}":[]}' @@ -1220,6 +1371,7 @@ ruleName: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-column-name-case.name} appliedDialectTypes: - 'OB_ORACLE' + - 'ORACLE' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-column-name-case.is-uppercase}":false, "${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-column-name-case.is-lowercase}":false}' - enabled: 1 level: 0 @@ -1227,6 +1379,7 @@ ruleName: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-name-case.name} appliedDialectTypes: - 'OB_ORACLE' + - 'ORACLE' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-name-case.is-uppercase}":false, "${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-name-case.is-lowercase}":false}' - enabled: 1 level: 0 @@ -1244,6 +1397,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -1254,6 +1408,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.missing-required-columns.column-names}":[]}' @@ -1273,6 +1428,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-alter-statement.max-alter-count}":10}' @@ -1283,6 +1439,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -1293,6 +1450,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-naming.name-pattern}":null}' @@ -1303,6 +1461,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.prohibited-datatype-exists.datatype-names}":[]}' @@ -1313,6 +1472,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-index-datatypes.allowed-datatypes}":["int", "varchar2", "number", "float", "bigint"]}' @@ -1332,6 +1492,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-drop-object-types.allowed-object-types}":[]}' @@ -1342,6 +1503,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -1352,6 +1514,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -1362,6 +1525,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -1372,6 +1536,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' propertiesJson: '{}' - enabled: 1 level: 1 @@ -1384,6 +1549,26 @@ - 'ORACLE' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' +- enabled: 1 + level: 1 + rulesetName: ${com.oceanbase.odc.builtin-resource.regulation.ruleset.default-prod-ruleset.name} + ruleName: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.create-table-like-exists.name} + appliedDialectTypes: + - 'OB_MYSQL' + - 'MYSQL' + - 'ODP_SHARDING_OB_MYSQL' + propertiesJson: '{}' +- enabled: 1 + level: 1 + rulesetName: ${com.oceanbase.odc.builtin-resource.regulation.ruleset.default-prod-ruleset.name} + ruleName: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.create-table-as-exists.name} + appliedDialectTypes: + - 'OB_MYSQL' + - 'OB_ORACLE' + - 'ORACLE' + - 'MYSQL' + - 'ODP_SHARDING_OB_MYSQL' + propertiesJson: '{}' - enabled: 1 level: 1 rulesetName: ${com.oceanbase.odc.builtin-resource.regulation.ruleset.default-prod-ruleset.name} @@ -1391,6 +1576,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -1401,6 +1587,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -1411,6 +1598,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -1421,6 +1609,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -1431,6 +1620,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -1441,6 +1631,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-in-expr.max-in-expr-count}":200}' @@ -1451,6 +1642,8 @@ appliedDialectTypes: - 'OB_MYSQL' - 'MYSQL' + - 'OB_ORACLE' + - 'ORACLE' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-sql-affected-rows.allowed-max-sql-affected-count}":1000}' - enabled: 1 level: 1 @@ -1467,6 +1660,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-table-join.max-join-table-count}":10}' @@ -1477,6 +1671,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-index-keys.max-index-count}":10}' @@ -1487,6 +1682,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-column-refs-in-index.max-column-ref-count}":100}' @@ -1497,6 +1693,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' - enabled: 1 @@ -1506,6 +1703,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-columns.max-column-definition-count}":100}' @@ -1516,6 +1714,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-long-char-length.max-char-length}":1000}' @@ -1526,6 +1725,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -1536,6 +1736,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -1546,6 +1747,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -1556,6 +1758,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.table-name-in-black-list.black-list}":[]}' @@ -1584,6 +1787,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-datatypes.allowed-datatypes}":["int", "varchar2", "varchar", "number", "float", "bigint"]}' @@ -1603,6 +1807,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-column-refs-in-primary-key.max-column-ref-count}":100}' @@ -1613,6 +1818,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-index-naming.name-pattern}":null}' @@ -1623,6 +1829,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-unique-index-naming.name-pattern}":null}' @@ -1633,6 +1840,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -1643,6 +1851,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -1662,6 +1871,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -1672,6 +1882,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -1682,6 +1893,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-is-nullable.nullable-datatype-list}":[]}' @@ -1692,6 +1904,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-default-value-exists.no-default-value-datatype-list}":[]}' @@ -1702,6 +1915,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -1712,6 +1926,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-name-in-black-list.black-list}":[]}' @@ -1721,6 +1936,7 @@ ruleName: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-column-name-case.name} appliedDialectTypes: - 'OB_ORACLE' + - 'ORACLE' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-column-name-case.is-uppercase}":false, "${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-column-name-case.is-lowercase}":false}' - enabled: 1 level: 0 @@ -1728,6 +1944,7 @@ ruleName: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-name-case.name} appliedDialectTypes: - 'OB_ORACLE' + - 'ORACLE' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-name-case.is-uppercase}":false, "${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-name-case.is-lowercase}":false}' - enabled: 1 level: 0 @@ -1745,6 +1962,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -1755,6 +1973,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.missing-required-columns.column-names}":[]}' @@ -1774,6 +1993,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-alter-statement.max-alter-count}":10}' @@ -1784,6 +2004,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -1794,6 +2015,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-naming.name-pattern}":null}' @@ -1804,6 +2026,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.prohibited-datatype-exists.datatype-names}":[]}' @@ -1814,6 +2037,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-index-datatypes.allowed-datatypes}":["int", "varchar2", "number", "float", "bigint"]}' @@ -1833,6 +2057,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-drop-object-types.allowed-object-types}":[]}' @@ -1843,6 +2068,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -1853,6 +2079,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -1863,6 +2090,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' propertiesJson: '{}' - enabled: 1 level: 1 @@ -1875,6 +2103,26 @@ - 'ORACLE' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' +- enabled: 1 + level: 1 + rulesetName: ${com.oceanbase.odc.builtin-resource.regulation.ruleset.default-default-ruleset.name} + ruleName: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.create-table-like-exists.name} + appliedDialectTypes: + - 'OB_MYSQL' + - 'MYSQL' + - 'ODP_SHARDING_OB_MYSQL' + propertiesJson: '{}' +- enabled: 1 + level: 1 + rulesetName: ${com.oceanbase.odc.builtin-resource.regulation.ruleset.default-default-ruleset.name} + ruleName: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.create-table-as-exists.name} + appliedDialectTypes: + - 'OB_MYSQL' + - 'OB_ORACLE' + - 'ORACLE' + - 'MYSQL' + - 'ODP_SHARDING_OB_MYSQL' + propertiesJson: '{}' - enabled: 1 level: 1 rulesetName: ${com.oceanbase.odc.builtin-resource.regulation.ruleset.default-default-ruleset.name} @@ -1882,6 +2130,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -1892,6 +2141,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -1902,6 +2152,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -1912,6 +2163,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -1922,6 +2174,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -1932,6 +2185,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-in-expr.max-in-expr-count}":200}' @@ -1942,6 +2196,8 @@ appliedDialectTypes: - 'OB_MYSQL' - 'MYSQL' + - 'OB_ORACLE' + - 'ORACLE' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-sql-affected-rows.allowed-max-sql-affected-count}":1000}' - enabled: 1 level: 1 @@ -1958,6 +2214,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-table-join.max-join-table-count}":10}' @@ -1968,6 +2225,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-index-keys.max-index-count}":10}' @@ -1978,6 +2236,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-column-refs-in-index.max-column-ref-count}":100}' @@ -1988,6 +2247,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' - enabled: 1 @@ -1997,6 +2257,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-columns.max-column-definition-count}":100}' @@ -2007,6 +2268,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-long-char-length.max-char-length}":1000}' @@ -2017,6 +2279,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -2027,6 +2290,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -2037,6 +2301,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -2047,6 +2312,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.table-name-in-black-list.black-list}":[]}' @@ -2075,6 +2341,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-datatypes.allowed-datatypes}":["int", "varchar2", "varchar", "number", "float", "bigint"]}' @@ -2094,6 +2361,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-column-refs-in-primary-key.max-column-ref-count}":100}' @@ -2104,6 +2372,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-index-naming.name-pattern}":null}' @@ -2114,6 +2383,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-unique-index-naming.name-pattern}":null}' @@ -2124,6 +2394,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -2134,6 +2405,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -2153,6 +2425,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -2163,6 +2436,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -2173,6 +2447,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-is-nullable.nullable-datatype-list}":[]}' @@ -2183,6 +2458,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-default-value-exists.no-default-value-datatype-list}":[]}' @@ -2193,6 +2469,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -2203,6 +2480,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-name-in-black-list.black-list}":[]}' @@ -2212,6 +2490,7 @@ ruleName: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-column-name-case.name} appliedDialectTypes: - 'OB_ORACLE' + - 'ORACLE' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-column-name-case.is-uppercase}":false, "${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-column-name-case.is-lowercase}":false}' - enabled: 1 level: 0 @@ -2219,6 +2498,7 @@ ruleName: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-name-case.name} appliedDialectTypes: - 'OB_ORACLE' + - 'ORACLE' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-name-case.is-uppercase}":false, "${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-name-case.is-lowercase}":false}' - enabled: 1 level: 0 @@ -2236,6 +2516,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -2246,6 +2527,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.missing-required-columns.column-names}":[]}' @@ -2265,6 +2547,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-alter-statement.max-alter-count}":10}' @@ -2275,6 +2558,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{}' @@ -2285,6 +2569,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-naming.name-pattern}":null}' @@ -2295,6 +2580,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.prohibited-datatype-exists.datatype-names}":[]}' @@ -2305,6 +2591,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-index-datatypes.allowed-datatypes}":["int", "varchar2", "number", "float", "bigint"]}' @@ -2324,6 +2611,7 @@ appliedDialectTypes: - 'OB_MYSQL' - 'OB_ORACLE' + - 'ORACLE' - 'MYSQL' - 'ODP_SHARDING_OB_MYSQL' propertiesJson: '{"${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-drop-object-types.allowed-object-types}":[]}' \ No newline at end of file diff --git a/server/odc-migrate/src/main/resources/init-config/init/regulation-rule-metadata.yaml b/server/odc-migrate/src/main/resources/init-config/init/regulation-rule-metadata.yaml index 0a84e57a55..c9a686d128 100644 --- a/server/odc-migrate/src/main/resources/init-config/init/regulation-rule-metadata.yaml +++ b/server/odc-migrate/src/main/resources/init-config/init/regulation-rule-metadata.yaml @@ -16,6 +16,8 @@ value: DORIS - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE - id: 2 name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.not-allowed-export-resultset.name} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.not-allowed-export-resultset.description} @@ -36,6 +38,8 @@ value: DORIS - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE - id: 3 name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.max-return-rows.name} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.max-return-rows.description} @@ -54,6 +58,8 @@ value: DORIS - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE propertyMetadatas: - name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.max-return-rows.metadata.name} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.max-return-rows.metadata.description} @@ -81,6 +87,8 @@ value: DORIS - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE propertyMetadatas: - name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.allow-sql-types.metadata.name} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.allow-sql-types.metadata.description} @@ -94,8 +102,10 @@ - CREATE - DROP - ALTER + - ALTER_SESSION - REPLACE - SET + - SET_SESSION - USE_DB - EXPLAIN - SHOW @@ -106,6 +116,8 @@ - SORT - DESC - TRUNCATE + - CALL + - COMMENT_ON - OTHERS candidates: - UPDATE @@ -115,8 +127,10 @@ - CREATE - DROP - ALTER + - ALTER_SESSION - REPLACE - SET + - SET_SESSION - USE_DB - EXPLAIN - SHOW @@ -127,6 +141,8 @@ - SORT - DESC - TRUNCATE + - CALL + - COMMENT_ON - OTHERS - id: 5 name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.max-execute-sqls.name} @@ -148,6 +164,8 @@ value: DORIS - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE propertyMetadatas: - name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.max-execute-sqls.metadata.name} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.max-execute-sqls.metadata.description} @@ -177,13 +195,15 @@ value: DORIS - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE - id: 7 name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.index-column-calculation.name} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.index-column-calculation.description} type: SQL_CHECK builtIn: 1 - labels: + labels: - label: SUB_TYPE value: DQL - label: SUB_TYPE @@ -196,12 +216,14 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE - id: 8 name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.index-column-fuzzy-match.name} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.index-column-fuzzy-match.description} type: SQL_CHECK builtIn: 1 - labels: + labels: - label: SUB_TYPE value: DQL - label: SUB_TYPE @@ -220,12 +242,14 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE - id: 9 name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-not-null-exists-not-in.name} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-not-null-exists-not-in.description} type: SQL_CHECK builtIn: 1 - labels: + labels: - label: SUB_TYPE value: DQL - label: SUB_TYPE @@ -244,12 +268,14 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE - id: 10 name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-specific-column-exists.name} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-specific-column-exists.description} type: SQL_CHECK builtIn: 1 - labels: + labels: - label: SUB_TYPE value: DML - label: SUB_TYPE @@ -262,12 +288,14 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE - id: 11 name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-valid-where-clause.name} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-valid-where-clause.description} type: SQL_CHECK builtIn: 1 - labels: + labels: - label: SUB_TYPE value: DML - label: SUB_TYPE @@ -284,6 +312,8 @@ value: DORIS - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE - id: 12 name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-where-clause-exists.name} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-where-clause-exists.description} @@ -306,12 +336,14 @@ value: DORIS - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE - id: 13 name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-in-expr.name} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-in-expr.description} type: SQL_CHECK builtIn: 1 - labels: + labels: - label: SUB_TYPE value: DQL - label: SUB_TYPE @@ -330,19 +362,21 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL - propertyMetadatas: + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE + propertyMetadatas: - name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-in-expr.max-in-expr-count} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-in-expr.max-in-expr-count.description} type: INTEGER componentType: INPUT_NUMBER - defaultValues: + defaultValues: - !!int 200 - id: 14 name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-table-join.name} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-table-join.description} type: SQL_CHECK builtIn: 1 - labels: + labels: - label: SUB_TYPE value: DQL - label: SUB_TYPE @@ -355,19 +389,21 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL - propertyMetadatas: + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE + propertyMetadatas: - name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-table-join.max-join-table-count} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-table-join.max-join-table-count.description} type: INTEGER componentType: INPUT_NUMBER - defaultValues: + defaultValues: - !!int 10 - id: 15 name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-index-keys.name} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-index-keys.description} type: SQL_CHECK builtIn: 1 - labels: + labels: - label: SUB_TYPE value: DDL - label: SUB_TYPE @@ -380,19 +416,21 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL - propertyMetadatas: + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE + propertyMetadatas: - name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-index-keys.max-index-count} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-index-keys.max-index-count.description} type: INTEGER componentType: INPUT_NUMBER - defaultValues: + defaultValues: - !!int 10 - id: 16 name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-column-refs-in-index.name} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-column-refs-in-index.description} type: SQL_CHECK builtIn: 1 - labels: + labels: - label: SUB_TYPE value: DDL - label: SUB_TYPE @@ -409,12 +447,14 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL - propertyMetadatas: + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE + propertyMetadatas: - name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-column-refs-in-index.max-column-ref-count} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-column-refs-in-index.max-column-ref-count.description} type: INTEGER componentType: INPUT_NUMBER - defaultValues: + defaultValues: - !!int 100 - id: 17 name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.prefer-local-index.name} @@ -425,19 +465,21 @@ - label: SUB_TYPE value: DDL - label: SUB_TYPE - value: TABLE + value: TABLE - label: SUPPORTED_DIALECT_TYPE value: OB_MYSQL - label: SUPPORTED_DIALECT_TYPE value: OB_ORACLE - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE - id: 18 name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-columns.name} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-columns.description} type: SQL_CHECK builtIn: 1 - labels: + labels: - label: SUB_TYPE value: DDL - label: SUB_TYPE @@ -450,19 +492,21 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL - propertyMetadatas: + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE + propertyMetadatas: - name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-columns.max-column-definition-count} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-columns.max-column-definition-count.description} type: INTEGER componentType: INPUT_NUMBER - defaultValues: + defaultValues: - !!int 100 - id: 19 name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-long-char-length.name} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-long-char-length.description} type: SQL_CHECK builtIn: 1 - labels: + labels: - label: SUB_TYPE value: DDL - label: SUB_TYPE @@ -477,19 +521,21 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL - propertyMetadatas: + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE + propertyMetadatas: - name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-long-char-length.max-char-length} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-long-char-length.max-char-length.description} type: INTEGER componentType: INPUT_NUMBER - defaultValues: + defaultValues: - !!int 1000 - id: 20 name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.foreign-constraint-exists.name} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.foreign-constraint-exists.description} type: SQL_CHECK builtIn: 1 - labels: + labels: - label: SUB_TYPE value: DDL - label: SUB_TYPE @@ -504,12 +550,14 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE - id: 21 name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-primary-key-exists.name} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-primary-key-exists.description} type: SQL_CHECK builtIn: 1 - labels: + labels: - label: SUB_TYPE value: DDL - label: SUB_TYPE @@ -522,12 +570,14 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE - id: 22 name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-table-comment-exists.name} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-table-comment-exists.description} type: SQL_CHECK builtIn: 1 - labels: + labels: - label: SUB_TYPE value: DDL - label: SUB_TYPE @@ -540,12 +590,14 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE - id: 23 name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.table-name-in-black-list.name} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.table-name-in-black-list.description} type: SQL_CHECK builtIn: 1 - labels: + labels: - label: SUB_TYPE value: DDL - label: SUB_TYPE @@ -560,7 +612,9 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL - propertyMetadatas: + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE + propertyMetadatas: - name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.table-name-in-black-list.black-list} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.table-name-in-black-list.black-list.description} type: STRING_LIST @@ -570,7 +624,7 @@ description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-charset.description} type: SQL_CHECK builtIn: 1 - labels: + labels: - label: SUB_TYPE value: DDL - label: SUB_TYPE @@ -583,7 +637,7 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL - propertyMetadatas: + propertyMetadatas: - name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-charset.allowed-charsets} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-charset.allowed-charsets.description} type: STRING_LIST @@ -593,7 +647,7 @@ description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-collation.description} type: SQL_CHECK builtIn: 1 - labels: + labels: - label: SUB_TYPE value: DDL - label: SUB_TYPE @@ -606,7 +660,7 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL - propertyMetadatas: + propertyMetadatas: - name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-collation.allowed-collations} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-collation.allowed-collations.description} type: STRING_LIST @@ -616,7 +670,7 @@ description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-datatypes.description} type: SQL_CHECK builtIn: 1 - labels: + labels: - label: SUB_TYPE value: DDL - label: SUB_TYPE @@ -631,7 +685,9 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL - propertyMetadatas: + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE + propertyMetadatas: - name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-datatypes.allowed-datatypes} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-datatypes.allowed-datatypes.description} type: STRING_LIST @@ -648,7 +704,7 @@ description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-auto-increment.description} type: SQL_CHECK builtIn: 1 - labels: + labels: - label: SUB_TYPE value: DDL - label: SUB_TYPE @@ -664,7 +720,7 @@ description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-column-refs-in-primary-key.description} type: SQL_CHECK builtIn: 1 - labels: + labels: - label: SUB_TYPE value: DDL - label: SUB_TYPE @@ -679,19 +735,21 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL - propertyMetadatas: + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE + propertyMetadatas: - name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-column-refs-in-primary-key.max-column-ref-count} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-column-refs-in-primary-key.max-column-ref-count.description} type: INTEGER componentType: INPUT_NUMBER - defaultValues: + defaultValues: - !!int 100 - id: 29 name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-index-naming.name} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-index-naming.description} type: SQL_CHECK builtIn: 1 - labels: + labels: - label: SUB_TYPE value: DDL - label: SUB_TYPE @@ -708,7 +766,9 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL - propertyMetadatas: + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE + propertyMetadatas: - name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-index-naming.name-pattern} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-index-naming.name-pattern.description} type: STRING @@ -718,7 +778,7 @@ description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-unique-index-naming.description} type: SQL_CHECK builtIn: 1 - labels: + labels: - label: SUB_TYPE value: DDL - label: SUB_TYPE @@ -735,7 +795,9 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL - propertyMetadatas: + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE + propertyMetadatas: - name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-unique-index-naming.name-pattern} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-unique-index-naming.name-pattern.description} type: STRING @@ -745,7 +807,7 @@ description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-index-name-exists.description} type: SQL_CHECK builtIn: 1 - labels: + labels: - label: SUB_TYPE value: DDL - label: SUB_TYPE @@ -760,12 +822,14 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE - id: 32 name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.zerofill-exists.name} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.zerofill-exists.description} type: SQL_CHECK builtIn: 1 - labels: + labels: - label: SUB_TYPE value: DDL - label: SUB_TYPE @@ -783,7 +847,7 @@ description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-charset-exists.description} type: SQL_CHECK builtIn: 1 - labels: + labels: - label: SUB_TYPE value: DDL - label: SUB_TYPE @@ -798,12 +862,14 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE - id: 34 name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-collation-exists.name} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-collation-exists.description} type: SQL_CHECK builtIn: 1 - labels: + labels: - label: SUB_TYPE value: DDL - label: SUB_TYPE @@ -818,12 +884,14 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE - id: 35 name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-is-nullable.name} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-is-nullable.description} type: SQL_CHECK builtIn: 1 - labels: + labels: - label: SUB_TYPE value: DDL - label: SUB_TYPE @@ -838,7 +906,9 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL - propertyMetadatas: + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE + propertyMetadatas: - name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-is-nullable.nullable-datatype-list} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-is-nullable.nullable-datatype-list.description} type: STRING_LIST @@ -848,7 +918,7 @@ description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-default-value-exists.description} type: SQL_CHECK builtIn: 1 - labels: + labels: - label: SUB_TYPE value: DDL - label: SUB_TYPE @@ -863,7 +933,9 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL - propertyMetadatas: + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE + propertyMetadatas: - name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-default-value-exists.no-default-value-datatype-list} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-default-value-exists.no-default-value-datatype-list.description} type: STRING_LIST @@ -873,7 +945,7 @@ description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.no-column-comment-exists.description} type: SQL_CHECK builtIn: 1 - labels: + labels: - label: SUB_TYPE value: DDL - label: SUB_TYPE @@ -888,12 +960,14 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE - id: 38 name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-name-in-black-list.name} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-name-in-black-list.description} type: SQL_CHECK builtIn: 1 - labels: + labels: - label: SUB_TYPE value: DDL - label: SUB_TYPE @@ -908,7 +982,9 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL - propertyMetadatas: + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE + propertyMetadatas: - name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-name-in-black-list.black-list} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.column-name-in-black-list.black-list.description} type: STRING_LIST @@ -918,7 +994,7 @@ description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-column-name-case.description} type: SQL_CHECK builtIn: 1 - labels: + labels: - label: SUB_TYPE value: DDL - label: SUB_TYPE @@ -927,31 +1003,33 @@ value: ALTER - label: SUPPORTED_DIALECT_TYPE value: OB_ORACLE - propertyMetadatas: + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE + propertyMetadatas: - name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-column-name-case.is-uppercase} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-column-name-case.is-uppercase.description} type: BOOLEAN componentType: RADIO - defaultValues: + defaultValues: - false candidates: - true - - false + - false - name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-column-name-case.is-lowercase} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-column-name-case.is-lowercase.description} type: BOOLEAN componentType: RADIO - defaultValues: + defaultValues: - false candidates: - true - - false + - false - id: 40 name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-name-case.name} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-name-case.description} type: SQL_CHECK builtIn: 1 - labels: + labels: - label: SUB_TYPE value: DDL - label: SUB_TYPE @@ -960,12 +1038,14 @@ value: ALTER - label: SUPPORTED_DIALECT_TYPE value: OB_ORACLE - propertyMetadatas: + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE + propertyMetadatas: - name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-name-case.is-uppercase} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-name-case.is-uppercase.description} type: BOOLEAN componentType: RADIO - defaultValues: + defaultValues: - false candidates: - true @@ -974,7 +1054,7 @@ description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-name-case.is-lowercase.description} type: BOOLEAN componentType: RADIO - defaultValues: + defaultValues: - false candidates: - true @@ -984,7 +1064,7 @@ description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-auto-increment.description} type: SQL_CHECK builtIn: 1 - labels: + labels: - label: SUB_TYPE value: DDL - label: SUB_TYPE @@ -995,19 +1075,19 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL - propertyMetadatas: + propertyMetadatas: - name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-auto-increment.init-value} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-table-auto-increment.init-value.description} type: INTEGER componentType: INPUT_NUMBER - defaultValues: + defaultValues: - !!int 1 - id: 42 name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.select-star-exists.name} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.select-star-exists.description} type: SQL_CHECK builtIn: 1 - labels: + labels: - label: SUB_TYPE value: DQL - label: SUB_TYPE @@ -1020,12 +1100,14 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE - id: 43 name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.missing-required-columns.name} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.missing-required-columns.description} type: SQL_CHECK builtIn: 1 - labels: + labels: - label: SUB_TYPE value: DDL - label: SUB_TYPE @@ -1038,7 +1120,9 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL - propertyMetadatas: + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE + propertyMetadatas: - name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.missing-required-columns.column-names} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.missing-required-columns.column-names.description} type: STRING_LIST @@ -1048,7 +1132,7 @@ description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-auto-increment-unsigned.description} type: SQL_CHECK builtIn: 1 - labels: + labels: - label: SUB_TYPE value: DDL - label: SUB_TYPE @@ -1066,7 +1150,7 @@ description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-alter-statement.description} type: SQL_CHECK builtIn: 1 - labels: + labels: - label: SUB_TYPE value: DDL - label: SUB_TYPE @@ -1079,19 +1163,21 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL - propertyMetadatas: + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE + propertyMetadatas: - name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-alter-statement.max-alter-count} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.too-many-alter-statement.max-alter-count.description} type: INTEGER componentType: INPUT_NUMBER - defaultValues: + defaultValues: - !!int 10 - id: 46 name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.not-null-column-without-default-value.name} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.not-null-column-without-default-value.description} type: SQL_CHECK builtIn: 1 - labels: + labels: - label: SUB_TYPE value: DDL - label: SUB_TYPE @@ -1106,12 +1192,14 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE - id: 47 name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-naming.name} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-naming.description} type: SQL_CHECK builtIn: 1 - labels: + labels: - label: SUB_TYPE value: DDL - label: SUB_TYPE @@ -1126,7 +1214,9 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL - propertyMetadatas: + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE + propertyMetadatas: - name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-naming.name-pattern} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-pk-naming.name-pattern.description} type: STRING @@ -1136,7 +1226,7 @@ description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.prohibited-datatype-exists.description} type: SQL_CHECK builtIn: 1 - labels: + labels: - label: SUB_TYPE value: DDL - label: SUB_TYPE @@ -1151,7 +1241,9 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL - propertyMetadatas: + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE + propertyMetadatas: - name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.prohibited-datatype-exists.datatype-names} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.prohibited-datatype-exists.datatype-names.description} type: STRING_LIST @@ -1178,12 +1270,14 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL - propertyMetadatas: + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE + propertyMetadatas: - name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-index-datatypes.allowed-datatypes} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-index-datatypes.allowed-datatypes.description} type: STRING_LIST componentType: SELECT_TAGS - defaultValues: + defaultValues: - int - varchar2 - number @@ -1211,6 +1305,8 @@ value: DORIS - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE - id: 51 name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.not-allowed-create-pl.name} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.not-allowed-create-pl.description} @@ -1231,6 +1327,8 @@ value: DORIS - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE - id: 52 name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.external-sql-interceptor.name} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.external-sql-interceptor.description} @@ -1251,6 +1349,8 @@ value: DORIS - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE propertyMetadatas: - name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.external-sql-interceptor.metadata.name} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-console.external-sql-interceptor.metadata.description} @@ -1274,6 +1374,8 @@ value: MYSQL - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE propertyMetadatas: - name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-drop-object-types.allowed-object-types} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-drop-object-types.allowed-object-types.description} @@ -1329,6 +1431,8 @@ value: OB_ORACLE - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE - id: 55 name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-auto-increment-datatypes.name} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-auto-increment-datatypes.description} @@ -1368,6 +1472,8 @@ value: OB_ORACLE - label: SUPPORTED_DIALECT_TYPE value: ODP_SHARDING_OB_MYSQL + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE - id: 57 name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.offline-schema-change-exists.name} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.offline-schema-change-exists.description} @@ -1380,6 +1486,8 @@ value: OB_MYSQL - label: SUPPORTED_DIALECT_TYPE value: OB_ORACLE + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE - id: 58 name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.truncate-table-exists.name} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.truncate-table-exists.description} @@ -1416,6 +1524,10 @@ value: OB_MYSQL - label: SUPPORTED_DIALECT_TYPE value: MYSQL + - label: SUPPORTED_DIALECT_TYPE + value: OB_ORACLE + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE propertyMetadatas: - name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-sql-affected-rows.allowed-max-sql-affected-count} description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.restrict-sql-affected-rows.allowed-max-sql-affected-count.description} @@ -1440,4 +1552,40 @@ - label: SUPPORTED_DIALECT_TYPE value: OB_MYSQL - label: SUPPORTED_DIALECT_TYPE - value: MYSQL \ No newline at end of file + value: MYSQL +- id: 61 + name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.create-table-like-exists.name} + description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.create-table-like-exists.description} + type: SQL_CHECK + builtIn: 1 + labels: + - label: SUB_TYPE + value: DDL + - label: SUB_TYPE + value: TABLE + - label: SUPPORTED_DIALECT_TYPE + value: OB_MYSQL + - label: SUPPORTED_DIALECT_TYPE + value: MYSQL + - label: SUPPORTED_DIALECT_TYPE + value: ODP_SHARDING_OB_MYSQL +- id: 62 + name: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.create-table-as-exists.name} + description: ${com.oceanbase.odc.builtin-resource.regulation.rule.sql-check.create-table-as-exists.description} + type: SQL_CHECK + builtIn: 1 + labels: + - label: SUB_TYPE + value: DDL + - label: SUB_TYPE + value: TABLE + - label: SUPPORTED_DIALECT_TYPE + value: MYSQL + - label: SUPPORTED_DIALECT_TYPE + value: ORACLE + - label: SUPPORTED_DIALECT_TYPE + value: OB_MYSQL + - label: SUPPORTED_DIALECT_TYPE + value: OB_ORACLE + - label: SUPPORTED_DIALECT_TYPE + value: ODP_SHARDING_OB_MYSQL diff --git a/server/odc-migrate/src/main/resources/migrate/common/R_2_0_0__initialize_version_diff_config.sql b/server/odc-migrate/src/main/resources/migrate/common/R_2_0_0__initialize_version_diff_config.sql index 0adc9490b3..ce35102a64 100644 --- a/server/odc-migrate/src/main/resources/migrate/common/R_2_0_0__initialize_version_diff_config.sql +++ b/server/odc-migrate/src/main/resources/migrate/common/R_2_0_0__initialize_version_diff_config.sql @@ -259,3 +259,9 @@ insert into `odc_version_diff_config`(`config_key`,`db_mode`,`config_value`,`min insert into `odc_version_diff_config`(`config_key`,`db_mode`,`config_value`,`min_version`,`gmt_create`) values('support_column_group', 'OB_MYSQL', 'true', '4.3.0', CURRENT_TIMESTAMP) ON DUPLICATE KEY update `config_key`=`config_key`; insert into `odc_version_diff_config`(`config_key`,`db_mode`,`config_value`,`min_version`,`gmt_create`) values('support_column_group', 'OB_ORACLE', 'true', '4.3.0', CURRENT_TIMESTAMP) ON DUPLICATE KEY update `config_key`=`config_key`; + +-- supports ob external table +insert into `odc_version_diff_config`(`config_key`,`db_mode`,`config_value`,`min_version`,`gmt_create`) values('support_external_table', 'OB_MYSQL', 'true', '4.3.2', CURRENT_TIMESTAMP) ON DUPLICATE KEY update `config_key`=`config_key`; +insert into `odc_version_diff_config`(`config_key`,`db_mode`,`config_value`,`min_version`,`gmt_create`) values('support_external_table', 'OB_ORACLE', 'true', '4.3.2', CURRENT_TIMESTAMP) ON DUPLICATE KEY update `config_key`=`config_key`; + +insert into `odc_version_diff_config`(`config_key`,`db_mode`,`config_value`,`min_version`,`gmt_create`) values('support_kill_session','ORACLE','true','0',CURRENT_TIMESTAMP) ON DUPLICATE KEY update `config_value`=VALUES(`config_value`); diff --git a/server/odc-migrate/src/main/resources/migrate/common/V_4_3_3_1__alter_integration.sql b/server/odc-migrate/src/main/resources/migrate/common/V_4_3_3_1__alter_integration.sql new file mode 100644 index 0000000000..3f84b6a06a --- /dev/null +++ b/server/odc-migrate/src/main/resources/migrate/common/V_4_3_3_1__alter_integration.sql @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2024 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +alter table `integration_integration` modify column `secret` mediumtext DEFAULT null; \ No newline at end of file diff --git a/server/odc-migrate/src/main/resources/migrate/common/V_4_3_3_2__alter_resource_resource_add_index.sql b/server/odc-migrate/src/main/resources/migrate/common/V_4_3_3_2__alter_resource_resource_add_index.sql new file mode 100644 index 0000000000..ddf84db49a --- /dev/null +++ b/server/odc-migrate/src/main/resources/migrate/common/V_4_3_3_2__alter_resource_resource_add_index.sql @@ -0,0 +1,4 @@ +-- +-- Add constraint (resource_type, status) to `resource_resource` table +-- +alter table `resource_resource` add index `type_status`(`resource_type`, `status`); \ No newline at end of file diff --git a/server/odc-migrate/src/main/resources/migrate/common/V_4_3_3_3__iam_permission.yaml b/server/odc-migrate/src/main/resources/migrate/common/V_4_3_3_3__iam_permission.yaml new file mode 100644 index 0000000000..15b4fcd630 --- /dev/null +++ b/server/odc-migrate/src/main/resources/migrate/common/V_4_3_3_3__iam_permission.yaml @@ -0,0 +1,78 @@ +kind: resource +version: v2 +templates: + - metadata: + allow_duplicate: false + table_name: iam_permission + unique_keys: ["action", "organization_id", "resource_identifier", "type"] + specs: + - column_name: id + default_value: 73 + data_type: java.lang.Long + - column_name: action + value: "OWNER" + - column_name: resource_identifier + value: "ODC_PROJECT:*" + - column_name: organization_id + value: ${ORGANIZATION_ID} + data_type: java.lang.Long + - column_name: type + value: "SYSTEM" + - column_name: creator_id + value: ${CREATOR_ID} + data_type: java.lang.Long + - column_name: is_builtin + value: true + data_type: java.lang.Boolean + - column_name: description + value: "global project owner permission" + - metadata: + allow_duplicate: false + table_name: iam_permission + unique_keys: ["action", "organization_id", "resource_identifier", "type"] + specs: + - column_name: id + default_value: 74 + data_type: java.lang.Long + - column_name: action + value: "DBA" + - column_name: resource_identifier + value: "ODC_PROJECT:*" + - column_name: organization_id + value: ${ORGANIZATION_ID} + data_type: java.lang.Long + - column_name: type + value: "SYSTEM" + - column_name: creator_id + value: ${CREATOR_ID} + data_type: java.lang.Long + - column_name: is_builtin + value: true + data_type: java.lang.Boolean + - column_name: description + value: "global project DBA permission" + - metadata: + allow_duplicate: false + table_name: iam_permission + unique_keys: ["action", "organization_id", "resource_identifier", "type"] + specs: + - column_name: id + default_value: 75 + data_type: java.lang.Long + - column_name: action + value: "SECURITY_ADMINISTRATOR" + - column_name: resource_identifier + value: "ODC_PROJECT:*" + - column_name: organization_id + value: ${ORGANIZATION_ID} + data_type: java.lang.Long + - column_name: type + value: "SYSTEM" + - column_name: creator_id + value: ${CREATOR_ID} + data_type: java.lang.Long + - column_name: is_builtin + value: true + data_type: java.lang.Boolean + - column_name: description + value: "global project Security Administrator permission" \ No newline at end of file diff --git a/server/odc-migrate/src/main/resources/migrate/common/V_4_3_3_4__iam_role.yaml b/server/odc-migrate/src/main/resources/migrate/common/V_4_3_3_4__iam_role.yaml new file mode 100644 index 0000000000..db18fd9b26 --- /dev/null +++ b/server/odc-migrate/src/main/resources/migrate/common/V_4_3_3_4__iam_role.yaml @@ -0,0 +1,81 @@ +kind: resource +version: v2 +templates: + - metadata: + allow_duplicate: false + table_name: iam_role + unique_keys: ["name", "organization_id"] + specs: + - column_name: id + default_value: 5 + data_type: java.lang.Long + - column_name: name + value: "global_project_owner" + - column_name: type + value: "CUSTOM" + - column_name: organization_id + value: ${ORGANIZATION_ID} + data_type: java.lang.Long + - column_name: is_enabled + value: true + data_type: java.lang.Boolean + - column_name: creator_id + value: ${CREATOR_ID} + data_type: java.lang.Long + - column_name: is_builtin + value: true + data_type: java.lang.Boolean + - column_name: description + value: "global project owner, who is the owner of all projects" + - metadata: + allow_duplicate: false + table_name: iam_role + unique_keys: [ "name", "organization_id" ] + specs: + - column_name: id + default_value: 6 + data_type: java.lang.Long + - column_name: name + value: "global_project_dba" + - column_name: type + value: "CUSTOM" + - column_name: organization_id + value: ${ORGANIZATION_ID} + data_type: java.lang.Long + - column_name: is_enabled + value: true + data_type: java.lang.Boolean + - column_name: creator_id + value: ${CREATOR_ID} + data_type: java.lang.Long + - column_name: is_builtin + value: true + data_type: java.lang.Boolean + - column_name: description + value: "global project owner, who is the DBA of all projects" + - metadata: + allow_duplicate: false + table_name: iam_role + unique_keys: [ "name", "organization_id" ] + specs: + - column_name: id + default_value: 7 + data_type: java.lang.Long + - column_name: name + value: "global_project_security_administrator" + - column_name: type + value: "CUSTOM" + - column_name: organization_id + value: ${ORGANIZATION_ID} + data_type: java.lang.Long + - column_name: is_enabled + value: true + data_type: java.lang.Boolean + - column_name: creator_id + value: ${CREATOR_ID} + data_type: java.lang.Long + - column_name: is_builtin + value: true + data_type: java.lang.Boolean + - column_name: description + value: "global project owner, who is the security administrator of all projects" diff --git a/server/odc-migrate/src/main/resources/migrate/common/V_4_3_3_5__iam_role_permission.yaml b/server/odc-migrate/src/main/resources/migrate/common/V_4_3_3_5__iam_role_permission.yaml new file mode 100644 index 0000000000..952322539b --- /dev/null +++ b/server/odc-migrate/src/main/resources/migrate/common/V_4_3_3_5__iam_role_permission.yaml @@ -0,0 +1,66 @@ +kind: resource +version: v2 +templates: + - metadata: + allow_duplicate: false + table_name: iam_role_permission + unique_keys: [ "role_id", "permission_id" ] + specs: + - column_name: role_id + value_from: + field_ref: + ref_file: migrate/common/V_4_3_3_4__iam_role.yaml + field_path: templates.0.specs.0.value + - column_name: permission_id + value_from: + field_ref: + ref_file: migrate/common/V_4_3_3_3__iam_permission.yaml + field_path: templates.0.specs.0.value + - column_name: organization_id + value: ${ORGANIZATION_ID} + data_type: java.lang.Long + - column_name: creator_id + value: ${CREATOR_ID} + data_type: java.lang.Long + - metadata: + allow_duplicate: false + table_name: iam_role_permission + unique_keys: [ "role_id", "permission_id" ] + specs: + - column_name: role_id + value_from: + field_ref: + ref_file: migrate/common/V_4_3_3_4__iam_role.yaml + field_path: templates.1.specs.0.value + - column_name: permission_id + value_from: + field_ref: + ref_file: migrate/common/V_4_3_3_3__iam_permission.yaml + field_path: templates.1.specs.0.value + - column_name: organization_id + value: ${ORGANIZATION_ID} + data_type: java.lang.Long + - column_name: creator_id + value: ${CREATOR_ID} + data_type: java.lang.Long + - metadata: + allow_duplicate: false + table_name: iam_role_permission + unique_keys: [ "role_id", "permission_id" ] + specs: + - column_name: role_id + value_from: + field_ref: + ref_file: migrate/common/V_4_3_3_4__iam_role.yaml + field_path: templates.2.specs.0.value + - column_name: permission_id + value_from: + field_ref: + ref_file: migrate/common/V_4_3_3_3__iam_permission.yaml + field_path: templates.2.specs.0.value + - column_name: organization_id + value: ${ORGANIZATION_ID} + data_type: java.lang.Long + - column_name: creator_id + value: ${CREATOR_ID} + data_type: java.lang.Long \ No newline at end of file diff --git a/server/odc-migrate/src/main/resources/migrate/common/V_4_3_3_6__complete_connect_database.sql b/server/odc-migrate/src/main/resources/migrate/common/V_4_3_3_6__complete_connect_database.sql new file mode 100644 index 0000000000..6c3a76cb56 --- /dev/null +++ b/server/odc-migrate/src/main/resources/migrate/common/V_4_3_3_6__complete_connect_database.sql @@ -0,0 +1,6 @@ +UPDATE connect_database +SET connect_type = ( + SELECT cc.type FROM connect_connection cc + WHERE connect_database.connection_id = cc.id + ) +WHERE connect_type IS NULL AND type = 'PHYSICAL'; \ No newline at end of file diff --git a/server/odc-server/pom.xml b/server/odc-server/pom.xml index 6107e5dc14..d940a825b2 100644 --- a/server/odc-server/pom.xml +++ b/server/odc-server/pom.xml @@ -6,7 +6,7 @@ com.oceanbase odc-parent - 4.3.2-SNAPSHOT + 4.3.3-SNAPSHOT ../../pom.xml odc-server @@ -35,6 +35,10 @@ + + commons-codec + commons-codec + com.h2database h2 @@ -138,6 +142,21 @@ com.oceanbase odc-migrate + + junit + junit + test + + + org.mockito + mockito-core + test + + + org.mockito + mockito-inline + test + diff --git a/server/odc-server/src/main/java/com/oceanbase/odc/agent/OdcAgent.java b/server/odc-server/src/main/java/com/oceanbase/odc/agent/OdcAgent.java new file mode 100644 index 0000000000..b7bd7150e9 --- /dev/null +++ b/server/odc-server/src/main/java/com/oceanbase/odc/agent/OdcAgent.java @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.agent; + +import com.oceanbase.odc.agent.runtime.TaskApplication; +import com.oceanbase.odc.server.module.Modules; + +import lombok.extern.slf4j.Slf4j; + +/** + * main class for Odc agent + * + * @author longpeng.zlp + * @date 2024/8/9 15:31 + */ +@Slf4j +public class OdcAgent { + public static void main(String[] args) { + + log.info("ODC start as task executor mode"); + try { + Modules.load(); + new TaskApplication().run(args); + } catch (Throwable e) { + log.error("Task existed abnormal", e); + } + log.info("Task executor exit."); + } +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/task/DefaultTaskResultBuilder.java b/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/DefaultTaskResultBuilder.java similarity index 71% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/task/DefaultTaskResultBuilder.java rename to server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/DefaultTaskResultBuilder.java index 375152829f..59b6279ab3 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/task/DefaultTaskResultBuilder.java +++ b/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/DefaultTaskResultBuilder.java @@ -13,34 +13,32 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.task.executor.task; +package com.oceanbase.odc.agent.runtime; import com.oceanbase.odc.common.json.JsonUtils; import com.oceanbase.odc.common.util.ExceptionUtils; +import com.oceanbase.odc.service.task.Task; +import com.oceanbase.odc.service.task.executor.TaskResult; import com.oceanbase.odc.service.task.util.JobUtils; -import lombok.extern.slf4j.Slf4j; - /** * @author yaobin * @date 2024-01-12 * @since 4.2.4 */ -@Slf4j -public class DefaultTaskResultBuilder { - - public static DefaultTaskResult build(BaseTask task) { - DefaultTaskResult result = new DefaultTaskResult(); +class DefaultTaskResultBuilder { + public static TaskResult build(TaskContainer taskContainer) { + TaskResult result = new TaskResult(); + Task task = taskContainer.getTask(); result.setResultJson(JsonUtils.toJson(task.getTaskResult())); - result.setStatus(task.getStatus()); + result.setStatus(taskContainer.getStatus()); result.setProgress(task.getProgress()); result.setJobIdentity(task.getJobContext().getJobIdentity()); result.setExecutorEndpoint(JobUtils.getExecutorPoint()); return result; } - public static void assignErrorMessage(DefaultTaskResult result, BaseTask task) { - Throwable e = task.getError(); + public static void assignErrorMessage(TaskResult result, Throwable e) { result.setErrorMessage(null == e ? null : ExceptionUtils.getRootCauseReason(e, 3)); } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/server/EmbedServer.java b/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/EmbedServer.java similarity index 98% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/server/EmbedServer.java rename to server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/EmbedServer.java index 45ff839840..f753a7b12c 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/server/EmbedServer.java +++ b/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/EmbedServer.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.task.executor.server; +package com.oceanbase.odc.agent.runtime; import java.net.InetSocketAddress; import java.util.concurrent.LinkedBlockingQueue; @@ -28,6 +28,8 @@ import com.oceanbase.odc.common.json.JsonUtils; import com.oceanbase.odc.common.util.StringUtils; import com.oceanbase.odc.service.common.util.UrlUtils; +import com.oceanbase.odc.service.task.executor.TraceDecoratorThreadFactory; +import com.oceanbase.odc.service.task.executor.TraceDecoratorUtils; import com.oceanbase.odc.service.task.util.JobUtils; import io.netty.bootstrap.ServerBootstrap; @@ -63,7 +65,7 @@ * @since 4.2.4 */ @Slf4j -public class EmbedServer { +class EmbedServer { private ExecutorRequestHandler requestHandler; private Thread thread; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/server/ExecutorRequestHandler.java b/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/ExecutorRequestHandler.java similarity index 84% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/server/ExecutorRequestHandler.java rename to server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/ExecutorRequestHandler.java index aef86ef5d6..89c257daf4 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/server/ExecutorRequestHandler.java +++ b/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/ExecutorRequestHandler.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.task.executor.server; +package com.oceanbase.odc.agent.runtime; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -26,13 +26,10 @@ import com.oceanbase.odc.service.common.response.SuccessResponse; import com.oceanbase.odc.service.common.util.UrlUtils; import com.oceanbase.odc.service.task.constants.JobExecutorUrls; +import com.oceanbase.odc.service.task.executor.TaskResult; import com.oceanbase.odc.service.task.executor.logger.LogBiz; import com.oceanbase.odc.service.task.executor.logger.LogBizImpl; import com.oceanbase.odc.service.task.executor.logger.LogUtils; -import com.oceanbase.odc.service.task.executor.task.BaseTask; -import com.oceanbase.odc.service.task.executor.task.DefaultTaskResult; -import com.oceanbase.odc.service.task.executor.task.DefaultTaskResultBuilder; -import com.oceanbase.odc.service.task.executor.task.Task; import com.oceanbase.odc.service.task.schedule.JobIdentity; import com.oceanbase.odc.service.task.util.JobUtils; @@ -45,7 +42,7 @@ * @since 4.2.4 */ @Slf4j -public class ExecutorRequestHandler { +class ExecutorRequestHandler { private final Pattern queryLogUrlPattern = Pattern.compile(String.format(JobExecutorUrls.QUERY_LOG, "([0-9]+)")); private final Pattern stopTaskPattern = Pattern.compile(String.format(JobExecutorUrls.STOP_TASK, "([0-9]+)")); @@ -88,25 +85,25 @@ public SuccessResponse process(HttpMethod httpMethod, String uri, String matcher = modifyParametersPattern.matcher(path); if (matcher.find()) { JobIdentity ji = getJobIdentity(matcher); - Task task = ThreadPoolTaskExecutor.getInstance().getTask(ji); - boolean result = task.modify(JobUtils.fromJsonToMap(requestData)); + TaskRuntimeInfo runtimeInfo = ThreadPoolTaskExecutor.getInstance().getTaskRuntimeInfo(ji); + boolean result = runtimeInfo.getTaskContainer().modify(JobUtils.fromJsonToMap(requestData)); return Responses.ok(result); } matcher = getResultPattern.matcher(path); if (matcher.find()) { JobIdentity ji = getJobIdentity(matcher); - BaseTask task = ThreadPoolTaskExecutor.getInstance().getTask(ji); - TaskMonitor taskMonitor = task.getTaskMonitor(); - DefaultTaskResult result = DefaultTaskResultBuilder.build(task); + TaskRuntimeInfo runtimeInfo = ThreadPoolTaskExecutor.getInstance().getTaskRuntimeInfo(ji); + TaskMonitor taskMonitor = runtimeInfo.getTaskMonitor(); + TaskResult result = DefaultTaskResultBuilder.build(runtimeInfo.getTaskContainer()); if (taskMonitor != null && MapUtils.isNotEmpty(taskMonitor.getLogMetadata())) { result.setLogMetadata(taskMonitor.getLogMetadata()); // assign final error message - DefaultTaskResultBuilder.assignErrorMessage(result, task); + DefaultTaskResultBuilder.assignErrorMessage(result, runtimeInfo.getTaskContainer().getError()); taskMonitor.markLogMetaCollected(); log.info("Task log metadata collected, ji={}.", ji.getId()); } - DefaultTaskResult copiedResult = ObjectUtil.deepCopy(result, DefaultTaskResult.class); + TaskResult copiedResult = ObjectUtil.deepCopy(result, TaskResult.class); return Responses.ok(copiedResult); } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/server/ExitHelper.java b/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/ExitHelper.java similarity index 92% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/server/ExitHelper.java rename to server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/ExitHelper.java index b4adbbf4f8..472039836e 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/server/ExitHelper.java +++ b/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/ExitHelper.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.task.executor.server; +package com.oceanbase.odc.agent.runtime; import java.util.concurrent.CountDownLatch; @@ -25,7 +25,7 @@ * @since 4.2.4 */ @Slf4j -public class ExitHelper { +class ExitHelper { private static final CountDownLatch LATCH = new CountDownLatch(1); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/TaskApplication.java b/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/TaskApplication.java similarity index 91% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/TaskApplication.java rename to server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/TaskApplication.java index 84b5a3758f..c3b3e8274f 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/TaskApplication.java +++ b/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/TaskApplication.java @@ -13,8 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -package com.oceanbase.odc.service.task.executor; +package com.oceanbase.odc.agent.runtime; import java.io.File; import java.net.URI; @@ -30,16 +29,12 @@ import com.oceanbase.odc.common.util.StringUtils; import com.oceanbase.odc.common.util.SystemUtils; import com.oceanbase.odc.core.shared.Verify; +import com.oceanbase.odc.service.task.Task; import com.oceanbase.odc.service.task.caller.JobContext; import com.oceanbase.odc.service.task.caller.JobEnvironmentEncryptor; import com.oceanbase.odc.service.task.constants.JobEnvKeyConstants; import com.oceanbase.odc.service.task.exception.TaskRuntimeException; import com.oceanbase.odc.service.task.executor.context.JobContextProviderFactory; -import com.oceanbase.odc.service.task.executor.server.EmbedServer; -import com.oceanbase.odc.service.task.executor.server.ExitHelper; -import com.oceanbase.odc.service.task.executor.server.TaskFactory; -import com.oceanbase.odc.service.task.executor.server.ThreadPoolTaskExecutor; -import com.oceanbase.odc.service.task.executor.task.BaseTask; import com.oceanbase.odc.service.task.util.JobUtils; import lombok.extern.slf4j.Slf4j; @@ -64,7 +59,7 @@ public void run(String[] args) { try { server.start(); log.info("Starting embed server."); - BaseTask task = TaskFactory.create(context.getJobClass()); + Task task = TaskFactory.create(context.getJobClass()); ThreadPoolTaskExecutor.getInstance().execute(task, context); ExitHelper.await(); } catch (Exception e) { @@ -147,7 +142,7 @@ private void setLog4JConfigXml() { } } - LoggerContext context = (org.apache.logging.log4j.core.LoggerContext) LogManager.getContext(false); + LoggerContext context = (LoggerContext) LogManager.getContext(false); // this will force a reconfiguration, MDC context will to take effect context.setConfigLocation(taskLogFile); } diff --git a/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/TaskContainer.java b/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/TaskContainer.java new file mode 100644 index 0000000000..c780b20144 --- /dev/null +++ b/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/TaskContainer.java @@ -0,0 +1,178 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.agent.runtime; + +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; + +import com.oceanbase.odc.core.shared.constant.TaskStatus; +import com.oceanbase.odc.service.objectstorage.cloud.CloudObjectStorageService; +import com.oceanbase.odc.service.task.ExceptionListener; +import com.oceanbase.odc.service.task.Task; +import com.oceanbase.odc.service.task.TaskContext; +import com.oceanbase.odc.service.task.caller.JobContext; + +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; + +/** + * @author gaoda.xy + * @date 2023/11/22 20:16 + */ +@Slf4j +final class TaskContainer implements ExceptionListener { + // if task has been closed + private final AtomicBoolean closed = new AtomicBoolean(false); + // status maintained by task container + private volatile TaskStatus status = TaskStatus.PREPARING; + // task event listener for task event notify + @Getter + protected final TaskMonitor taskMonitor; + // task context for task to init + protected final TaskContext taskContext; + // task instance holding + @Getter + private final Task task; + // only save latest exception if any + // it will be cleaned if been fetched + protected AtomicReference latestException = new AtomicReference<>(); + + public TaskContainer(JobContext jobContext, CloudObjectStorageService cloudObjectStorageService, + TaskReporter taskReporter, // assignable for test + Task task) { + this.taskContext = createTaskContext(jobContext, cloudObjectStorageService); + this.task = task; + this.taskMonitor = new TaskMonitor(this, taskReporter, cloudObjectStorageService); + } + + protected TaskContext createTaskContext(JobContext jobContext, + CloudObjectStorageService cloudObjectStorageService) { + ExceptionListener exceptionListener = this; + return new TaskContext() { + @Override + public ExceptionListener getExceptionListener() { + return exceptionListener; + } + + @Override + public JobContext getJobContext() { + return jobContext; + } + + @Override + public CloudObjectStorageService getSharedStorage() { + return cloudObjectStorageService; + } + }; + } + + /** + * 1. init task 2. run task 3. maintain status + */ + public void runTask() { + try { + task.init(taskContext); + updateStatus(TaskStatus.RUNNING); + taskMonitor.monitor(); + if (task.start()) { + updateStatus(TaskStatus.DONE); + } else { + updateStatus(TaskStatus.FAILED); + } + } catch (Throwable e) { + log.warn("Task failed, id={}.", getJobId(), e); + updateStatus(TaskStatus.FAILED); + onException(e); + } finally { + close(); + } + } + + public boolean stop() { + try { + if (getStatus().isTerminated()) { + log.warn("Task is already finished and cannot be canceled, id={}, status={}.", getJobId(), getStatus()); + } else { + task.stop(); + // doRefresh cannot execute if update status to 'canceled'. + updateStatus(TaskStatus.CANCELED); + } + return true; + } catch (Throwable e) { + log.warn("Stop task failed, id={}", getJobId(), e); + return false; + } finally { + close(); + } + } + + public boolean modify(Map jobParameters) { + if (Objects.isNull(jobParameters) || jobParameters.isEmpty()) { + log.warn("Job parameter cannot be null, id={}", getJobId()); + return false; + } + if (getStatus().isTerminated()) { + log.warn("Task is already finished, cannot modify parameters, id={}", getJobId()); + return false; + } + task.modify(jobParameters); + return true; + } + + + private void close() { + if (closed.compareAndSet(false, true)) { + try { + task.close(); + } catch (Throwable e) { + // do nothing + } + log.info("Task completed, id={}, status={}.", getJobId(), getStatus()); + taskMonitor.finalWork(); + } + } + + public TaskStatus getStatus() { + return status; + } + + + private void updateStatus(TaskStatus status) { + log.info("Update task status, id={}, status={}.", getJobId(), status); + this.status = status; + } + + protected Map getJobParameters() { + return task.getJobContext().getJobParameters(); + } + + private Long getJobId() { + return task.getJobContext().getJobIdentity().getId(); + } + + public Throwable getError() { + Throwable e = latestException.getAndSet(null); + log.info("retrieve exception = {}", null == e ? null : e.getMessage()); + return e; + } + + public void onException(Throwable e) { + log.info("found exception", e); + this.latestException.set(e); + } +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/server/TaskExecutor.java b/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/TaskExecutor.java similarity index 76% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/server/TaskExecutor.java rename to server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/TaskExecutor.java index efdbc3080a..346d74052d 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/server/TaskExecutor.java +++ b/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/TaskExecutor.java @@ -13,23 +13,24 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +package com.oceanbase.odc.agent.runtime; -package com.oceanbase.odc.service.task.executor.server; - +import com.oceanbase.odc.service.task.Task; import com.oceanbase.odc.service.task.caller.JobContext; -import com.oceanbase.odc.service.task.executor.task.BaseTask; import com.oceanbase.odc.service.task.schedule.JobIdentity; /** * @author gaoda.xy * @date 2023/11/24 11:18 */ -public interface TaskExecutor { +interface TaskExecutor { - void execute(BaseTask task, JobContext jc); + void execute(Task task, JobContext jc); boolean cancel(JobIdentity ji); - BaseTask getTask(JobIdentity ji); + TaskRuntimeInfo getTaskRuntimeInfo(JobIdentity ji); + + boolean taskExist(JobIdentity ji); } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/server/TaskFactory.java b/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/TaskFactory.java similarity index 77% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/server/TaskFactory.java rename to server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/TaskFactory.java index a37376008e..7ee7addd1b 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/server/TaskFactory.java +++ b/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/TaskFactory.java @@ -13,25 +13,24 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.task.executor.server; +package com.oceanbase.odc.agent.runtime; +import com.oceanbase.odc.service.task.Task; import com.oceanbase.odc.service.task.exception.TaskRuntimeException; -import com.oceanbase.odc.service.task.executor.task.BaseTask; -import com.oceanbase.odc.service.task.executor.task.Task; /** * @author gaoda.xy * @date 2023/11/24 11:01 */ -public class TaskFactory { +class TaskFactory { - public static BaseTask create(String jobClass) { + public static Task create(String jobClass) { try { Class c = Class.forName(jobClass); if (!Task.class.isAssignableFrom(c)) { throw new TaskRuntimeException("Job class is not implements Task. name={}" + jobClass); } - return (BaseTask) c.newInstance(); + return (Task) c.newInstance(); } catch (Exception e) { throw new TaskRuntimeException(e); } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/server/TaskMonitor.java b/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/TaskMonitor.java similarity index 75% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/server/TaskMonitor.java rename to server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/TaskMonitor.java index 07e9a2d882..05526a314e 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/server/TaskMonitor.java +++ b/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/TaskMonitor.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.task.executor.server; +package com.oceanbase.odc.agent.runtime; import java.util.HashMap; import java.util.Map; @@ -26,18 +26,19 @@ import org.apache.commons.lang3.StringUtils; -import com.oceanbase.odc.common.util.ObjectUtil; +import com.google.common.annotations.VisibleForTesting; +import com.oceanbase.odc.core.shared.constant.TaskStatus; import com.oceanbase.odc.core.task.TaskThreadFactory; import com.oceanbase.odc.service.objectstorage.cloud.CloudObjectStorageService; +import com.oceanbase.odc.service.task.Task; import com.oceanbase.odc.service.task.constants.JobAttributeKeyConstants; import com.oceanbase.odc.service.task.constants.JobConstants; import com.oceanbase.odc.service.task.constants.JobParametersKeyConstants; import com.oceanbase.odc.service.task.constants.JobServerUrls; -import com.oceanbase.odc.service.task.enums.JobStatus; +import com.oceanbase.odc.service.task.executor.HeartbeatRequest; +import com.oceanbase.odc.service.task.executor.TaskResult; +import com.oceanbase.odc.service.task.executor.TraceDecoratorThreadFactory; import com.oceanbase.odc.service.task.executor.logger.LogBizImpl; -import com.oceanbase.odc.service.task.executor.task.BaseTask; -import com.oceanbase.odc.service.task.executor.task.DefaultTaskResult; -import com.oceanbase.odc.service.task.executor.task.DefaultTaskResultBuilder; import com.oceanbase.odc.service.task.util.JobUtils; import lombok.extern.slf4j.Slf4j; @@ -48,12 +49,12 @@ * @since 4.2.4 */ @Slf4j -public class TaskMonitor { +class TaskMonitor { private static final int REPORT_RESULT_RETRY_TIMES = Integer.MAX_VALUE; private static final long WAIT_AFTER_LOG_METADATA_COLLECT_MILLS = 5000L; private final TaskReporter reporter; - private final BaseTask task; + private final TaskContainer taskContainer; private final CloudObjectStorageService cloudObjectStorageService; private volatile long startTimeMilliSeconds; private ScheduledExecutorService reportScheduledExecutor; @@ -61,9 +62,10 @@ public class TaskMonitor { private Map logMetadata = new HashMap<>(); private AtomicLong logMetaCollectedMillis = new AtomicLong(0L); - public TaskMonitor(BaseTask task, CloudObjectStorageService cloudObjectStorageService) { - this.task = task; - this.reporter = new TaskReporter(task.getJobContext().getHostUrls());; + public TaskMonitor(TaskContainer task, TaskReporter taskReporter, + CloudObjectStorageService cloudObjectStorageService) { + this.taskContainer = task; + this.reporter = taskReporter; this.cloudObjectStorageService = cloudObjectStorageService; } @@ -84,9 +86,9 @@ public void monitor() { new TraceDecoratorThreadFactory(new TaskThreadFactory(("Task-Monitor-Job-" + getJobId()))); this.reportScheduledExecutor = Executors.newSingleThreadScheduledExecutor(threadFactory); reportScheduledExecutor.scheduleAtFixedRate(() -> { - if (isTimeout() && !getTask().getStatus().isTerminated()) { + if (isTimeout() && !getTaskContainer().getStatus().isTerminated()) { log.info("Task timeout, try stop, jobId={}", getJobId()); - getTask().stop(); + getTaskContainer().stop(); } try { if (JobUtils.getExecutorPort().isPresent()) { @@ -136,24 +138,25 @@ private void destroy(ExecutorService executorService) { } } - private void reportTaskResult() { + @VisibleForTesting + protected void reportTaskResult() { if (JobUtils.isReportDisabled()) { return; } - DefaultTaskResult taskResult = DefaultTaskResultBuilder.build(getTask()); - DefaultTaskResult copiedResult = ObjectUtil.deepCopy(taskResult, DefaultTaskResult.class); - if (copiedResult.getStatus().isTerminated()) { + TaskResult taskResult = DefaultTaskResultBuilder.build(getTaskContainer()); + if (taskResult.getStatus().isTerminated()) { log.info("job {} status {} is terminate, monitor report be ignored.", - copiedResult.getJobIdentity().getId(), copiedResult.getStatus()); + taskResult.getJobIdentity().getId(), taskResult.getStatus()); return; } - getReporter().report(JobServerUrls.TASK_UPLOAD_RESULT, copiedResult); + getReporter().report(JobServerUrls.TASK_UPLOAD_RESULT, taskResult); log.info("Report task info, id: {}, status: {}, progress: {}%, result: {}", getJobId(), - copiedResult.getStatus(), String.format("%.2f", copiedResult.getProgress()), getTask().getTaskResult()); + taskResult.getStatus(), String.format("%.2f", taskResult.getProgress()), getTask().getTaskResult()); } - private boolean isTimeout() { + @VisibleForTesting + protected boolean isTimeout() { String milliSecStr = getTask().getJobContext().getJobParameters() .get(JobParametersKeyConstants.TASK_EXECUTION_TIMEOUT_MILLIS); @@ -169,9 +172,10 @@ private boolean isTimeout() { } } - private void doFinal() { + @VisibleForTesting + protected void doFinal() { - DefaultTaskResult finalResult = DefaultTaskResultBuilder.build(getTask()); + TaskResult finalResult = DefaultTaskResultBuilder.build(getTaskContainer()); // Report final result log.info("Task id: {}, finished with status: {}, start to report final result", getJobId(), finalResult.getStatus()); @@ -190,16 +194,18 @@ private void doFinal() { if (JobUtils.isReportEnabled()) { // assign error for last report - DefaultTaskResultBuilder.assignErrorMessage(finalResult, getTask()); + DefaultTaskResultBuilder.assignErrorMessage(finalResult, taskContainer.getError()); // Report finish signal to task server - reportTaskResultWithRetry(finalResult, REPORT_RESULT_RETRY_TIMES); + reportTaskResultWithRetry(finalResult, REPORT_RESULT_RETRY_TIMES, + JobConstants.REPORT_TASK_INFO_INTERVAL_SECONDS); } else { waitForTaskResultPulled(); } log.info("Task id: {} exit.", getJobId()); } - private void waitForTaskResultPulled() { + @VisibleForTesting + protected void waitForTaskResultPulled() { long currentTimeMillis = System.currentTimeMillis(); while (this.logMetaCollectedMillis.get() == 0 || currentTimeMillis > this.logMetaCollectedMillis.get() + WAIT_AFTER_LOG_METADATA_COLLECT_MILLS) { @@ -215,21 +221,29 @@ private void waitForTaskResultPulled() { getJobId(), this.logMetaCollectedMillis.get()); } - private void uploadLogFileToCloudStorage(DefaultTaskResult finalResult) { - Map logMap = new HashMap<>(); + @VisibleForTesting + protected void uploadLogFileToCloudStorage(TaskResult finalResult) { + + Map logMap = finalResult.getLogMetadata(); + Map logMetaData = new HashMap<>(); + // append previous result + if (null != logMap) { + logMetaData.putAll(logMap); + } if (cloudObjectStorageService != null && cloudObjectStorageService.supported() && JobUtils.isK8sRunModeOfEnv()) { - logMap = new LogBizImpl().uploadLogFileToCloudStorage(finalResult.getJobIdentity(), - cloudObjectStorageService); + logMetaData.putAll(new LogBizImpl().uploadLogFileToCloudStorage(finalResult.getJobIdentity(), + cloudObjectStorageService)); } else { - logMap.put(JobAttributeKeyConstants.LOG_STORAGE_FAILED_REASON, + logMetaData.put(JobAttributeKeyConstants.LOG_STORAGE_FAILED_REASON, "cloudObjectStorageService is null or not supported"); } - finalResult.setLogMetadata(logMap); + finalResult.setLogMetadata(logMetaData); } - private void reportTaskResultWithRetry(DefaultTaskResult result, int retries) { - if (result.getStatus() == JobStatus.DONE) { + @VisibleForTesting + protected boolean reportTaskResultWithRetry(TaskResult result, int retries, int retryIntervalSeconds) { + if (result.getStatus() == TaskStatus.DONE) { result.setProgress(100.0); } int retryTimes = 0; @@ -238,16 +252,17 @@ private void reportTaskResultWithRetry(DefaultTaskResult result, int retries) { boolean success = reporter.report(JobServerUrls.TASK_UPLOAD_RESULT, result); if (success) { log.info("Report task result successfully"); - break; + return true; } else { log.warn("Report task result failed, will retry after {} seconds, remaining retries: {}", - JobConstants.REPORT_TASK_INFO_INTERVAL_SECONDS, retries - retryTimes); - Thread.sleep(JobConstants.REPORT_TASK_INFO_INTERVAL_SECONDS * 1000L); + retryIntervalSeconds, retries - retryTimes); + Thread.sleep(retryIntervalSeconds * 1000L); } } catch (Throwable e) { log.warn("Report task result failed, taskId: {}", getJobId(), e); } } + return false; } private HeartbeatRequest buildHeartRequest() { @@ -257,15 +272,20 @@ private HeartbeatRequest buildHeartRequest() { return request; } - private BaseTask getTask() { - return task; + private TaskContainer getTaskContainer() { + return taskContainer; } - private TaskReporter getReporter() { + @VisibleForTesting + protected TaskReporter getReporter() { return reporter; } private Long getJobId() { return getTask().getJobContext().getJobIdentity().getId(); } + + private Task getTask() { + return getTaskContainer().getTask(); + } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/server/TaskReporter.java b/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/TaskReporter.java similarity index 96% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/server/TaskReporter.java rename to server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/TaskReporter.java index 88805089db..8ea5fb4c86 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/server/TaskReporter.java +++ b/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/TaskReporter.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.task.executor.server; +package com.oceanbase.odc.agent.runtime; import java.text.MessageFormat; import java.util.List; @@ -32,7 +32,7 @@ * @date 2023/11/30 19:41 */ @Slf4j -public class TaskReporter { +class TaskReporter { private final List hostUrls; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/NullK8sJobClientSelector.java b/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/TaskRuntimeInfo.java similarity index 65% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/NullK8sJobClientSelector.java rename to server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/TaskRuntimeInfo.java index 06d770ee53..b596132db2 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/NullK8sJobClientSelector.java +++ b/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/TaskRuntimeInfo.java @@ -13,11 +13,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.task.caller; +package com.oceanbase.odc.agent.runtime; -public class NullK8sJobClientSelector implements K8sJobClientSelector { - @Override - public K8sJobClient select(JobContext jobContext) { - throw new UnsupportedOperationException("K8sJobClientSelector is not configured."); - } +import java.util.concurrent.Future; + +import lombok.Data; + +/** + * @author longpeng.zlp + * @date 2024/10/24 14:32 + */ +@Data +class TaskRuntimeInfo { + private final TaskContainer taskContainer; + private Future future; + private final TaskMonitor taskMonitor; } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/server/ThreadPoolTaskExecutor.java b/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/ThreadPoolTaskExecutor.java similarity index 55% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/server/ThreadPoolTaskExecutor.java rename to server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/ThreadPoolTaskExecutor.java index f24b616644..e20d3bffc9 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/server/ThreadPoolTaskExecutor.java +++ b/server/odc-server/src/main/java/com/oceanbase/odc/agent/runtime/ThreadPoolTaskExecutor.java @@ -13,11 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +package com.oceanbase.odc.agent.runtime; -package com.oceanbase.odc.service.task.executor.server; - -import java.util.HashMap; import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -28,12 +28,14 @@ import com.oceanbase.odc.common.concurrent.ExecutorUtils; import com.oceanbase.odc.core.shared.PreConditions; import com.oceanbase.odc.core.task.TaskThreadFactory; +import com.oceanbase.odc.service.objectstorage.cloud.CloudObjectStorageService; +import com.oceanbase.odc.service.objectstorage.cloud.model.ObjectStorageConfiguration; +import com.oceanbase.odc.service.task.Task; import com.oceanbase.odc.service.task.caller.JobContext; -import com.oceanbase.odc.service.task.executor.task.BaseTask; -import com.oceanbase.odc.service.task.executor.task.ExceptionListener; -import com.oceanbase.odc.service.task.executor.task.Task; -import com.oceanbase.odc.service.task.executor.task.TaskContext; +import com.oceanbase.odc.service.task.executor.TraceDecoratorThreadFactory; import com.oceanbase.odc.service.task.schedule.JobIdentity; +import com.oceanbase.odc.service.task.util.CloudObjectStorageServiceBuilder; +import com.oceanbase.odc.service.task.util.JobUtils; import lombok.extern.slf4j.Slf4j; @@ -44,11 +46,10 @@ * @date 2023/11/24 11:22 */ @Slf4j -public class ThreadPoolTaskExecutor implements TaskExecutor { +class ThreadPoolTaskExecutor implements TaskExecutor { private static final TaskExecutor TASK_EXECUTOR = new ThreadPoolTaskExecutor(); - private final Map> tasks = new HashMap<>(); - private final Map> futures = new HashMap<>(); + private final Map tasks = new ConcurrentHashMap<>(); private final ExecutorService executor; private ThreadPoolTaskExecutor() { @@ -61,38 +62,54 @@ public static TaskExecutor getInstance() { } @Override - synchronized public void execute(BaseTask task, JobContext jc) { + synchronized public void execute(Task task, JobContext jc) { JobIdentity jobIdentity = jc.getJobIdentity(); log.info("Start to execute task, jobIdentity={}.", jobIdentity.getId()); if (tasks.containsKey(jobIdentity)) { throw new IllegalArgumentException("Task already exists, jobIdentity=" + jobIdentity.getId()); } + // init cloud objet storage service and task monitor + CloudObjectStorageService cloudObjectStorageService = buildCloudStorageService(jc); + TaskReporter taskReporter = new TaskReporter(jc.getHostUrls()); + TaskContainer taskContainer = new TaskContainer<>(jc, cloudObjectStorageService, taskReporter, task); + TaskRuntimeInfo taskRuntimeInfo = new TaskRuntimeInfo(taskContainer, taskContainer.getTaskMonitor()); + // first put data in map, avoid concurrent thread access caused task not found exception + tasks.put(jobIdentity, taskRuntimeInfo); Future future = executor.submit(() -> { try { - task.start(new TaskContext() { - @Override - public ExceptionListener getExceptionListener() { - return task; - } - - @Override - public JobContext getJobContext() { - return jc; - } - }); + taskContainer.runTask(); } catch (Exception e) { log.error("Task start failed, jobIdentity={}.", jobIdentity.getId(), e); - task.onException(e); + taskContainer.onException(e); } }); - futures.put(jobIdentity, future); - tasks.put(jobIdentity, task); + taskRuntimeInfo.setFuture(future); + } + + /** + * build task monitor + * + * @param jobContext + * @return + */ + protected CloudObjectStorageService buildCloudStorageService(JobContext jobContext) { + Optional storageConfig = JobUtils.getObjectStorageConfiguration(); + CloudObjectStorageService cloudObjectStorageService = null; + try { + if (storageConfig.isPresent()) { + cloudObjectStorageService = CloudObjectStorageServiceBuilder.build(storageConfig.get()); + } + } catch (Throwable e) { + log.warn("Init cloud object storage service failed, id={}.", jobContext.getJobIdentity().getId(), e); + } + return cloudObjectStorageService; } @Override public boolean cancel(JobIdentity ji) { - Task task = getTask(ji); + TaskRuntimeInfo runtimeInfo = getTaskRuntimeInfo(ji); + TaskContainer task = runtimeInfo.getTaskContainer(); Future stopFuture = executor.submit(task::stop); boolean result = false; try { @@ -117,9 +134,14 @@ public boolean cancel(JobIdentity ji) { } @Override - public BaseTask getTask(JobIdentity ji) { - BaseTask task = tasks.get(ji); - PreConditions.notNull(task, "task", "Task not found, jobIdentity=" + ji.getId()); - return task; + public TaskRuntimeInfo getTaskRuntimeInfo(JobIdentity ji) { + TaskRuntimeInfo runtimeInfo = tasks.get(ji); + PreConditions.notNull(runtimeInfo, "task", "Task not found, jobIdentity=" + ji.getId()); + return runtimeInfo; + } + + @Override + public boolean taskExist(JobIdentity ji) { + return tasks.get(ji) != null; } } diff --git a/server/odc-server/src/main/java/com/oceanbase/odc/server/OdcServer.java b/server/odc-server/src/main/java/com/oceanbase/odc/server/OdcServer.java index b9c3f86245..822e513f92 100644 --- a/server/odc-server/src/main/java/com/oceanbase/odc/server/OdcServer.java +++ b/server/odc-server/src/main/java/com/oceanbase/odc/server/OdcServer.java @@ -19,7 +19,6 @@ import java.time.LocalDateTime; import java.util.Map; -import java.util.Objects; import java.util.Properties; import javax.validation.Validation; @@ -46,11 +45,7 @@ import com.oceanbase.odc.core.alarm.AlarmUtils; import com.oceanbase.odc.core.authority.interceptor.MethodAuthorizedPostProcessor; import com.oceanbase.odc.migrate.AbstractMetaDBMigrate; -import com.oceanbase.odc.server.module.Modules; import com.oceanbase.odc.service.config.SystemConfigBootstrap; -import com.oceanbase.odc.service.task.constants.JobConstants; -import com.oceanbase.odc.service.task.constants.JobEnvKeyConstants; -import com.oceanbase.odc.service.task.executor.TaskApplication; import lombok.extern.slf4j.Slf4j; @@ -83,14 +78,7 @@ public OdcServer(@Qualifier("metadbMigrate") AbstractMetaDBMigrate metadbMigrate * @param args */ public static void main(String[] args) { - if (Objects.equals(SystemUtils.getEnvOrProperty(JobEnvKeyConstants.ODC_BOOT_MODE), - JobConstants.ODC_BOOT_MODE_EXECUTOR)) { - log.info("ODC start as task executor mode"); - Modules.load(); - new TaskApplication().run(args); - log.info("Task executor exit."); - return; - } + AlarmUtils.alarm(SERVER_RESTART, LocalDateTime.now().toString()); initEnv(); System.setProperty("spring.cloud.bootstrap.enabled", "true"); PluginSpringApplication.run(OdcServer.class, args); diff --git a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v1/DBTableControllerV1.java b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v1/DBTableControllerV1.java index f1eeec011a..7c769b78c2 100644 --- a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v1/DBTableControllerV1.java +++ b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v1/DBTableControllerV1.java @@ -35,6 +35,7 @@ import com.oceanbase.odc.service.session.ConnectSessionService; import com.oceanbase.odc.service.state.model.StateName; import com.oceanbase.odc.service.state.model.StatefulRoute; +import com.oceanbase.tools.dbbrowser.model.DBObjectType; import com.oceanbase.tools.dbbrowser.model.DBTable; import io.swagger.annotations.ApiOperation; @@ -70,7 +71,7 @@ public OdcResult detail(@PathVariable String sid) { // parse sid and database name, sid:1-1:d:database:t:tb1 ResourceIdentifier i = ResourceIDParser.parse(sid); return OdcResult.ok(new OdcDBTable(tableService.getTable( - sessionService.nullSafeGet(i.getSid(), true), i.getDatabase(), i.getTable()))); + sessionService.nullSafeGet(i.getSid(), true), i.getDatabase(), i.getTable(), DBObjectType.TABLE))); } @ApiOperation(value = "getUpdateSql", notes = "获取修改表名的sql") diff --git a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/ConnectSessionController.java b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/ConnectSessionController.java index daac207cf9..2ceb511946 100644 --- a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/ConnectSessionController.java +++ b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/ConnectSessionController.java @@ -47,8 +47,8 @@ import com.oceanbase.odc.service.connection.model.DBSessionResp; import com.oceanbase.odc.service.connection.model.MultiSessionsReq; import com.oceanbase.odc.service.db.session.DBSessionService; +import com.oceanbase.odc.service.db.session.KillResult; import com.oceanbase.odc.service.db.session.KillSessionOrQueryReq; -import com.oceanbase.odc.service.db.session.KillSessionResult; import com.oceanbase.odc.service.dml.ValueEncodeType; import com.oceanbase.odc.service.partitionplan.PartitionPlanService; import com.oceanbase.odc.service.partitionplan.model.PartitionPlanPreViewResp; @@ -213,7 +213,7 @@ public SuccessResponse killQuery(@PathVariable String sessionId) { */ @ApiOperation(value = "kill session", notes = "终止会话接口") @RequestMapping(value = "/sessions/killSession", method = RequestMethod.POST) - public SuccessResponse> killSession(@RequestBody KillSessionOrQueryReq req) { + public SuccessResponse> killSession(@RequestBody KillSessionOrQueryReq req) { return Responses.success(consoleService.killSessionOrQuery(req)); } diff --git a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/DBPLController.java b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/DBPLController.java index 39052b30d7..469783fbbd 100644 --- a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/DBPLController.java +++ b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/DBPLController.java @@ -27,6 +27,7 @@ import com.oceanbase.odc.service.common.response.Responses; import com.oceanbase.odc.service.common.response.SuccessResponse; import com.oceanbase.odc.service.common.util.SidUtils; +import com.oceanbase.odc.service.db.DBPLModifyHelper; import com.oceanbase.odc.service.db.DBPLService; import com.oceanbase.odc.service.db.model.CallFunctionReq; import com.oceanbase.odc.service.db.model.CallFunctionResp; @@ -34,6 +35,8 @@ import com.oceanbase.odc.service.db.model.CallProcedureResp; import com.oceanbase.odc.service.db.model.CompileResult; import com.oceanbase.odc.service.db.model.DBMSOutput; +import com.oceanbase.odc.service.db.model.EditPLReq; +import com.oceanbase.odc.service.db.model.EditPLResp; import com.oceanbase.odc.service.db.model.PLIdentity; import com.oceanbase.odc.service.session.ConnectSessionService; import com.oceanbase.odc.service.state.model.StateName; @@ -50,6 +53,8 @@ public class DBPLController { private DBPLService plService; @Autowired private ConnectSessionService sessionService; + @Autowired + private DBPLModifyHelper dBPLmodifyHelper; @ApiOperation(value = "compile", notes = "compile a pl object") @RequestMapping(value = "/compile/{sid}", method = RequestMethod.POST) @@ -110,4 +115,12 @@ public SuccessResponse getNameAndType(@PathVariable String sid, @Req return Responses.ok(PLIdentity.of(identity.getType(), identity.getName())); } + @ApiOperation(value = "editPL", notes = "edit DDL of a pl object") + @RequestMapping(value = "/editPL/{sid}", method = RequestMethod.POST) + @StatefulRoute(stateName = StateName.DB_SESSION, stateIdExpression = "#sid") + public SuccessResponse editPL(@PathVariable String sid, @RequestBody EditPLReq editPLReq) + throws Exception { + return Responses.ok(this.dBPLmodifyHelper.editPL(SidUtils.getSessionId(sid), editPLReq)); + } + } diff --git a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/DBSchemaController.java b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/DBSchemaController.java index 24d908a351..9d9586880e 100644 --- a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/DBSchemaController.java +++ b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/DBSchemaController.java @@ -16,7 +16,10 @@ package com.oceanbase.odc.server.web.controller.v2; import java.sql.SQLException; +import java.util.Collections; +import java.util.List; +import org.apache.commons.collections4.CollectionUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; @@ -28,6 +31,7 @@ import com.oceanbase.odc.service.connection.table.TableService; import com.oceanbase.odc.service.connection.table.model.QueryTableParams; import com.oceanbase.odc.service.connection.table.model.Table; +import com.oceanbase.tools.dbbrowser.model.DBObjectType; import io.swagger.annotations.ApiOperation; @@ -48,10 +52,15 @@ public class DBSchemaController { @RequestMapping(value = "/tables", method = RequestMethod.GET) public ListResponse
list(@RequestParam(name = "databaseId") Long databaseId, @RequestParam(name = "includePermittedAction", required = false, - defaultValue = "false") boolean includePermittedAction) + defaultValue = "false") boolean includePermittedAction, + @RequestParam(required = false, name = "type") List types) throws SQLException, InterruptedException { + if (CollectionUtils.isEmpty(types)) { + types = Collections.singletonList(DBObjectType.TABLE); + } QueryTableParams params = QueryTableParams.builder() .databaseId(databaseId) + .types(types) .includePermittedAction(includePermittedAction) .build(); return Responses.list(tableService.list(params)); diff --git a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/DBTableController.java b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/DBTableController.java index ccd0b8089d..90b7d9bb76 100644 --- a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/DBTableController.java +++ b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/DBTableController.java @@ -40,10 +40,13 @@ import com.oceanbase.odc.service.session.ConnectSessionService; import com.oceanbase.odc.service.state.model.StateName; import com.oceanbase.odc.service.state.model.StatefulRoute; +import com.oceanbase.tools.dbbrowser.model.DBObjectType; import com.oceanbase.tools.dbbrowser.model.DBSchema; import com.oceanbase.tools.dbbrowser.model.DBTable; import com.oceanbase.tools.dbbrowser.model.datatype.DataType; +import io.swagger.annotations.ApiOperation; + @RestController @RequestMapping("api/v2/connect/sessions") public class DBTableController { @@ -69,16 +72,18 @@ public ListResponse listTables(@PathVariable String sessionId, } } + @ApiOperation(value = "getTable", notes = "get table") @GetMapping(value = {"/{sessionId}/databases/{databaseName}/tables/{tableName}", "/{sessionId}/currentDatabase/tables/{tableName}"}) @StatefulRoute(stateName = StateName.DB_SESSION, stateIdExpression = "#sessionId") public SuccessResponse getTable(@PathVariable String sessionId, @PathVariable(required = false) String databaseName, - @PathVariable String tableName) { + @PathVariable String tableName, + @RequestParam(required = false, name = "type", defaultValue = "TABLE") DBObjectType type) { Base64.Decoder decoder = Base64.getDecoder(); tableName = new String(decoder.decode(tableName)); ConnectionSession session = sessionService.nullSafeGet(sessionId, true); - return Responses.success(tableService.getTable(session, databaseName, tableName)); + return Responses.success(tableService.getTable(session, databaseName, tableName, type)); } @PostMapping(value = {"/{sessionId}/databases/{databaseName}/tables/generateCreateTableDDL", @@ -120,4 +125,15 @@ public ListResponse getPartitionKeyDataTypes(@PathVariable String sess return Responses.list(this.partitionPlanService.getPartitionKeyDataTypes(sessionId, databaseId, tableName)); } + @ApiOperation(value = "syncExternalTableFiles", notes = "sync external table files") + @PostMapping(value = "/{sessionId}/databases/{databaseName}/externalTables/" + + "{externalTableName}/syncExternalTableFiles") + public SuccessResponse syncExternalTableFiles(@PathVariable String sessionId, + @PathVariable(required = false) String databaseName, + @PathVariable(required = true, name = "externalTableName") String externalTableName) { + Base64.Decoder decoder = Base64.getDecoder(); + externalTableName = new String(decoder.decode(externalTableName)); + ConnectionSession session = sessionService.nullSafeGet(sessionId, true); + return Responses.success(tableService.syncExternalTableFiles(session, databaseName, externalTableName)); + } } diff --git a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/FlowInstanceController.java b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/FlowInstanceController.java index b98c5b7be0..e9f2a0e340 100644 --- a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/FlowInstanceController.java +++ b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/FlowInstanceController.java @@ -118,7 +118,7 @@ public PaginatedResponse listFlowInstances( @RequestParam(name = "approveByCurrentUser") Boolean approveByCurrentUser, @RequestParam(required = false, name = "containsAll", defaultValue = "false") Boolean containsAll, @RequestParam(required = false, name = "parentInstanceId") Long parentInstanceId, - @RequestParam(required = false, name = "projectId") Long projectId) { + @RequestParam(required = false, name = "projectId") Set projectIds) { QueryFlowInstanceParams params = QueryFlowInstanceParams.builder() .connectionIds(connectionIds) .id(fuzzySearchKeyword) @@ -132,7 +132,7 @@ public PaginatedResponse listFlowInstances( .approveByCurrentUser(approveByCurrentUser) .containsAll(containsAll) .parentInstanceId(parentInstanceId) - .projectId(projectId) + .projectIds(projectIds) .build(); return Responses.paginated(flowInstanceService.list(pageable, params)); } @@ -199,14 +199,14 @@ public SuccessResponse getMetaInfo() { @ApiOperation(value = "getResult", notes = "获取任务结果") @RequestMapping(value = "/{id:[\\d]+}/tasks/result", method = RequestMethod.GET) public ListResponse getResult(@PathVariable Long id) throws IOException { - return Responses.list(flowTaskInstanceService.getResult(id, false)); + return Responses.list(flowTaskInstanceService.getResult(id)); } @ApiOperation(value = "getResult", notes = "获取任务结果") @RequestMapping(value = "/{flowInstanceId:[\\d]+}/tasks/{nodeInstanceId:[\\d]+}/result", method = RequestMethod.GET) public ListResponse getResult( @PathVariable Long flowInstanceId, @PathVariable Long nodeInstanceId) throws IOException { - return Responses.list(flowTaskInstanceService.getResult(flowInstanceId, nodeInstanceId, false)); + return Responses.list(flowTaskInstanceService.getResult(flowInstanceId, nodeInstanceId)); } @ApiOperation(value = "download", notes = "下载任务数据，仅用于 模拟数据、导出、数据库变更 任务") diff --git a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/ProjectController.java b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/ProjectController.java index b5e4178e43..f0b94657e5 100644 --- a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/ProjectController.java +++ b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/ProjectController.java @@ -16,6 +16,7 @@ package com.oceanbase.odc.server.web.controller.v2; import java.util.List; +import java.util.Set; import javax.validation.Valid; @@ -35,6 +36,7 @@ import com.oceanbase.odc.service.collaboration.project.model.Project.ProjectMember; import com.oceanbase.odc.service.collaboration.project.model.QueryProjectParams; import com.oceanbase.odc.service.collaboration.project.model.SetArchivedReq; +import com.oceanbase.odc.service.collaboration.project.model.TicketReference; import com.oceanbase.odc.service.common.response.ListResponse; import com.oceanbase.odc.service.common.response.PaginatedResponse; import com.oceanbase.odc.service.common.response.Responses; @@ -131,4 +133,16 @@ public SuccessResponse setArchived(@PathVariable Long id, return Responses.success(projectService.setArchived(id, setArchivedReq)); } + @ApiOperation(value = "batchDeleteProject", notes = "Delete projects") + @RequestMapping(value = "/projects/batchDelete", method = RequestMethod.POST) + public SuccessResponse batchDelete(@RequestBody Set ids) throws InterruptedException { + return Responses.success(projectService.batchDelete(ids)); + } + + @ApiOperation(value = "listUnfinishedTickets", notes = "List unfinished ticket references of a project") + @RequestMapping(value = "/projects/{id:[\\d]+}/unfinishedTickets", method = RequestMethod.GET) + public SuccessResponse listUnfinishedTickets(@PathVariable Long id) { + return Responses.success(projectService.getProjectTicketReference(id)); + } + } diff --git a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/SSOController.java b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/SSOController.java index 14ba29ea17..611bf555c1 100644 --- a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/SSOController.java +++ b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/SSOController.java @@ -27,8 +27,10 @@ import com.oceanbase.odc.service.common.response.Responses; import com.oceanbase.odc.service.common.response.SuccessResponse; +import com.oceanbase.odc.service.integration.IntegrationService; +import com.oceanbase.odc.service.integration.SSOCredential; import com.oceanbase.odc.service.integration.model.IntegrationConfig; -import com.oceanbase.odc.service.integration.oauth2.Oauth2StateManager; +import com.oceanbase.odc.service.integration.oauth2.SSOStateManager; import com.oceanbase.odc.service.integration.oauth2.SSOTestInfo; import com.oceanbase.odc.service.integration.oauth2.TestLoginManager; import com.oceanbase.odc.service.state.model.StateName; @@ -42,12 +44,15 @@ public class SSOController { private TestLoginManager testLoginManager; @Autowired - private Oauth2StateManager oauth2StateManager; + private SSOStateManager SSOStateManager; + + @Autowired + private IntegrationService integrationService; @PostMapping(value = "/test/start") public SuccessResponse addTestClientRegistration(@RequestBody IntegrationConfig config, - @RequestParam(required = false) String type) { - return Responses.ok(testLoginManager.getSSOTestInfo(config, type)); + @RequestParam(required = false) String type, @RequestParam(required = false) String odcBackUrl) { + return Responses.ok(testLoginManager.getSSOTestInfo(config, type, odcBackUrl)); } /** @@ -65,7 +70,12 @@ public SuccessResponse testUserInfo(String testId) { @GetMapping("/state") @StatefulRoute(stateName = StateName.UUID_STATEFUL_ID, stateIdExpression = "#state") public SuccessResponse> getTestClientInfo(@RequestParam String state) { - return Responses.ok(oauth2StateManager.getStateParameters(state)); + return Responses.ok(SSOStateManager.getStateParameters(state)); + } + + @GetMapping("/credential") + public SuccessResponse generateSSOCredential() { + return Responses.ok(integrationService.generateSSOCredential()); } } diff --git a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/ScheduleControllerHist.java b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/ScheduleControllerHist.java index 2422368e12..bcee282d1e 100644 --- a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/ScheduleControllerHist.java +++ b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/ScheduleControllerHist.java @@ -71,7 +71,7 @@ public PaginatedResponse list( @RequestParam(required = false, name = "startTime") Date startTime, @RequestParam(required = false, name = "endTime") Date endTime, @RequestParam(required = false, name = "creator") String creator, - @RequestParam(required = false, name = "projectId") Long projectId) { + @RequestParam(required = false, name = "projectId") Set projectIds) { QueryScheduleParams req = QueryScheduleParams.builder() .id(id) @@ -81,7 +81,7 @@ public PaginatedResponse list( .startTime(startTime) .endTime(endTime) .creator(creator) - .projectId(projectId) + .projectIds(projectIds) .build(); return Responses.paginated(scheduleService.list(pageable, req)); diff --git a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/TaskController.java b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/TaskController.java index 57f0b4847a..7476b5685c 100644 --- a/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/TaskController.java +++ b/server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/TaskController.java @@ -26,10 +26,10 @@ import com.oceanbase.odc.service.common.response.Responses; import com.oceanbase.odc.service.common.response.SuccessResponse; import com.oceanbase.odc.service.datasecurity.DataMaskingService; -import com.oceanbase.odc.service.task.executor.server.HeartbeatRequest; -import com.oceanbase.odc.service.task.executor.task.DefaultTaskResult; -import com.oceanbase.odc.service.task.runtime.QuerySensitiveColumnReq; -import com.oceanbase.odc.service.task.runtime.QuerySensitiveColumnResp; +import com.oceanbase.odc.service.task.base.databasechange.QuerySensitiveColumnReq; +import com.oceanbase.odc.service.task.base.databasechange.QuerySensitiveColumnResp; +import com.oceanbase.odc.service.task.executor.HeartbeatRequest; +import com.oceanbase.odc.service.task.executor.TaskResult; import com.oceanbase.odc.service.task.service.TaskFrameworkService; import io.swagger.annotations.ApiOperation; @@ -53,7 +53,7 @@ public class TaskController { @ApiOperation(value = "updateResult", notes = "update task result") @RequestMapping(value = "/result", method = RequestMethod.POST) - public SuccessResponse updateResult(@RequestBody DefaultTaskResult taskResult) { + public SuccessResponse updateResult(@RequestBody TaskResult taskResult) { if (log.isDebugEnabled()) { log.debug("Accept task result {}.", JsonUtils.toJson(taskResult)); } diff --git a/server/odc-server/src/main/resources/config/application-clientMode.yml b/server/odc-server/src/main/resources/config/application-clientMode.yml index 6ee099ef2a..395a1da8c9 100644 --- a/server/odc-server/src/main/resources/config/application-clientMode.yml +++ b/server/odc-server/src/main/resources/config/application-clientMode.yml @@ -21,6 +21,7 @@ spring: - org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration - org.springframework.boot.actuate.autoconfigure.security.servlet.ManagementWebSecurityAutoConfiguration - org.springframework.boot.autoconfigure.security.oauth2.client.servlet.OAuth2ClientAutoConfiguration + - org.springframework.boot.autoconfigure.security.saml2.Saml2RelyingPartyAutoConfiguration - org.springframework.boot.actuate.autoconfigure.metrics.export.prometheus.PrometheusMetricsExportAutoConfiguration jpa: database-platform: org.hibernate.dialect.H2Dialect diff --git a/server/odc-server/src/main/resources/config/application.yml b/server/odc-server/src/main/resources/config/application.yml index ec691b7e59..7afa094517 100644 --- a/server/odc-server/src/main/resources/config/application.yml +++ b/server/odc-server/src/main/resources/config/application.yml @@ -53,6 +53,10 @@ server: servlet: session: timeout: 8h + cookie: + name: JSESSIONID + http-only: true + secure: true encoding: charset: UTF-8 tomcat: diff --git a/server/odc-server/src/main/resources/data.sql b/server/odc-server/src/main/resources/data.sql index b3441fef7f..b64d204ef5 100644 --- a/server/odc-server/src/main/resources/data.sql +++ b/server/odc-server/src/main/resources/data.sql @@ -847,10 +847,22 @@ INSERT INTO config_system_configuration(`key`, `value`, `description`) VALUES('o INSERT INTO config_system_configuration(`key`, `value`, `description`) VALUES('odc.integration.git.repository-max-cached-size', '1000', 'The maximum number of repository copies cached in ODC') ON DUPLICATE KEY UPDATE `id`=`id`; + -- + -- v4.3.3 + -- + INSERT INTO config_system_configuration(`key`, `value`, `description`) VALUES('server.servlet.session.cookie.secure', + 'false', 'Enable secure cookie or not, default value false') ON DUPLICATE KEY UPDATE `id`=`id`; + INSERT INTO config_system_configuration(`key`, `value`, `description`) VALUES('odc.features.logicaldatabase.enabled', 'true', 'Whether to enable the logical database feature, default is true, indicating enabled.') ON DUPLICATE KEY UPDATE `id`=`id`; INSERT INTO `config_system_configuration` (`key`, `value`, `application`, `profile`, `label`, `description`) VALUES ('odc.session.kill-query-or-session.max-supported-ob-version', '4.2.5', 'odc', 'default', 'master', 'Max OBVersion kill session or kill query supported, only take effect when value greater than 0') -ON DUPLICATE KEY UPDATE `id`=`id`; \ No newline at end of file +ON DUPLICATE KEY UPDATE `id`=`id`; +INSERT INTO config_system_configuration ( `key`, `value`, `description` ) VALUES('odc.task.dlm.session-limiting.enabled', 'true', +'Explosion-proof current limiting switch of mysql/oracle' ) +ON DUPLICATE KEY UPDATE `id` = `id`; +INSERT INTO config_system_configuration ( `key`, `value`, `description` ) VALUES('odc.task.dlm.session-limiting-ratio', '25', +'The ratio of oracle/mysql active sessions to the maximum number of connections allowed' ) +ON DUPLICATE KEY UPDATE `id` = `id`; \ No newline at end of file diff --git a/server/odc-server/src/main/resources/log4j2-task.xml b/server/odc-server/src/main/resources/log4j2-task.xml index 4697ceeb3c..7c9e6965e1 100644 --- a/server/odc-server/src/main/resources/log4j2-task.xml +++ b/server/odc-server/src/main/resources/log4j2-task.xml @@ -55,6 +55,11 @@ [%d{yyyy-MM-dd HH:mm:ss z}] [%X{taskId}] [%p] %m%n + + + + + @@ -78,6 +83,11 @@ [%d{yyyy-MM-dd HH:mm:ss z}] [%X{taskId}] [%p] %m%n + + + + + @@ -99,7 +109,7 @@ - + @@ -117,6 +127,9 @@ + + + diff --git a/server/integration-test/src/test/java/com/oceanbase/odc/service/task/EmbedServerTest.java b/server/odc-server/src/test/java/com/oceanbase/odc/agent/runtime/EmbedServerTest.java similarity index 90% rename from server/integration-test/src/test/java/com/oceanbase/odc/service/task/EmbedServerTest.java rename to server/odc-server/src/test/java/com/oceanbase/odc/agent/runtime/EmbedServerTest.java index 8b425de686..b253f53eb6 100644 --- a/server/integration-test/src/test/java/com/oceanbase/odc/service/task/EmbedServerTest.java +++ b/server/odc-server/src/test/java/com/oceanbase/odc/agent/runtime/EmbedServerTest.java @@ -13,14 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -package com.oceanbase.odc.service.task; +package com.oceanbase.odc.agent.runtime; import org.junit.Ignore; import org.junit.Test; -import com.oceanbase.odc.service.task.executor.server.EmbedServer; - import lombok.extern.slf4j.Slf4j; /** diff --git a/server/odc-server/src/test/java/com/oceanbase/odc/agent/runtime/SimpleTask.java b/server/odc-server/src/test/java/com/oceanbase/odc/agent/runtime/SimpleTask.java new file mode 100644 index 0000000000..6315299dde --- /dev/null +++ b/server/odc-server/src/test/java/com/oceanbase/odc/agent/runtime/SimpleTask.java @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.agent.runtime; + +import com.oceanbase.odc.service.task.base.TaskBase; +import com.oceanbase.odc.service.task.caller.JobContext; + +/** + * @author longpeng.zlp + * @date 2024/11/12 15:07 + */ +public final class SimpleTask extends TaskBase { + private final boolean shouldThrowException; + + public SimpleTask() { + this.shouldThrowException = false; + } + + public SimpleTask(boolean shouldThrowException) { + this.shouldThrowException = shouldThrowException; + } + + @Override + protected void doInit(JobContext context) throws Exception {} + + @Override + public boolean start() throws Exception { + if (shouldThrowException) { + throw new IllegalStateException("exception should be thrown"); + } + return true; + } + + @Override + public void stop() throws Exception {} + + @Override + public void close() throws Exception {} + + @Override + public double getProgress() { + return 100; + } + + @Override + public String getTaskResult() { + return "res"; + } +} diff --git a/server/odc-server/src/test/java/com/oceanbase/odc/agent/runtime/TaskApplicationTest.java b/server/odc-server/src/test/java/com/oceanbase/odc/agent/runtime/TaskApplicationTest.java new file mode 100644 index 0000000000..6c8880075e --- /dev/null +++ b/server/odc-server/src/test/java/com/oceanbase/odc/agent/runtime/TaskApplicationTest.java @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.agent.runtime; + +import java.util.HashMap; +import java.util.Map; + +import org.junit.Assert; +import org.junit.Test; + +import com.oceanbase.odc.common.json.JsonUtils; +import com.oceanbase.odc.core.shared.constant.TaskStatus; +import com.oceanbase.odc.service.task.caller.DefaultJobContext; +import com.oceanbase.odc.service.task.constants.JobEnvKeyConstants; +import com.oceanbase.odc.service.task.schedule.JobIdentity; +import com.oceanbase.odc.service.task.util.JobUtils; + +/** + * @author yaobin + * @date 2023-12-14 + * @since 4.2.4 + */ +public class TaskApplicationTest { + @Test + public void test_executeDatabaseChangeTask_run() { + Long exceptedTaskId = System.currentTimeMillis(); + JobIdentity jobIdentity = JobIdentity.of(exceptedTaskId); + setJobContextInSystemProperty(jobIdentity); + startTaskApplication(); + assertRunningResult(jobIdentity); + } + + + private void setJobContextInSystemProperty(JobIdentity jobIdentity) { + Map envMap = buildConfig(jobIdentity); + JobUtils.encryptEnvironments(envMap); + envMap.forEach(System::setProperty); + } + + private void assertRunningResult(JobIdentity ji) { + + try { + long endTime = System.currentTimeMillis() + 20000; + TaskExecutor taskExecutor = ThreadPoolTaskExecutor.getInstance(); + while (System.currentTimeMillis() < endTime) { + if (!taskExecutor.taskExist(ji)) { + Thread.sleep(1000); + continue; + } + TaskRuntimeInfo taskRuntimeInfo = taskExecutor.getTaskRuntimeInfo(ji); + TaskContainer task = taskRuntimeInfo.getTaskContainer(); + task.taskMonitor.markLogMetaCollected(); + if (TaskStatus.DONE == task.getStatus()) { + return; + } else { + Thread.sleep(1000); + } + } + // not check pass + Assert.assertFalse(true); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + + } + + private Map buildConfig(JobIdentity jobIdentity) { + Map ret = new HashMap<>(); + ret.put(JobEnvKeyConstants.ODC_TASK_RUN_MODE, "K8S"); + ret.put(JobEnvKeyConstants.ODC_BOOT_MODE, "TASK_EXECUTOR"); + ret.put(JobEnvKeyConstants.ODC_EXECUTOR_USER_ID, "1"); + DefaultJobContext defaultJobContext = new DefaultJobContext(); + defaultJobContext.setJobClass(SimpleTask.class.getName()); + defaultJobContext.setJobIdentity(jobIdentity); + ret.put(JobEnvKeyConstants.ODC_JOB_CONTEXT, JsonUtils.toJson(defaultJobContext)); + ret.put(JobEnvKeyConstants.ODC_LOG_DIRECTORY, "log"); + ret.put(JobEnvKeyConstants.ODC_EXECUTOR_PORT, "8080"); + ret.put(JobEnvKeyConstants.REPORT_ENABLED, "false"); + return ret; + } + + private void startTaskApplication() { + new Thread(() -> new TaskApplication().run(null)).start(); + } + +} diff --git a/server/odc-server/src/test/java/com/oceanbase/odc/agent/runtime/TaskContainerTest.java b/server/odc-server/src/test/java/com/oceanbase/odc/agent/runtime/TaskContainerTest.java new file mode 100644 index 0000000000..f978141bf0 --- /dev/null +++ b/server/odc-server/src/test/java/com/oceanbase/odc/agent/runtime/TaskContainerTest.java @@ -0,0 +1,106 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.agent.runtime; + +import java.util.HashMap; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.ArgumentMatchers; +import org.mockito.MockedStatic; +import org.mockito.Mockito; + +import com.oceanbase.odc.common.util.StringUtils; +import com.oceanbase.odc.common.util.SystemUtils; +import com.oceanbase.odc.service.objectstorage.cloud.CloudObjectStorageService; +import com.oceanbase.odc.service.task.Task; +import com.oceanbase.odc.service.task.caller.DefaultJobContext; +import com.oceanbase.odc.service.task.caller.JobContext; +import com.oceanbase.odc.service.task.constants.JobEnvKeyConstants; +import com.oceanbase.odc.service.task.executor.TaskResult; +import com.oceanbase.odc.service.task.schedule.JobIdentity; + +/** + * @author longpeng.zlp + * @date 2024/10/11 14:11 + */ +public class TaskContainerTest { + private DefaultJobContext jobContext; + + @Before + public void init() { + jobContext = new DefaultJobContext(); + jobContext.setJobParameters(new HashMap<>()); + jobContext.setJobProperties(new HashMap<>()); + JobIdentity jobIdentity = new JobIdentity(); + jobIdentity.setId(1L); + jobContext.setJobIdentity(jobIdentity); + jobContext.setJobClass(SimpleTask.class.getName()); + } + + + @Test + public void testExceptionListenerNormal() { + try (MockedStatic mockSystemUtil = Mockito.mockStatic(SystemUtils.class)) { + mockSystemUtil.when(() -> { + SystemUtils.getEnvOrProperty(JobEnvKeyConstants.ODC_EXECUTOR_PORT); + }).thenReturn("9099"); + SimpleTask dummyBaseTask = new SimpleTask(false); + TaskContainer taskContainer = buildTaskContainer(jobContext, dummyBaseTask); + taskContainer.runTask(); + TaskReporter taskReporter = taskContainer.taskMonitor.getReporter(); + ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(TaskResult.class); + Mockito.verify(taskReporter).report(ArgumentMatchers.any(), argumentCaptor.capture()); + Assert.assertNull(argumentCaptor.getValue().getErrorMessage()); + } + } + + @Test + public void testExceptionListenerWithException() { + try (MockedStatic mockSystemUtil = Mockito.mockStatic(SystemUtils.class)) { + mockSystemUtil.when(() -> { + SystemUtils.getEnvOrProperty(JobEnvKeyConstants.ODC_EXECUTOR_PORT); + }).thenReturn("9099"); + SimpleTask dummyBaseTask = new SimpleTask(true); + TaskContainer taskContainer = buildTaskContainer(jobContext, dummyBaseTask); + taskContainer.runTask(); + TaskReporter taskReporter = taskContainer.taskMonitor.getReporter(); + ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(TaskResult.class); + Mockito.verify(taskReporter).report(ArgumentMatchers.any(), argumentCaptor.capture()); + Assert.assertTrue( + StringUtils.contains(argumentCaptor.getValue().getErrorMessage(), "exception should be thrown")); + } + } + + @Test + public void testTaskContainerOnException() { + TaskContainer taskContainer = new TaskContainer<>(jobContext, Mockito.mock( + CloudObjectStorageService.class), Mockito.mock(TaskReporter.class), new SimpleTask(false)); + Assert.assertNull(taskContainer.getError()); + taskContainer.onException(new Throwable("error")); + Throwable ex = taskContainer.getError(); + Assert.assertEquals(ex.getMessage(), "error"); + Assert.assertNull(taskContainer.getError()); + } + + private static TaskContainer buildTaskContainer(JobContext jobContext, Task task) { + TaskReporter taskReporter = Mockito.mock(TaskReporter.class); + Mockito.when(taskReporter.report(ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(true); + return new TaskContainer<>(jobContext, Mockito.mock(CloudObjectStorageService.class), taskReporter, task); + } +} diff --git a/server/odc-server/src/test/java/com/oceanbase/odc/agent/runtime/TaskMonitorTest.java b/server/odc-server/src/test/java/com/oceanbase/odc/agent/runtime/TaskMonitorTest.java new file mode 100644 index 0000000000..ccc96ca2fd --- /dev/null +++ b/server/odc-server/src/test/java/com/oceanbase/odc/agent/runtime/TaskMonitorTest.java @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.agent.runtime; + +import org.junit.Assert; +import org.junit.Test; +import org.mockito.ArgumentMatchers; +import org.mockito.Mockito; + +import com.oceanbase.odc.service.objectstorage.cloud.CloudObjectStorageService; +import com.oceanbase.odc.service.task.executor.TaskResult; + +/** + * @author longpeng.zlp + * @date 2024/11/7 17:45 + */ +public class TaskMonitorTest { + + @Test + public void testTaskMonitorReportRetryFailed() { + TaskReporter taskReporter = Mockito.mock(TaskReporter.class); + Mockito.when(taskReporter.report(ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(false); + TaskMonitor taskMonitor = new TaskMonitor(Mockito.mock(TaskContainer.class), taskReporter, Mockito.mock( + CloudObjectStorageService.class)); + Assert.assertFalse(taskMonitor.reportTaskResultWithRetry(new TaskResult(), 3, 1)); + } + + @Test + public void testTaskMonitorReportRetrySuccess() { + TaskReporter taskReporter = Mockito.mock(TaskReporter.class); + Mockito.when(taskReporter.report(ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(true); + TaskMonitor taskMonitor = new TaskMonitor(Mockito.mock(TaskContainer.class), taskReporter, Mockito.mock( + CloudObjectStorageService.class)); + Assert.assertTrue(taskMonitor.reportTaskResultWithRetry(new TaskResult(), 3, 1)); + } +} diff --git a/server/odc-service/pom.xml b/server/odc-service/pom.xml index c3ec2c1df2..1590a546b1 100644 --- a/server/odc-service/pom.xml +++ b/server/odc-service/pom.xml @@ -1,11 +1,11 @@ + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 com.oceanbase odc-parent - 4.3.2-SNAPSHOT + 4.3.3-SNAPSHOT ../../pom.xml odc-service @@ -116,6 +116,18 @@ org.bouncycastle bcpkix-jdk15on + + org.bouncycastle + bcprov-jdk18on + + + org.bouncycastle + bcpkix-jdk18on + + + org.bouncycastle + bcutil-jdk18on + org.springframework.cloud spring-cloud-context @@ -141,6 +153,10 @@ spring-security-oauth2-jose 5.7.0 + + org.springframework.security + spring-security-saml2-service-provider + org.springframework.security spring-security-web diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/config/BaseAuthConfiguration.java b/server/odc-service/src/main/java/com/oceanbase/odc/config/BaseAuthConfiguration.java index 396dfb777f..5f4edb1899 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/config/BaseAuthConfiguration.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/config/BaseAuthConfiguration.java @@ -34,9 +34,9 @@ import com.oceanbase.odc.core.authority.permission.PermissionProvider; import com.oceanbase.odc.core.authority.session.factory.DefaultSecuritySessionFactory; import com.oceanbase.odc.metadb.iam.PermissionRepository; -import com.oceanbase.odc.metadb.iam.resourcerole.UserResourceRoleRepository; import com.oceanbase.odc.service.iam.ResourcePermissionExtractor; import com.oceanbase.odc.service.iam.ResourceRoleBasedPermissionExtractor; +import com.oceanbase.odc.service.iam.ResourceRoleService; import com.oceanbase.odc.service.iam.auth.AuthenticationFacade; import com.oceanbase.odc.service.iam.auth.DefaultPermissionProvider; import com.oceanbase.odc.service.iam.auth.EmptyAuthenticator; @@ -55,10 +55,10 @@ public abstract class BaseAuthConfiguration { @Bean public SecurityManager servletSecurityManager(PermissionRepository permissionRepository, - ResourcePermissionExtractor permissionMapper, UserResourceRoleRepository resourceRoleRepository, + ResourcePermissionExtractor permissionMapper, ResourceRoleService resourceRoleService, ResourceRoleBasedPermissionExtractor resourceRoleBasedPermissionExtractor, AuthenticationFacade authenticationFacade) { - Collection authorizers = authorizers(permissionRepository, permissionMapper, resourceRoleRepository, + Collection authorizers = authorizers(permissionRepository, permissionMapper, resourceRoleService, resourceRoleBasedPermissionExtractor); DefaultAuthorizerManager authorizerManager = new DefaultAuthorizerManager(authorizers); PermissionStrategy permissionStrategy = permissionStrategy(); @@ -94,7 +94,6 @@ protected ReturnValueProvider returnValueProvider(AuthorizerManager manager, Per } protected abstract Collection authorizers(PermissionRepository permissionRepository, - ResourcePermissionExtractor resourcePermissionExtractor, UserResourceRoleRepository resourceRoleRepository, + ResourcePermissionExtractor resourcePermissionExtractor, ResourceRoleService resourceRoleService, ResourceRoleBasedPermissionExtractor resourceRoleBasedPermissionExtractor); - } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/config/BeanConfiguration.java b/server/odc-service/src/main/java/com/oceanbase/odc/config/BeanConfiguration.java index a4b60b267a..8ea01f8a72 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/config/BeanConfiguration.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/config/BeanConfiguration.java @@ -28,13 +28,10 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Primary; import org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder; import org.springframework.security.crypto.password.PasswordEncoder; import org.springframework.security.web.firewall.DefaultHttpFirewall; import org.springframework.security.web.firewall.HttpFirewall; -import org.springframework.session.web.http.CookieSerializer; -import org.springframework.session.web.http.DefaultCookieSerializer; import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.databind.BeanProperty; @@ -111,14 +108,6 @@ public CloudMetadataClient cloudMetadataClient() { return new NullCloudMetadataClient(); } - @Bean - @Primary - public CookieSerializer cookieSerializer() { - DefaultCookieSerializer serializer = new DefaultCookieSerializer(); - serializer.setCookieName("JSESSIONID"); - return serializer; - } - @Slf4j public static class NullCloudMetadataClient implements CloudMetadataClient { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/config/DefaultAuthConfiguration.java b/server/odc-service/src/main/java/com/oceanbase/odc/config/DefaultAuthConfiguration.java index b74408bdb1..f9e14e3d8a 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/config/DefaultAuthConfiguration.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/config/DefaultAuthConfiguration.java @@ -20,9 +20,10 @@ import com.oceanbase.odc.core.authority.auth.Authorizer; import com.oceanbase.odc.metadb.iam.PermissionRepository; -import com.oceanbase.odc.metadb.iam.resourcerole.UserResourceRoleRepository; import com.oceanbase.odc.service.iam.ResourcePermissionExtractor; import com.oceanbase.odc.service.iam.ResourceRoleBasedPermissionExtractor; +import com.oceanbase.odc.service.iam.ResourceRoleService; +import com.oceanbase.odc.service.iam.auth.ComposedAuthorizer; import com.oceanbase.odc.service.iam.auth.DefaultAuthorizer; import com.oceanbase.odc.service.iam.auth.ResourceRoleAuthorizer; @@ -38,10 +39,13 @@ public class DefaultAuthConfiguration extends BaseAuthConfiguration { @Override protected Collection authorizers(PermissionRepository permissionRepository, - ResourcePermissionExtractor resourcePermissionExtractor, UserResourceRoleRepository resourceRoleRepository, + ResourcePermissionExtractor resourcePermissionExtractor, ResourceRoleService resourceRoleService, ResourceRoleBasedPermissionExtractor resourceRoleBasedPermissionExtractor) { - return Arrays.asList(new DefaultAuthorizer(permissionRepository, resourcePermissionExtractor), - new ResourceRoleAuthorizer(resourceRoleRepository, resourceRoleBasedPermissionExtractor)); + DefaultAuthorizer defaultAuthorizer = new DefaultAuthorizer(permissionRepository, resourcePermissionExtractor); + ResourceRoleAuthorizer resourceRoleAuthorizer = + new ResourceRoleAuthorizer(resourceRoleService, resourceRoleBasedPermissionExtractor); + ComposedAuthorizer composedAuthorizer = new ComposedAuthorizer(defaultAuthorizer, resourceRoleAuthorizer); + return Arrays.asList(defaultAuthorizer, resourceRoleAuthorizer, composedAuthorizer); } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/config/HookConfiguration.java b/server/odc-service/src/main/java/com/oceanbase/odc/config/HookConfiguration.java index a2d6ff048e..57a715a17a 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/config/HookConfiguration.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/config/HookConfiguration.java @@ -147,7 +147,7 @@ private void projectReferenceCheck(Long userId, Long organizationId) { .collect(Collectors.toMap(ProjectEntity::getId, p -> p)); Map> projectId2ResourceRoleNames = - resourceRoleService.getProjectId2ResourceRoleNames(userId).entrySet().stream() + resourceRoleService.getProjectId2ResourceRoleNames(userId, organizationId).entrySet().stream() .filter(e -> e.getValue().contains(ResourceRoleName.OWNER) || e.getValue().contains(ResourceRoleName.DBA)) .filter(e -> id2Project.containsKey(e.getKey())) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/collaboration/ProjectRepository.java b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/collaboration/ProjectRepository.java index 1d74410ffc..a4df551c82 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/collaboration/ProjectRepository.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/collaboration/ProjectRepository.java @@ -21,6 +21,8 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.JpaSpecificationExecutor; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; public interface ProjectRepository extends JpaRepository, JpaSpecificationExecutor { Optional findByIdAndOrganizationId(Long id, Long organizationId); @@ -34,4 +36,8 @@ public interface ProjectRepository extends JpaRepository, J Optional findByUniqueIdentifier(String uniqueIdentifier); List findByUniqueIdentifierIn(Collection uniqueIdentifiers); + + @Query(value = "select p.id from collaboration_project p where p.organization_id = :organizationId", + nativeQuery = true) + List findIdsByOrganizationId(@Param("organizationId") Long organizationId); } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/connection/DatabaseRepository.java b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/connection/DatabaseRepository.java index 094953a252..3cb2f93412 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/connection/DatabaseRepository.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/connection/DatabaseRepository.java @@ -42,7 +42,7 @@ public interface DatabaseRepository extends JpaRepository, List findByProjectId(Long projectId); - List findByProjectIdIn(List projectIds); + List findByProjectIdIn(Collection projectIds); List findByProjectIdAndExisted(Long projectId, Boolean existed); @@ -74,9 +74,10 @@ public interface DatabaseRepository extends JpaRepository, @Modifying @Transactional @Query(value = "update connect_database t set t.object_sync_status = :#{#status.name()} " - + "where t.object_sync_status = :#{#originalStatus.name()} and t.object_last_sync_time < :syncTime", - nativeQuery = true) - int setObjectSyncStatusByObjectSyncStatusAndObjectLastSyncTimeBefore(@Param("status") DBObjectSyncStatus status, + + "where t.object_sync_status = :#{#originalStatus.name()} " + + "and (t.object_last_sync_time < :syncTime or t.object_last_sync_time is null)", nativeQuery = true) + int setObjectSyncStatusByObjectSyncStatusAndObjectLastSyncTimeIsNullOrBefore( + @Param("status") DBObjectSyncStatus status, @Param("originalStatus") DBObjectSyncStatus originalStatus, @Param("syncTime") Date syncTime); @Modifying diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/dbobject/DBObjectRepository.java b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/dbobject/DBObjectRepository.java index 1f8fcc874f..90be41f7f4 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/dbobject/DBObjectRepository.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/dbobject/DBObjectRepository.java @@ -43,7 +43,7 @@ public interface DBObjectRepository extends OdcJpaRepository findByDatabaseIdAndTypeIn(Long databaseId, Collection types); - List findByDatabaseIdInAndType(Collection databaseIds, DBObjectType type); + List findByDatabaseIdInAndTypeIn(Collection databaseIds, Collection types); /** * list physical tables that not belongs to any logical tables in a physical database diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/flow/FlowInstanceSpecs.java b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/flow/FlowInstanceSpecs.java index ed8d27727b..dbaa041185 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/flow/FlowInstanceSpecs.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/flow/FlowInstanceSpecs.java @@ -98,7 +98,7 @@ public static Specification parentInstanceIdIn(Collection projectIdEquals(Long projectId) { - return SpecificationUtil.columnEqual(FLOW_INSTANCE_PROJECT_ID, projectId); + public static Specification projectIdIn(Collection projectIds) { + return SpecificationUtil.columnIn(FLOW_INSTANCE_PROJECT_ID, projectIds); } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/iam/RoleEntity.java b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/iam/RoleEntity.java index da59e2fa5c..6c3f50862b 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/iam/RoleEntity.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/iam/RoleEntity.java @@ -46,9 +46,11 @@ public class RoleEntity { @GeneratedValue(strategy = GenerationType.IDENTITY) private Long id; + @Column(name = "name", nullable = false) private String name; @Enumerated(value = EnumType.STRING) + @Column(name = "type", nullable = false) private RoleType type; @Column(name = "is_enabled", nullable = false) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/iam/UserDatabasePermissionRepository.java b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/iam/UserDatabasePermissionRepository.java index 7356b3c6ff..7fd21326a1 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/iam/UserDatabasePermissionRepository.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/iam/UserDatabasePermissionRepository.java @@ -17,6 +17,7 @@ import java.util.Collection; import java.util.List; +import java.util.Set; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; @@ -43,4 +44,8 @@ List findNotExpiredByUserIdAndDatabaseIdIn(@Param( List findByUserIdAndProjectId(@Param("userId") Long userId, @Param("projectId") Long projectId); + @Query(value = "select v.* from list_user_database_permission_view v where v.project_id in (:projectIds)", + nativeQuery = true) + List findByProjectIdIn(@Param("projectIds") Set projectIds); + } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/iam/UserRoleRepository.java b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/iam/UserRoleRepository.java index 7b9b5898ce..a5391053b6 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/iam/UserRoleRepository.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/iam/UserRoleRepository.java @@ -25,6 +25,8 @@ import org.springframework.data.repository.query.Param; import org.springframework.transaction.annotation.Transactional; +import com.oceanbase.odc.service.iam.model.UserGlobalResourceRole; + /** * @author gaoda.xy * @date 2022/12/5 19:20 @@ -42,6 +44,22 @@ public interface UserRoleRepository List findByOrganizationIdAndUserIdIn(Long organizationId, Collection userIds); + @Query("SELECT new com.oceanbase.odc.service.iam.model.UserGlobalResourceRole(ur.userId, r.name) " + + + "FROM RoleEntity r " + + "JOIN UserRoleEntity ur ON r.id = ur.roleId " + + "WHERE r.organizationId=:organizationId and r.name IN (:names)") + List findByOrganizationIdAndNameIn(@Param("organizationId") Long organizationId, + @Param("names") Collection names); + + @Query("SELECT new com.oceanbase.odc.service.iam.model.UserGlobalResourceRole(ur.userId, r.name) " + + + "FROM RoleEntity r " + + "JOIN UserRoleEntity ur ON r.id = ur.roleId " + + "WHERE r.organizationId=:organizationId and ur.userId=:userId and r.name IN (:names)") + List findByOrganizationIdAndUserIdAndNameIn(@Param("organizationId") Long organizationId, + @Param("userId") Long userId, @Param("names") Collection names); + @Modifying @Transactional @Query(value = "update iam_user_role set role_id=:roleId where id=:id", nativeQuery = true) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/iam/UserTablePermissionRepository.java b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/iam/UserTablePermissionRepository.java index f71ba5c8b2..da54af9546 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/iam/UserTablePermissionRepository.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/iam/UserTablePermissionRepository.java @@ -17,6 +17,7 @@ import java.util.Collection; import java.util.List; +import java.util.Set; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; @@ -51,4 +52,8 @@ List findNotExpiredByUserIdAndTableIdIn(@Param("userI List findByUserIdAndProjectId(@Param("userId") Long userId, @Param("projectId") Long projectId); + @Query(value = "select v.* from list_user_table_permission_view v where v.project_id in (:projectIds)", + nativeQuery = true) + List findByProjectIdIn(@Param("projectIds") Set projectIds); + } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/iam/resourcerole/UserResourceRoleRepository.java b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/iam/resourcerole/UserResourceRoleRepository.java index cfeb1a7437..d860cb0edc 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/iam/resourcerole/UserResourceRoleRepository.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/iam/resourcerole/UserResourceRoleRepository.java @@ -35,8 +35,6 @@ public interface UserResourceRoleRepository extends OdcJpaRepository findByUserId(Long userId); - List findByResourceId(Long resourceId); - @Modifying @Transactional @Query(value = "delete from iam_user_resource_role t where t.user_id =:userId", nativeQuery = true) @@ -103,10 +101,10 @@ List findByResourceIdAndTypeAndName(@Param("resourceId") @Param("resourceType") ResourceType resourceType, @Param("roleName") String roleName); @Query(value = "select i_urr.* from iam_user_resource_role i_urr inner join iam_resource_role i_rr on i_urr.resource_role_id = i_rr.id " - + "where i_rr.resource_type = :#{#resourceType.name()} and i_urr.user_id = :userId", + + "where i_rr.resource_type = :#{#resourceType.name()} and i_urr.user_id = :userId and i_urr.organization_id = :organizationId", nativeQuery = true) - List findByUserIdAndResourceType(@Param("userId") Long userId, - @Param("resourceType") ResourceType resourceType); + List findByUserIdAndResourceTypeAndOrganizationId(@Param("userId") Long userId, + @Param("resourceType") ResourceType resourceType, @Param("organizationId") Long organizationId); default List batchCreate(List entities) { String sql = InsertSqlTemplateBuilder.from("iam_user_resource_role") diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/resource/ResourceEntity.java b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/resource/ResourceEntity.java index 6666cf6afe..fd5acb3bed 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/resource/ResourceEntity.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/resource/ResourceEntity.java @@ -37,7 +37,7 @@ /** * resource table definition for task - * + * * @author longpeng.zlp * @date 2024/8/14 17:37 */ @@ -48,6 +48,8 @@ public class ResourceEntity { public static final String CREATE_TIME = "createTime"; public static final String STATUS = "status"; + public static final String TYPE = "resourceType"; + /** * id for task */ diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/resource/ResourceRepository.java b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/resource/ResourceRepository.java index dbfff2a52e..28faf34665 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/resource/ResourceRepository.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/resource/ResourceRepository.java @@ -31,7 +31,7 @@ /** * jdbc query for resource_resource table - * + * * @author longpeng.zlp * @date 2024/8/14 17:51 */ diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/schedule/ScheduleRepository.java b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/schedule/ScheduleRepository.java index 428bebfeaf..ee8dde0c7b 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/schedule/ScheduleRepository.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/schedule/ScheduleRepository.java @@ -57,7 +57,7 @@ public interface ScheduleRepository extends OdcJpaRepository findByIdAndJobName(Long id, String scheduleId); - List findByJobNameAndStatusIn(String jobName, List statuses); - @Query(value = "select * from schedule_task where job_name=:jobName and job_group = :jobGroup order by id desc limit 1", nativeQuery = true) Optional getLatestScheduleTaskByJobNameAndJobGroup(@Param("jobName") String jobName, diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/task/JobEntity.java b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/task/JobEntity.java index a743a85552..27cea4bc61 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/metadb/task/JobEntity.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/metadb/task/JobEntity.java @@ -78,6 +78,7 @@ public class JobEntity implements Serializable { @Column(name = "execution_times", nullable = false) private Integer executionTimes; + // can parse to resource id @Column(name = "executor_identifier") private String executorIdentifier; @@ -125,5 +126,4 @@ public class JobEntity implements Serializable { @Generated(GenerationTime.ALWAYS) @Column(name = "update_time", insertable = false, updatable = false) private Date updateTime; - } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/collaboration/project/ProjectService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/collaboration/project/ProjectService.java index 25e09b1f06..eff4ab5f7e 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/collaboration/project/ProjectService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/collaboration/project/ProjectService.java @@ -31,8 +31,10 @@ import javax.validation.constraints.NotNull; import org.apache.commons.collections4.CollectionUtils; +import org.quartz.SchedulerException; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Lazy; import org.springframework.data.domain.Example; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; @@ -70,13 +72,14 @@ import com.oceanbase.odc.metadb.iam.resourcerole.ResourceRoleRepository; import com.oceanbase.odc.metadb.iam.resourcerole.UserResourceRoleEntity; import com.oceanbase.odc.metadb.iam.resourcerole.UserResourceRoleRepository; -import com.oceanbase.odc.metadb.schedule.ScheduleRepository; import com.oceanbase.odc.service.collaboration.project.model.Project; import com.oceanbase.odc.service.collaboration.project.model.Project.ProjectMember; import com.oceanbase.odc.service.collaboration.project.model.QueryProjectParams; import com.oceanbase.odc.service.collaboration.project.model.SetArchivedReq; +import com.oceanbase.odc.service.collaboration.project.model.TicketReference; import com.oceanbase.odc.service.common.model.InnerUser; import com.oceanbase.odc.service.connection.ConnectionService; +import com.oceanbase.odc.service.flow.FlowInstanceService; import com.oceanbase.odc.service.iam.HorizontalDataPermissionValidator; import com.oceanbase.odc.service.iam.ProjectPermissionValidator; import com.oceanbase.odc.service.iam.ResourceRoleService; @@ -85,6 +88,9 @@ import com.oceanbase.odc.service.iam.auth.AuthorizationFacade; import com.oceanbase.odc.service.iam.model.User; import com.oceanbase.odc.service.iam.model.UserResourceRole; +import com.oceanbase.odc.service.schedule.ScheduleService; +import com.oceanbase.odc.service.schedule.model.ScheduleOverviewHist; +import com.oceanbase.odc.service.schedule.model.ScheduleType; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; @@ -145,7 +151,8 @@ public class ProjectService { private ConnectionService connectionService; @Autowired - private ScheduleRepository scheduleRepository; + @Lazy + private ScheduleService scheduleService; @Autowired private HorizontalDataPermissionValidator horizontalDataPermissionValidator; @@ -153,6 +160,10 @@ public class ProjectService { @Autowired private ProjectPermissionValidator projectPermissionValidator; + @Autowired + @Lazy + private FlowInstanceService flowInstanceService; + @Value("${odc.integration.bastion.enabled:false}") private boolean bastionEnabled; @@ -220,13 +231,14 @@ public Project create(@NotNull @Valid Project project) { } @PreAuthenticate(hasAnyResourceRole = {"OWNER", "DBA", "DEVELOPER", "SECURITY_ADMINISTRATOR", "PARTICIPANT"}, + actions = {"OWNER", "DBA", "DEVELOPER", "PARTICIPANT", "SECURITY_ADMINISTRATOR"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) @Transactional(rollbackFor = Exception.class) public Project detail(@NotNull Long id) { ProjectEntity entity = repository.findByIdAndOrganizationId(id, currentOrganizationId()) .orElseThrow(() -> new NotFoundException(ResourceType.ODC_PROJECT, "id", id)); List userResourceRoles = - resourceRoleService.listByResourceTypeAndId(ResourceType.ODC_PROJECT, entity.getId()); + resourceRoleService.listByResourceTypeAndResourceId(ResourceType.ODC_PROJECT, entity.getId()); Project project = entityToModel(entity, userResourceRoles); project.setDbObjectLastSyncTime(getEarliestObjectSyncTime(id)); return project; @@ -237,7 +249,7 @@ public List getProjectMembers(@NotNull Long projectId, @NotNull L ProjectEntity entity = repository.findByIdAndOrganizationId(projectId, organizationId) .orElseThrow(() -> new NotFoundException(ResourceType.ODC_PROJECT, "id", projectId)); List userResourceRoles = - resourceRoleService.listByResourceTypeAndId(ResourceType.ODC_PROJECT, entity.getId()); + resourceRoleService.listByResourceTypeAndResourceId(ResourceType.ODC_PROJECT, entity.getId()); return userResourceRoles.stream().map(this::fromUserResourceRole).filter(Objects::nonNull) .collect(Collectors.toList()); } @@ -253,7 +265,8 @@ public Project getByIdentifier(@NotNull String uniqueIdentifier) { return entity.map(projectMapper::entityToModel).orElse(null); } - @PreAuthenticate(hasAnyResourceRole = {"OWNER"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) + @PreAuthenticate(hasAnyResourceRole = {"OWNER"}, actions = {"OWNER"}, resourceType = "ODC_PROJECT", + indexOfIdParam = 0) @Transactional(rollbackFor = Exception.class) public Project update(@NotNull Long id, @NotNull Project project) { ProjectEntity previous = repository.findByIdAndOrganizationId(id, currentOrganizationId()) @@ -270,10 +283,11 @@ public Project update(@NotNull Long id, @NotNull Project project) { previous.setName(project.getName()); ProjectEntity saved = repository.save(previous); return entityToModel(saved, - resourceRoleService.listByResourceTypeAndId(ResourceType.ODC_PROJECT, saved.getId())); + resourceRoleService.listByResourceTypeAndResourceId(ResourceType.ODC_PROJECT, saved.getId())); } - @PreAuthenticate(hasAnyResourceRole = {"OWNER"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) + @PreAuthenticate(hasAnyResourceRole = {"OWNER"}, actions = {"OWNER"}, resourceType = "ODC_PROJECT", + indexOfIdParam = 0) @Transactional(rollbackFor = Exception.class) public Project setArchived(Long id, @NotNull SetArchivedReq req) throws InterruptedException { ProjectEntity saved = setArchived(id, currentOrganizationId(), req); @@ -289,10 +303,9 @@ public ProjectEntity setArchived(Long id, @NonNull Long organizationId, @NotNull if (!req.getArchived()) { throw new BadRequestException("currently not allowed to recover projects"); } - if (scheduleRepository.getEnabledScheduleCountByProjectId(id) > 0) { - throw new BadRequestException("Please disable all active tickets in the project first."); - } + checkUnfinishedTickets(id); previous.setArchived(true); + previous.setName(previous.getName() + "_archived_" + System.currentTimeMillis()); ProjectEntity saved = repository.save(previous); List connectionEntities = connectionConfigRepository.findByProjectId(id).stream() .peek(e -> e.setProjectId(null)).collect(Collectors.toList()); @@ -318,11 +331,16 @@ public Page list(@Valid QueryProjectParams params, @NotNull Pageable pa Page projectEntities = innerList(params, pageable, UserResourceRole::isProjectMember); return projectEntities.map(project -> { List members = - resourceRoleService.listByResourceTypeAndId(ResourceType.ODC_PROJECT, project.getId()); + resourceRoleService.listByResourceTypeAndResourceId(ResourceType.ODC_PROJECT, project.getId()); return entityToModel(project, members); }); } + @SkipAuthorize("odc internal usage") + public List listByIds(@NotEmpty Set ids) { + return repository.findAllById(ids).stream().map(this::entityToModel).collect(Collectors.toList()); + } + private Page innerList(@Valid QueryProjectParams params, @NotNull Pageable pageable, @NotNull Predicate predicate) { List userResourceRoles = @@ -341,7 +359,8 @@ private Page innerList(@Valid QueryProjectParams params, @NotNull return repository.findAll(specs, pageable); } - @PreAuthenticate(hasAnyResourceRole = {"OWNER"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) + @PreAuthenticate(hasAnyResourceRole = {"OWNER"}, actions = {"OWNER"}, resourceType = "ODC_PROJECT", + indexOfIdParam = 0) @Transactional(rollbackFor = Exception.class) public Project createProjectMembers(@NonNull Long id, @NotEmpty List members) { ProjectEntity project = repository.findByIdAndOrganizationId(id, currentOrganizationId()) @@ -374,7 +393,8 @@ public Project createMembersSkipPermissionCheck(@NonNull Long projectId, @NonNul } - @PreAuthenticate(hasAnyResourceRole = {"OWNER"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) + @PreAuthenticate(hasAnyResourceRole = {"OWNER"}, actions = {"OWNER"}, resourceType = "ODC_PROJECT", + indexOfIdParam = 0) @Transactional(rollbackFor = Exception.class) public boolean deleteProjectMember(@NonNull Long projectId, @NonNull Long userId) { if (currentUserId().longValue() == userId.longValue()) { @@ -385,11 +405,51 @@ public boolean deleteProjectMember(@NonNull Long projectId, @NonNull Long userId return deleted; } + @Transactional(rollbackFor = Exception.class) + @SkipAuthorize("Internal usage") + public Boolean batchDelete(Set projectIds) { + projectPermissionValidator.checkProjectRole(projectIds, Collections.singletonList(ResourceRoleName.OWNER)); + repository.findByIdIn(projectIds).stream().forEach(project -> { + if (project.getBuiltin() || !project.getArchived()) { + throw new UnsupportedException(ErrorCodes.IllegalOperation, + new Object[] {"builtin or not-archived project, projectName=" + project.getName()}, + "Operation on builtin or not-archived project is not allowed"); + } + }); + repository.deleteAllById(projectIds); + // TODO: bind these resource delete logic in one place for better maintainability + resourceRoleService.deleteByResourceTypeAndIdIn(ResourceType.ODC_PROJECT, projectIds); + deleteMemberRelatedDatabasePermissions(projectIds); + deleteMemberRelatedTablePermissions(projectIds); + List relatedDatabaseIds = databaseRepository.findByProjectIdIn(projectIds).stream() + .map(DatabaseEntity::getId).collect(Collectors.toList()); + if (CollectionUtils.isNotEmpty(relatedDatabaseIds)) { + resourceRoleService.deleteByResourceTypeAndIdIn(ResourceType.ODC_DATABASE, relatedDatabaseIds); + } + return true; + } + + @Transactional(rollbackFor = Exception.class) + @SkipAuthorize("Internal usage") + @PreAuthenticate(hasAnyResourceRole = {"OWNER"}, actions = {"OWNER"}, resourceType = "ODC_PROJECT", + indexOfIdParam = 0) + public TicketReference getProjectTicketReference(@NonNull Long projectId) { + TicketReference reference = new TicketReference(); + reference.setUnfinishedFlowInstances( + flowInstanceService.listUnfinishedFlowInstances(Pageable.unpaged(), projectId).getContent()); + reference.setUnfinishedSchedules( + scheduleService.listUnfinishedSchedulesByProjectId(Pageable.unpaged(), projectId).getContent().stream() + .filter(schedule -> schedule.getType() != ScheduleType.PARTITION_PLAN).collect( + Collectors.toList())); + return reference; + } + @SkipAuthorize @Transactional(rollbackFor = Exception.class) public boolean deleteProjectMemberSkipPermissionCheck(@NonNull Long projectId, @NonNull Long userId) { - Set memberIds = resourceRoleService.listByResourceTypeAndId(ResourceType.ODC_PROJECT, projectId).stream() - .filter(Objects::nonNull).map(UserResourceRole::getUserId).collect(Collectors.toSet()); + Set memberIds = + resourceRoleService.listByResourceTypeAndResourceId(ResourceType.ODC_PROJECT, projectId).stream() + .filter(Objects::nonNull).map(UserResourceRole::getUserId).collect(Collectors.toSet()); if (!memberIds.contains(userId)) { throw new BadRequestException("User not belongs to this project"); } @@ -419,7 +479,8 @@ public void deleteUserRelatedProjectResources(@NonNull Long userId, @NonNull Str }); } - @PreAuthenticate(hasAnyResourceRole = {"OWNER"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) + @PreAuthenticate(hasAnyResourceRole = {"OWNER"}, actions = {"OWNER"}, resourceType = "ODC_PROJECT", + indexOfIdParam = 0) @Transactional(rollbackFor = Exception.class) public boolean updateProjectMember(@NonNull Long projectId, @NonNull Long userId, @NonNull List members) { @@ -436,7 +497,7 @@ public boolean updateProjectMemberSkipPermissionCheck(@NonNull Long projectId, @ ProjectEntity project = repository.findByIdAndOrganizationId(projectId, organizationId) .orElseThrow(() -> new NotFoundException(ResourceType.ODC_PROJECT, "id", projectId)); Map> userId2ResourceRoles = - resourceRoleService.listByResourceTypeAndId(ResourceType.ODC_PROJECT, project.getId()).stream() + resourceRoleService.listByResourceTypeAndResourceId(ResourceType.ODC_PROJECT, project.getId()).stream() .collect(Collectors.groupingBy(UserResourceRole::getUserId)); if (CollectionUtils.isEmpty(userId2ResourceRoles.keySet())) { return false; @@ -501,6 +562,35 @@ public void checkCurrentUserProjectRolePermissions(@NotNull Project project, projectPermissionValidator.checkProjectRole(project.getId(), roleNames); } + @SkipAuthorize("inside method permission check") + public void checkUnfinishedTickets(@NonNull Long projectId) { + if (flowInstanceService.listUnfinishedFlowInstances(Pageable.unpaged(), projectId).hasContent()) { + throw new BadRequestException( + "There exists unfinished tickets in the project, please stop them before archiving the project."); + } + List schedules = + scheduleService.listUnfinishedSchedulesByProjectId(Pageable.unpaged(), projectId).getContent(); + // no unfinished schedules + if (CollectionUtils.isEmpty(schedules)) { + return; + } + // There exists unfinished schedules(except for partition plans) in the project + if (schedules.stream().anyMatch(schedule -> schedule.getType() != ScheduleType.PARTITION_PLAN)) { + throw new BadRequestException( + "There exists unfinished schedule tasks in the project, please stop them before archiving the project."); + } + // Terminate all partition plans if exists + schedules.stream().filter(schedule -> schedule.getType() == ScheduleType.PARTITION_PLAN) + .forEach(schedule -> { + try { + scheduleService.innerTerminate(schedule.getId()); + } catch (SchedulerException e) { + log.warn("Failed to terminate partition plan schedule, scheduleId={}", schedule.getId()); + throw new RuntimeException(e); + } + }); + } + private Project entityToModel(ProjectEntity entity, List userResourceRoles) { Project project = entityToModelWithoutCurrentUser(entity, userResourceRoles); project.setCreator(currentInnerUser()); @@ -521,6 +611,8 @@ private Project entityToModel(ProjectEntity entity) { Project project = projectMapper.entityToModel(entity); project.setCreator(currentInnerUser()); project.setLastModifier(currentInnerUser()); + project.setCurrentUserResourceRoles( + getProjectId2ResourceRoleNames().getOrDefault(project.getId(), Collections.EMPTY_SET)); return project; } @@ -536,6 +628,7 @@ private ProjectMember fromUserResourceRole(UserResourceRole userResourceRole) { member.setName(user.getName()); member.setAccountName(user.getAccountName()); member.setUserEnabled(user.isEnabled()); + member.setDerivedFromGlobalProjectRole(userResourceRole.isDerivedFromGlobalProjectRole()); return member; } @@ -588,6 +681,24 @@ private void deleteMemberRelatedTablePermissions(@NonNull Long userId, @NonNull } } + private void deleteMemberRelatedDatabasePermissions(@NonNull Set projectIds) { + List permissionIds = userDatabasePermissionRepository.findByProjectIdIn(projectIds).stream() + .map(UserDatabasePermissionEntity::getId).collect(Collectors.toList()); + if (CollectionUtils.isNotEmpty(permissionIds)) { + permissionRepository.deleteByIds(permissionIds); + userPermissionRepository.deleteByPermissionIds(permissionIds); + } + } + + private void deleteMemberRelatedTablePermissions(@NonNull Set projectIds) { + List permissionIds = userTablePermissionRepository.findByProjectIdIn(projectIds).stream() + .map(UserTablePermissionEntity::getId).collect(Collectors.toList()); + if (CollectionUtils.isNotEmpty(permissionIds)) { + permissionRepository.deleteByIds(permissionIds); + userPermissionRepository.deleteByPermissionIds(permissionIds); + } + } + /** * 1. check if duplicate project name exists
* 2. check if all project members belong to current organization
diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/collaboration/project/model/Project.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/collaboration/project/model/Project.java index 4551a9fc4c..98a2b9ede0 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/collaboration/project/model/Project.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/collaboration/project/model/Project.java @@ -129,5 +129,8 @@ public static class ProjectMember { private String name; private ResourceRoleName role; + + @JsonProperty(access = Access.READ_ONLY) + private boolean derivedFromGlobalProjectRole; } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/collaboration/project/model/TicketReference.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/collaboration/project/model/TicketReference.java new file mode 100644 index 0000000000..f2a48b5c5b --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/collaboration/project/model/TicketReference.java @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.collaboration.project.model; + +import java.util.List; + +import com.oceanbase.odc.service.flow.model.FlowInstanceDetailResp; +import com.oceanbase.odc.service.schedule.model.ScheduleOverviewHist; + +import lombok.Data; + +/** + * @Author: Lebie + * @Date: 2024/11/28 17:24 + * @Description: [] + */ +@Data +public class TicketReference { + private List unfinishedFlowInstances; + + private List unfinishedSchedules; +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/common/util/UrlUtils.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/common/util/UrlUtils.java index 6ffcc99e8b..3178971558 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/common/util/UrlUtils.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/common/util/UrlUtils.java @@ -15,7 +15,9 @@ */ package com.oceanbase.odc.service.common.util; +import java.net.URL; import java.net.URLDecoder; +import java.net.URLEncoder; import java.util.ArrayList; import java.util.List; import java.util.Objects; @@ -60,6 +62,30 @@ public static String decode(String encode) { } } + public static String encode(String str) { + try { + return URLEncoder.encode(str, "UTF-8"); + } catch (Exception e) { + return null; + } + } + + @SneakyThrows + public static String getUrlHost(String targetUrl) { + URL url = new URL(targetUrl); + StringBuilder host = new StringBuilder(); + host.append(url.getProtocol()).append("://").append(url.getHost()); + int port = url.getPort(); + if (port <= 0) { + return host.toString(); + } + if ("http".equals(url.getProtocol()) && port != 80 || + "https".equals(url.getProtocol()) && port != 443) { + host.append(":").append(url.getPort()); + } + return host.toString(); + } + @Nullable public static String getQueryParameterFirst(String url, String parameterName) { List strings = getQueryParameter(url, parameterName); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionEventPublisher.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionEventPublisher.java new file mode 100644 index 0000000000..4920c18554 --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionEventPublisher.java @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.connection; + +import javax.annotation.PostConstruct; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import com.oceanbase.odc.common.event.AbstractEvent; +import com.oceanbase.odc.common.event.LocalEventPublisher; +import com.oceanbase.odc.service.connection.listener.UpdateDatasourceListener; + +import lombok.NonNull; + +/** + * @Author：tinker + * @Date: 2024/12/30 10:57 + * @Descripition: + */ + +@Component +public class ConnectionEventPublisher { + @Autowired + private LocalEventPublisher localEventPublisher; + + @Autowired + private UpdateDatasourceListener updateDatasourceListener; + + @PostConstruct + public void init() { + localEventPublisher.addEventListener(updateDatasourceListener); + } + + public void publishEvent(@NonNull T event) { + localEventPublisher.publishEvent(event); + } + +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionService.java index 9278d7af65..6f3137542b 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionService.java @@ -105,6 +105,7 @@ import com.oceanbase.odc.service.connection.ConnectionStatusManager.CheckState; import com.oceanbase.odc.service.connection.database.DatabaseService; import com.oceanbase.odc.service.connection.database.DatabaseSyncManager; +import com.oceanbase.odc.service.connection.event.UpsertDatasourceEvent; import com.oceanbase.odc.service.connection.model.ConnectProperties; import com.oceanbase.odc.service.connection.model.ConnectionConfig; import com.oceanbase.odc.service.connection.model.OBTenantEndpoint; @@ -221,6 +222,9 @@ public class ConnectionService { @Autowired private TransactionTemplate txTemplate; + @Autowired + private ConnectionEventPublisher connectionEventPublisher; + private final ConnectionMapper mapper = ConnectionMapper.INSTANCE; public static final String DEFAULT_MIN_PRIVILEGE = "read"; @@ -249,6 +253,7 @@ public ConnectionConfig create(@NotNull @Valid ConnectionConfig connection, @Not } }); databaseSyncManager.submitSyncDataSourceAndDBSchemaTask(saved); + connectionEventPublisher.publishEvent(new UpsertDatasourceEvent(saved)); return saved; } @@ -393,8 +398,11 @@ public ConnectionConfig getWithoutPermissionCheck(@NotNull Long id) { @SkipAuthorize("odc internal usage") public List listByOrganizationId(@NonNull Long organizationId) { - return entitiesToModels(repository.findByOrganizationIdOrderByNameAsc(organizationId), organizationId, true, + List connectionConfigs = entitiesToModels( + repository.findByOrganizationIdOrderByNameAsc(organizationId), organizationId, true, true); + fullFillAttributes(connectionConfigs); + return connectionConfigs; } @SkipAuthorize("odc internal usage") @@ -417,6 +425,7 @@ public List listSyncableDataSourcesByOrganizationIdIn(@NonNull @Transactional(rollbackFor = Exception.class) @PreAuthenticate(hasAnyResourceRole = {"OWNER, DBA, DEVELOPER, SECURITY_ADMINISTRATOR"}, + actions = {"OWNER", "DBA", "SECURITY_ADMINISTRATOR"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) public PaginatedData listByProjectId(@NotNull Long projectId, @NotNull Boolean basic) { List connections; @@ -604,6 +613,7 @@ public ConnectionConfig update(@NotNull Long id, @NotNull @Valid ConnectionConfi return updateConnectionConfig(id, connection, true); } + @SkipAuthorize("odc internal usage") public ConnectionConfig updateWithoutPermissionCheck(@NotNull Long id, @NotNull @Valid ConnectionConfig connection) { return updateConnectionConfig(id, connection, false); @@ -683,6 +693,7 @@ private ConnectionConfig updateConnectionConfig(Long id, ConnectionConfig connec } }); databaseSyncManager.submitSyncDataSourceAndDBSchemaTask(config); + connectionEventPublisher.publishEvent(new UpsertDatasourceEvent(config)); return config; } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionTesting.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionTesting.java index 666772cf20..ea7a4b31d5 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionTesting.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionTesting.java @@ -74,6 +74,8 @@ public class ConnectionTesting { private ConnectionSSLAdaptor connectionSSLAdaptor; @Autowired private CloudMetadataClient cloudMetadataClient; + @Autowired + private FileSystemConnectionTester fileSystemConnectionTesting; @Value("${odc.sdk.test-connect.query-timeout-seconds:2}") private int queryTimeoutSeconds = 2; @@ -102,6 +104,9 @@ public ConnectionTestResult test(@NotNull @Valid TestConnectionReq req) { public ConnectionTestResult test(@NonNull ConnectionConfig config) { ConnectType type = config.getType(); + if (type.getDialectType() == DialectType.FILE_SYSTEM) { + return fileSystemConnectionTesting.test(config); + } try { /** * 进行连接测试时需要关注的值有一个 {@link ConnectType}， 容易产生问题信息主要是两个：{@code username}, {@code defaultSchema} 首先分析 @@ -232,6 +237,7 @@ private ConnectionConfig reqToConnectionConfig(TestConnectionReq req) { config.setServiceName(req.getServiceName()); config.setUserRole(req.getUserRole()); config.setCatalogName(req.getCatalogName()); + config.setRegion(req.getRegion()); OBTenantEndpoint endpoint = req.getEndpoint(); if (Objects.nonNull(endpoint) && OceanBaseAccessMode.IC_PROXY == endpoint.getAccessMode()) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionValidator.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionValidator.java index b2fd42c5a5..fcab8f4504 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionValidator.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/ConnectionValidator.java @@ -21,6 +21,7 @@ import org.springframework.stereotype.Component; import com.oceanbase.odc.core.shared.PreConditions; +import com.oceanbase.odc.core.shared.constant.DialectType; import com.oceanbase.odc.core.shared.constant.ErrorCodes; import com.oceanbase.odc.core.shared.exception.AccessDeniedException; import com.oceanbase.odc.service.collaboration.environment.EnvironmentService; @@ -43,7 +44,9 @@ public class ConnectionValidator { void validateForUpsert(ConnectionConfig connection) { PreConditions.notNull(connection, "connection"); PreConditions.notBlank(connection.getHost(), "connection.host"); - PreConditions.notNull(connection.getPort(), "connection.port"); + if (connection.getDialectType() != DialectType.FILE_SYSTEM) { + PreConditions.notNull(connection.getPort(), "connection.port"); + } PreConditions.validNotSqlInjection(connection.getUsername(), "username"); PreConditions.validNotSqlInjection(connection.getClusterName(), "clusterName"); PreConditions.validNotSqlInjection(connection.getTenantName(), "tenantName"); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/FileSystemConnectionTester.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/FileSystemConnectionTester.java new file mode 100644 index 0000000000..43d5c37be3 --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/FileSystemConnectionTester.java @@ -0,0 +1,148 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.connection; + +import java.io.ByteArrayInputStream; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.text.MessageFormat; +import java.util.Collections; +import java.util.UUID; + +import org.springframework.stereotype.Component; + +import com.aliyun.oss.OSSErrorCode; +import com.aliyun.oss.OSSException; +import com.oceanbase.odc.core.shared.PreConditions; +import com.oceanbase.odc.core.shared.constant.ConnectType; +import com.oceanbase.odc.plugin.connect.api.TestResult; +import com.oceanbase.odc.service.cloud.model.CloudProvider; +import com.oceanbase.odc.service.connection.model.ConnectionConfig; +import com.oceanbase.odc.service.connection.model.ConnectionTestResult; +import com.oceanbase.odc.service.objectstorage.cloud.CloudResourceConfigurations; +import com.oceanbase.odc.service.objectstorage.cloud.client.CloudClient; +import com.oceanbase.odc.service.objectstorage.cloud.client.CloudException; +import com.oceanbase.odc.service.objectstorage.cloud.model.DeleteObjectsRequest; +import com.oceanbase.odc.service.objectstorage.cloud.model.ObjectMetadata; +import com.oceanbase.odc.service.objectstorage.cloud.model.ObjectStorageConfiguration; +import com.oceanbase.tools.migrator.common.exception.UnExpectedException; + +import lombok.NonNull; + +/** + * @Author：tinker + * @Date: 2024/11/19 11:24 + * @Descripition: + */ + +@Component +public class FileSystemConnectionTester { + + private static final String COS_ENDPOINT_PATTERN = "cos.{0}.myqcloud.com"; + private static final String OBS_ENDPOINT_PATTERN = "obs.{0}.myhuaweicloud.com"; + private static final String OSS_ENDPOINT_PATTERN = "oss-{0}.aliyuncs.com"; + private static final String S3_ENDPOINT_GLOBAL_PATTERN = "s3.{0}.amazonaws.com"; + private static final String S3_ENDPOINT_CN_PATTERN = "s3.{0}.amazonaws.com.cn"; + + private static final String TMP_FILE_NAME_PREFIX = "odc-test-object-"; + private static final String TMP_TEST_DATA = "This is a test object to check read and write permissions."; + + public ConnectionTestResult test(@NonNull ConnectionConfig config) { + PreConditions.notBlank(config.getPassword(), "AccessKeySecret"); + PreConditions.notBlank(config.getRegion(), "Region"); + URI uri = URI.create(config.getHost()); + ObjectStorageConfiguration storageConfig = new ObjectStorageConfiguration(); + storageConfig.setAccessKeyId(config.getUsername()); + storageConfig.setAccessKeySecret(config.getPassword()); + storageConfig.setBucketName(uri.getAuthority()); + storageConfig.setRegion(config.getRegion()); + storageConfig.setCloudProvider(getCloudProvider(config.getType())); + storageConfig.setPublicEndpoint(getEndPointByRegion(config.getType(), config.getRegion())); + try { + CloudClient cloudClient = + new CloudResourceConfigurations.CloudClientBuilder().generateCloudClient(storageConfig); + String objectKey = uri.getPath().endsWith("/") ? uri.getAuthority() + uri.getPath() + generateTempFileName() + : uri.getAuthority() + uri.getPath() + "/" + generateTempFileName(); + cloudClient.putObject(storageConfig.getBucketName(), objectKey, + new ByteArrayInputStream(TMP_TEST_DATA.getBytes(StandardCharsets.UTF_8)), new ObjectMetadata()); + DeleteObjectsRequest deleteObjectsRequest = new DeleteObjectsRequest(); + deleteObjectsRequest.setBucketName(storageConfig.getBucketName()); + deleteObjectsRequest.setKeys(Collections.singletonList(objectKey)); + cloudClient.deleteObjects(deleteObjectsRequest); + return ConnectionTestResult.success(config.getType()); + } catch (CloudException e) { + if (e.getCause() != null && e.getCause() instanceof OSSException) { + OSSException cause = (OSSException) e.getCause(); + switch (cause.getErrorCode()) { + case OSSErrorCode.ACCESS_DENIED: + return new ConnectionTestResult(TestResult.akAccessDenied(storageConfig.getAccessKeyId()), + config.getType()); + case OSSErrorCode.INVALID_ACCESS_KEY_ID: + return new ConnectionTestResult(TestResult.invalidAccessKeyId(storageConfig.getAccessKeyId()), + config.getType()); + case OSSErrorCode.SIGNATURE_DOES_NOT_MATCH: + return new ConnectionTestResult( + TestResult.signatureDoesNotMatch(storageConfig.getAccessKeyId()), config.getType()); + case OSSErrorCode.NO_SUCH_BUCKET: + return new ConnectionTestResult(TestResult.bucketNotExist(storageConfig.getBucketName()), + config.getType()); + default: + return new ConnectionTestResult(TestResult.unknownError(e), config.getType()); + } + } + // TODO:process s3 error message + return new ConnectionTestResult(TestResult.unknownError(e), config.getType()); + } + } + + private CloudProvider getCloudProvider(ConnectType type) { + switch (type) { + case COS: + return CloudProvider.TENCENT_CLOUD; + case OBS: + return CloudProvider.HUAWEI_CLOUD; + case S3A: + return CloudProvider.AWS; + case OSS: + return CloudProvider.ALIBABA_CLOUD; + default: + throw new UnExpectedException(); + } + } + + private static String getEndPointByRegion(ConnectType type, String region) { + switch (type) { + case COS: + return MessageFormat.format(COS_ENDPOINT_PATTERN, region); + case OSS: + return MessageFormat.format(OSS_ENDPOINT_PATTERN, region); + case OBS: + return MessageFormat.format(OBS_ENDPOINT_PATTERN, region); + case S3A: + // Note there is a difference of Top-Level Domain between cn and global regions. + if (region.startsWith("cn-")) { + return MessageFormat.format(S3_ENDPOINT_CN_PATTERN, region); + } + return MessageFormat.format(S3_ENDPOINT_GLOBAL_PATTERN, region); + default: + throw new IllegalArgumentException("regionToEndpoint is not applicable for storageType " + type); + } + } + + private String generateTempFileName() { + return TMP_FILE_NAME_PREFIX + UUID.randomUUID(); + } +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseService.java index 30f2f38269..9c27b53853 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/database/DatabaseService.java @@ -46,6 +46,7 @@ import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang.StringUtils; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.data.domain.Page; import org.springframework.data.domain.PageImpl; import org.springframework.data.domain.Pageable; @@ -64,6 +65,7 @@ import com.oceanbase.odc.core.authority.util.SkipAuthorize; import com.oceanbase.odc.core.session.ConnectionSession; import com.oceanbase.odc.core.shared.PreConditions; +import com.oceanbase.odc.core.shared.constant.DialectType; import com.oceanbase.odc.core.shared.constant.ErrorCodes; import com.oceanbase.odc.core.shared.constant.OrganizationType; import com.oceanbase.odc.core.shared.constant.ResourceRoleName; @@ -130,7 +132,7 @@ import com.oceanbase.odc.service.session.factory.DefaultConnectSessionFactory; import com.oceanbase.odc.service.session.factory.OBConsoleDataSourceFactory; import com.oceanbase.odc.service.session.model.SqlExecuteResult; -import com.oceanbase.odc.service.task.runtime.PreCheckTaskParameters.AuthorizedDatabase; +import com.oceanbase.odc.service.task.base.precheck.PreCheckTaskParameters.AuthorizedDatabase; import com.oceanbase.tools.dbbrowser.model.DBDatabase; import lombok.NonNull; @@ -224,6 +226,9 @@ public class DatabaseService { @Autowired private ConnectionSyncHistoryService connectionSyncHistoryService; + @Value("${odc.integration.bastion.enabled:false}") + private boolean bastionEnabled; + @Transactional(rollbackFor = Exception.class) @SkipAuthorize("internal authenticated") public Database detail(@NonNull Long id) { @@ -247,6 +252,12 @@ public Database detailSkipPermissionCheck(@NonNull Long id) { .orElseThrow(() -> new NotFoundException(ResourceType.ODC_DATABASE, "id", id)), true); } + @SkipAuthorize("internal usage") + public Database detailSkipPermissionCheckForRead(@NonNull Long id) { + return entityToModelForRead(databaseRepository.findById(id) + .orElseThrow(() -> new NotFoundException(ResourceType.ODC_DATABASE, "id", id)), true); + } + @SkipAuthorize("odc internal usage") public Database getBasicSkipPermissionCheck(Long id) { return databaseMapper.entityToModel(databaseRepository.findById(id) @@ -364,7 +375,7 @@ public Database create(@NonNull CreateDatabaseReq req) { || !connectionService.checkPermission(req.getDataSourceId(), Collections.singletonList("update"))) { throw new AccessDeniedException(); } - DataSource dataSource = new OBConsoleDataSourceFactory(connection, true, false).getDataSource(); + DataSource dataSource = new OBConsoleDataSourceFactory(connection, true, false, false).getDataSource(); try (Connection conn = dataSource.getConnection()) { createDatabase(req, conn, connection); DBDatabase dbDatabase = dbSchemaService.detail(connection.getDialectType(), conn, req.getName()); @@ -518,6 +529,9 @@ public Boolean internalSyncDataSourceSchemas(@NonNull Long dataSourceId) throws Optional organizationOpt = Optional.empty(); try { connection = connectionService.getForConnectionSkipPermissionCheck(dataSourceId); + if (connection.getDialectType() == DialectType.FILE_SYSTEM) { + return true; + } horizontalDataPermissionValidator.checkCurrentOrganization(connection); organizationOpt = organizationService.get(connection.getOrganizationId()); Organization organization = @@ -538,7 +552,7 @@ public Boolean internalSyncDataSourceSchemas(@NonNull Long dataSourceId) throws } } - @SkipAuthorize("internal usage") + @SkipAuthorize("odc internal usage") public int updateEnvironmentIdByConnectionId(@NotNull Long environmentId, @NotNull Long connectionId) { return databaseRepository.setEnvironmentIdByConnectionId(environmentId, connectionId); } @@ -546,7 +560,7 @@ public int updateEnvironmentIdByConnectionId(@NotNull Long environmentId, @NotNu private void syncTeamDataSources(ConnectionConfig connection) throws ExecutionException, InterruptedException, TimeoutException { Long currentProjectId = connection.getProjectId(); - boolean blockExcludeSchemas = dbSchemaSyncProperties.isBlockExclusionsWhenSyncDbToProject(); + boolean blockExcludeSchemas = dbSchemaSyncProperties.isBlockExclusionsWhenSyncDbToProject() && !bastionEnabled; List excludeSchemas = dbSchemaSyncProperties.getExcludeSchemas(connection.getDialectType()); DataSource teamDataSource = getDataSourceFactory(connection).getDataSource(); ExecutorService executorService = Executors.newFixedThreadPool(1); @@ -659,7 +673,8 @@ private void syncTeamDataSources(ConnectionConfig connection) } private OBConsoleDataSourceFactory getDataSourceFactory(ConnectionConfig connection) { - OBConsoleDataSourceFactory obConsoleDataSourceFactory = new OBConsoleDataSourceFactory(connection, true, false); + OBConsoleDataSourceFactory obConsoleDataSourceFactory = + new OBConsoleDataSourceFactory(connection, true, false, false); LocalEventPublisher localEventPublisher = new LocalEventPublisher(); localEventPublisher.addEventListener(new GetConnectionFailedEventListener()); obConsoleDataSourceFactory.setEventPublisher(localEventPublisher); @@ -671,7 +686,7 @@ private Long getProjectId(DatabaseEntity database, Long currentProjectId, List listUserForOsc(Long dataSourceId) { } @Transactional(rollbackFor = Exception.class) - @PreAuthenticate(hasAnyResourceRole = {"OWNER", "DBA"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) + @PreAuthenticate(hasAnyResourceRole = {"OWNER", "DBA"}, actions = {"OWNER", "DBA"}, resourceType = "ODC_PROJECT", + indexOfIdParam = 0) public boolean modifyDatabasesOwners(@NotNull Long projectId, @NotNull @Valid ModifyDatabaseOwnerReq req) { databaseRepository.findByIdIn(req.getDatabaseIds()).forEach(database -> { if (!projectId.equals(database.getProjectId())) { throw new AccessDeniedException(); } }); - Set memberIds = resourceRoleService.listByResourceTypeAndId(ResourceType.ODC_PROJECT, projectId).stream() - .map(UserResourceRole::getUserId).collect(Collectors.toSet()); + Set memberIds = + resourceRoleService.listByResourceTypeAndResourceId(ResourceType.ODC_PROJECT, projectId).stream() + .map(UserResourceRole::getUserId).collect(Collectors.toSet()); if (!memberIds.containsAll(req.getOwnerIds())) { throw new AccessDeniedException(); } @@ -851,8 +868,9 @@ public void updateObjectLastSyncTimeAndStatus(@NotNull Long databaseId, @Transactional(rollbackFor = Exception.class) public void refreshExpiredPendingDBObjectStatus() { Date syncDate = new Date(System.currentTimeMillis() - this.globalSearchProperties.getMaxPendingMillis()); - int affectRows = this.databaseRepository.setObjectSyncStatusByObjectSyncStatusAndObjectLastSyncTimeBefore( - DBObjectSyncStatus.INITIALIZED, DBObjectSyncStatus.PENDING, syncDate); + int affectRows = + this.databaseRepository.setObjectSyncStatusByObjectSyncStatusAndObjectLastSyncTimeIsNullOrBefore( + DBObjectSyncStatus.INITIALIZED, DBObjectSyncStatus.PENDING, syncDate); log.info("Refresh outdated pending objects status, syncDate={}, affectRows={}", syncDate, affectRows); } @@ -883,7 +901,8 @@ private void checkTransferable(@NonNull Collection databases, @N } if (CollectionUtils.isNotEmpty(req.getOwnerIds())) { Set memberIds = - resourceRoleService.listByResourceTypeAndId(ResourceType.ODC_PROJECT, req.getProjectId()).stream() + resourceRoleService.listByResourceTypeAndResourceId(ResourceType.ODC_PROJECT, req.getProjectId()) + .stream() .map(UserResourceRole::getUserId).collect(Collectors.toSet()); PreConditions.validArgumentState(memberIds.containsAll(req.getOwnerIds()), ErrorCodes.AccessDenied, null, "Invalid ownerIds"); @@ -897,7 +916,7 @@ private void checkTransferable(@NonNull Collection databases, @N ErrorCodes.AccessDenied, null, "Lack of update permission on current datasource"); Map id2Conn = connectionService.innerListByIds(connectionIds).stream() .collect(Collectors.toMap(ConnectionConfig::getId, c -> c, (c1, c2) -> c2)); - if (dbSchemaSyncProperties.isBlockExclusionsWhenSyncDbToProject()) { + if (dbSchemaSyncProperties.isBlockExclusionsWhenSyncDbToProject() && !bastionEnabled) { connectionIds = databases.stream().filter(database -> { ConnectionConfig connection = id2Conn.get(database.getConnectionId()); return connection != null && !dbSchemaSyncProperties.getExcludeSchemas(connection.getDialectType()) @@ -934,7 +953,7 @@ private Page entitiesToModels(Page entities, boolean i Map> databaseId2UserResourceRole = new HashMap<>(); Map userId2User = new HashMap<>(); List userResourceRoles = - resourceRoleService.listByResourceTypeAndIdIn(ResourceType.ODC_DATABASE, databaseIds); + resourceRoleService.listByResourceTypeAndResourceIdIn(ResourceType.ODC_DATABASE, databaseIds); if (CollectionUtils.isNotEmpty(userResourceRoles)) { databaseId2UserResourceRole = userResourceRoles.stream() .collect(Collectors.groupingBy(UserResourceRole::getResourceId, Collectors.toList())); @@ -976,6 +995,24 @@ private Page entitiesToModels(Page entities, boolean i }); } + private Database entityToModelForRead(DatabaseEntity entity, boolean includesPermittedAction) { + Database model = databaseMapper.entityToModel(entity); + if (Objects.nonNull(entity.getProjectId())) { + model.setProject(projectService.detail(entity.getProjectId())); + } + // for logical database, the connection id may be null + if (entity.getConnectionId() != null) { + model.setDataSource(connectionService.getBasicWithoutPermissionCheck(entity.getConnectionId())); + } + model.setEnvironment(environmentService.detailSkipPermissionCheck(entity.getEnvironmentId())); + if (includesPermittedAction) { + model.setAuthorizedPermissionTypes( + permissionHelper.getDBPermissions(Collections.singleton(entity.getId())).get(entity.getId())); + } + return model; + } + + private Database entityToModel(DatabaseEntity entity, boolean includesPermittedAction) { Database model = databaseMapper.entityToModel(entity); if (Objects.nonNull(entity.getProjectId())) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/task/TaskResult.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/event/UpsertDatasourceEvent.java similarity index 52% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/task/TaskResult.java rename to server/odc-service/src/main/java/com/oceanbase/odc/service/connection/event/UpsertDatasourceEvent.java index 9de2271f36..cb4f9bcd28 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/task/TaskResult.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/event/UpsertDatasourceEvent.java @@ -13,32 +13,26 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +package com.oceanbase.odc.service.connection.event; -package com.oceanbase.odc.service.task.executor.task; +import com.oceanbase.odc.common.event.AbstractEvent; +import com.oceanbase.odc.service.connection.model.ConnectionConfig; -import java.util.Map; - -import com.oceanbase.odc.service.task.enums.JobStatus; -import com.oceanbase.odc.service.task.schedule.JobIdentity; +import lombok.Getter; /** - * @author yaobin - * @date 2023-11-29 - * @since 4.2.4 + * @Author：tianke + * @Date: 2024/12/30 10:41 + * @Descripition: */ -public interface TaskResult { - - JobIdentity getJobIdentity(); - - JobStatus getStatus(); - - String getResultJson(); - - String getExecutorEndpoint(); +public class UpsertDatasourceEvent extends AbstractEvent { - String getErrorMessage(); + @Getter + private ConnectionConfig connectionConfig; - double getProgress(); + public UpsertDatasourceEvent(ConnectionConfig connectionConfig) { + super(connectionConfig, "UpsertDatasourceEvent"); + this.connectionConfig = connectionConfig; - Map getLogMetadata(); + } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/listener/UpdateDatasourceListener.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/listener/UpdateDatasourceListener.java new file mode 100644 index 0000000000..f52b9832bb --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/listener/UpdateDatasourceListener.java @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.connection.listener; + +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.springframework.util.CollectionUtils; + +import com.oceanbase.odc.common.event.AbstractEventListener; +import com.oceanbase.odc.core.shared.constant.DialectType; +import com.oceanbase.odc.metadb.connection.DatabaseEntity; +import com.oceanbase.odc.metadb.connection.DatabaseRepository; +import com.oceanbase.odc.service.connection.database.model.DatabaseSyncStatus; +import com.oceanbase.odc.service.connection.database.model.DatabaseType; +import com.oceanbase.odc.service.connection.event.UpsertDatasourceEvent; +import com.oceanbase.odc.service.connection.model.ConnectionConfig; +import com.oceanbase.odc.service.db.schema.model.DBObjectSyncStatus; + +import lombok.extern.slf4j.Slf4j; + +/** + * @Author：tinker + * @Date: 2024/12/30 10:47 + * @Descripition: + */ +@Slf4j +@Component +public class UpdateDatasourceListener extends AbstractEventListener { + + @Autowired + private DatabaseRepository databaseRepository; + + @Override + public void onEvent(UpsertDatasourceEvent event) { + + ConnectionConfig connectionConfig = event.getConnectionConfig(); + if (connectionConfig.getDialectType() != DialectType.FILE_SYSTEM) { + return; + } + List byConnectionId = databaseRepository.findByConnectionId(connectionConfig.getId()); + DatabaseEntity entity = null; + if (!CollectionUtils.isEmpty(byConnectionId)) { + List toBeDelete = byConnectionId.stream().filter( + o -> !connectionConfig.getHost().equals(o.getName())).map(DatabaseEntity::getId).collect( + Collectors.toList()); + if (!toBeDelete.isEmpty()) { + databaseRepository.deleteAllById(toBeDelete); + } + Optional existed = byConnectionId.stream().filter( + o -> connectionConfig.getHost().equals(o.getName())).findFirst(); + if (existed.isPresent()) { + entity = existed.get(); + } + } + // create or update + entity = entity == null ? new DatabaseEntity() : entity; + entity.setDatabaseId(com.oceanbase.odc.common.util.StringUtils.uuid()); + entity.setOrganizationId(connectionConfig.getOrganizationId()); + entity.setName(connectionConfig.getHost()); + entity.setProjectId(connectionConfig.getProjectId()); + entity.setConnectionId(connectionConfig.getId()); + entity.setEnvironmentId(connectionConfig.getEnvironmentId()); + entity.setSyncStatus(DatabaseSyncStatus.SUCCEEDED); + entity.setExisted(true); + entity.setObjectSyncStatus(DBObjectSyncStatus.SYNCED); + entity.setConnectType(connectionConfig.getType()); + entity.setType(DatabaseType.PHYSICAL); + databaseRepository.save(entity); + + } +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/LogicalTableService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/LogicalTableService.java index 05eca88989..a15984f755 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/LogicalTableService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/logicaldatabase/LogicalTableService.java @@ -267,8 +267,8 @@ public DetailLogicalTableResp detail(@NotNull Long logicalDatabaseId, @NotNull L TableExtensionPoint tableExtensionPoint = SchemaPluginUtil.getTableExtension(logicalDatabase.getConnectType().getDialectType()); if (tableExtensionPoint == null) { - throw new UnsupportedOperationException( - "Unsupported dialect " + logicalDatabase.getConnectType().getDialectType()); + throw new UnsupportedOperationException("the dialect " + logicalDatabase.getConnectType().getDialectType() + + " doesn't support the database object type " + DBObjectType.TABLE); } try (Connection connection = new DruidDataSourceFactory( connectionService.getForConnectionSkipPermissionCheck(physicalDatabase.getConnectionId())) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/model/ConnectProperties.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/model/ConnectProperties.java index 893c471017..33ad189c30 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/model/ConnectProperties.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/model/ConnectProperties.java @@ -25,8 +25,6 @@ import org.springframework.context.annotation.Configuration; import com.google.common.collect.Sets; -import com.oceanbase.odc.core.authority.permission.ConnectionPermission; -import com.oceanbase.odc.core.authority.permission.PrivateConnectionPermission; import lombok.Data; @@ -77,17 +75,7 @@ public Set getConnectionSupportedOperations(boolean temp, Set pe if (temp) { return getTempConnectionOperations(); } - if (CollectionUtils.isEmpty(permittedActions) - || permittedActions.contains(ConnectionPermission.CONNECTION_READWRITE) - || permittedActions.contains(PrivateConnectionPermission.CONNECTION_USE)) { - return getPersistentConnectionOperations(); - } - if (permittedActions.contains(ConnectionPermission.CONNECTION_READONLY)) { - Set operations = new HashSet<>(getPersistentConnectionOperations()); - operations.remove(DELETE_OPERATION); - return operations; - } - return NONE_OPERATIONS; + return getPersistentConnectionOperations(); } private Set getTempConnectionOperations() { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/model/ConnectionConfig.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/model/ConnectionConfig.java index b9bd8d1778..679ba0fdad 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/model/ConnectionConfig.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/model/ConnectionConfig.java @@ -546,6 +546,13 @@ public String getCloudProvider() { return o == null ? null : o.toString(); } + public void setRegion(@NotNull String region) { + if (this.attributes == null) { + attributes = new HashMap<>(); + } + attributes.put(REGION, region); + } + public String getRegion() { if (this.attributes == null) { return null; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/model/TestConnectionReq.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/model/TestConnectionReq.java index 8ca19b8fc9..1074d37549 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/model/TestConnectionReq.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/model/TestConnectionReq.java @@ -15,6 +15,7 @@ */ package com.oceanbase.odc.service.connection.model; +import java.util.HashMap; import java.util.Map; import java.util.Objects; @@ -46,6 +47,9 @@ @Data @ToString(exclude = {"password"}) public class TestConnectionReq implements CloudConnectionConfig, SSLConnectionConfig { + + private static final String REGION = "region"; + /** * Connection ID，用于编辑连接页面未传密码参数时从已保存的连接信息中获取对应密码字段 */ @@ -157,6 +161,21 @@ public DialectType getDialectType() { return this.dialectType; } + public void setRegion(String region) { + if (this.attributes == null) { + attributes = new HashMap<>(); + } + attributes.put(REGION, region); + } + + public String getRegion() { + if (this.attributes == null) { + return null; + } + Object o = attributes.get(REGION); + return o == null ? null : o.toString(); + } + public static TestConnectionReq fromConnection(ConnectionConfig connection, ConnectionAccountType accountType) { PreConditions.notNull(accountType, "AccountType"); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/table/TableService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/table/TableService.java index 6c02e0723e..ce7af41500 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/table/TableService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/table/TableService.java @@ -19,6 +19,7 @@ import java.sql.SQLException; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashSet; import java.util.LinkedHashSet; import java.util.List; @@ -29,6 +30,7 @@ import java.util.stream.Collectors; import javax.validation.Valid; +import javax.validation.constraints.NotNull; import org.apache.commons.collections4.CollectionUtils; import org.springframework.beans.factory.annotation.Autowired; @@ -47,17 +49,24 @@ import com.oceanbase.odc.core.shared.exception.UnsupportedException; import com.oceanbase.odc.metadb.dbobject.DBObjectEntity; import com.oceanbase.odc.metadb.dbobject.DBObjectRepository; +import com.oceanbase.odc.plugin.connect.api.InformationExtensionPoint; import com.oceanbase.odc.plugin.schema.api.TableExtensionPoint; +import com.oceanbase.odc.plugin.schema.api.ViewExtensionPoint; import com.oceanbase.odc.service.connection.database.DatabaseService; import com.oceanbase.odc.service.connection.database.model.Database; import com.oceanbase.odc.service.connection.model.ConnectionConfig; import com.oceanbase.odc.service.connection.table.model.QueryTableParams; import com.oceanbase.odc.service.connection.table.model.Table; import com.oceanbase.odc.service.db.schema.DBSchemaSyncService; +import com.oceanbase.odc.service.db.schema.syncer.DBSchemaSyncer; +import com.oceanbase.odc.service.db.schema.syncer.object.DBExternalTableSyncer; import com.oceanbase.odc.service.db.schema.syncer.object.DBTableSyncer; +import com.oceanbase.odc.service.db.schema.syncer.object.DBViewSyncer; +import com.oceanbase.odc.service.feature.VersionDiffConfigService; import com.oceanbase.odc.service.iam.auth.AuthenticationFacade; import com.oceanbase.odc.service.permission.DBResourcePermissionHelper; import com.oceanbase.odc.service.permission.database.model.DatabasePermissionType; +import com.oceanbase.odc.service.plugin.ConnectionPluginUtil; import com.oceanbase.odc.service.plugin.SchemaPluginUtil; import com.oceanbase.odc.service.session.factory.OBConsoleDataSourceFactory; import com.oceanbase.tools.dbbrowser.model.DBObjectIdentity; @@ -75,6 +84,9 @@ @Validated public class TableService { + @Autowired + private VersionDiffConfigService versionDiffConfigService; + @Autowired private DatabaseService databaseService; @@ -87,6 +99,12 @@ public class TableService { @Autowired private DBTableSyncer dbTableSyncer; + @Autowired + private DBExternalTableSyncer dbExternalTableSyncer; + + @Autowired + private DBViewSyncer dbViewSyncer; + @Autowired private JdbcLockRegistry lockRegistry; @@ -99,38 +117,106 @@ public class TableService { @Transactional(rollbackFor = Exception.class) @SkipAuthorize("permission check inside") public List
list(@NonNull @Valid QueryTableParams params) throws SQLException, InterruptedException { + List types = params.getTypes(); + if (CollectionUtils.isEmpty(types)) { + return Collections.emptyList(); + } Database database = databaseService.detail(params.getDatabaseId()); ConnectionConfig dataSource = database.getDataSource(); OBConsoleDataSourceFactory factory = new OBConsoleDataSourceFactory(dataSource, true); + List
tables = new ArrayList<>(); try (SingleConnectionDataSource ds = (SingleConnectionDataSource) factory.getDataSource(); Connection conn = ds.getConnection()) { - TableExtensionPoint point = SchemaPluginUtil.getTableExtension(dataSource.getDialectType()); - Set latestTableNames = point.list(conn, database.getName()) - .stream().map(DBObjectIdentity::getName).collect(Collectors.toCollection(LinkedHashSet::new)); - if (authenticationFacade.currentUser().getOrganizationType() == OrganizationType.INDIVIDUAL) { - return latestTableNames.stream().map(tableName -> { - Table table = new Table(); - table.setName(tableName); - table.setAuthorizedPermissionTypes(new HashSet<>(DatabasePermissionType.all())); - return table; - }).collect(Collectors.toList()); + TableExtensionPoint tableExtension = SchemaPluginUtil.getTableExtension(dataSource.getDialectType()); + if (tableExtension == null) { + throw new UnsupportedOperationException("the dialect " + dataSource.getDialectType() + + " doesn't support the database object type " + DBObjectType.TABLE); + } + if (types.contains(DBObjectType.TABLE)) { + Set latestTableNames = tableExtension.list(conn, database.getName(), DBObjectType.TABLE) + .stream().map(DBObjectIdentity::getName).collect(Collectors.toCollection(LinkedHashSet::new)); + generateListAndSyncDBTablesByTableType(params, database, dataSource, tables, conn, DBObjectType.TABLE, + latestTableNames); + } + if (types.contains(DBObjectType.EXTERNAL_TABLE)) { + InformationExtensionPoint point = + ConnectionPluginUtil.getInformationExtension(dataSource.getDialectType()); + String databaseProductVersion = point.getDBVersion(conn); + if (versionDiffConfigService.isExternalTableSupported(dataSource.getDialectType(), + databaseProductVersion)) { + Set latestExternalTableNames = + tableExtension.list(conn, database.getName(), DBObjectType.EXTERNAL_TABLE) + .stream().map(DBObjectIdentity::getName) + .collect(Collectors.toCollection(LinkedHashSet::new)); + generateListAndSyncDBTablesByTableType(params, database, dataSource, tables, conn, + DBObjectType.EXTERNAL_TABLE, latestExternalTableNames); + } + } + if (types.contains(DBObjectType.VIEW)) { + ViewExtensionPoint viewExtension = SchemaPluginUtil.getViewExtension(dataSource.getDialectType()); + if (viewExtension != null) { + Set latestViewNames = viewExtension.list(conn, database.getName()) + .stream().map(DBObjectIdentity::getName) + .collect(Collectors.toCollection(LinkedHashSet::new)); + generateListAndSyncDBTablesByTableType(params, database, dataSource, tables, conn, + DBObjectType.VIEW, + latestViewNames); + } } - List tables = + } + return tables; + } + + private void generateListAndSyncDBTablesByTableType(QueryTableParams params, Database database, + ConnectionConfig dataSource, List
tables, + Connection conn, DBObjectType tableType, Set latestTableNames) throws InterruptedException { + if (authenticationFacade.currentUser().getOrganizationType() == OrganizationType.INDIVIDUAL) { + tables.addAll(latestTableNames.stream().map(tableName -> { + Table table = new Table(); + table.setName(tableName); + table.setAuthorizedPermissionTypes(new HashSet<>(DatabasePermissionType.all())); + table.setType(tableType); + return table; + }).collect(Collectors.toList())); + } else { + List existTables = dbObjectRepository.findByDatabaseIdAndTypeOrderByNameAsc(params.getDatabaseId(), - DBObjectType.TABLE); - Set existTableNames = tables.stream().map(DBObjectEntity::getName).collect(Collectors.toSet()); - if (latestTableNames.size() != existTableNames.size() || !existTableNames.containsAll(latestTableNames)) { - syncDBTables(conn, database, dataSource.getDialectType()); - tables = dbObjectRepository.findByDatabaseIdAndTypeOrderByNameAsc(params.getDatabaseId(), - DBObjectType.TABLE); + tableType); + Set existTableNames = + existTables.stream().map(DBObjectEntity::getName).collect(Collectors.toSet()); + if (latestTableNames.size() != existTableNames.size() + || !existTableNames.containsAll(latestTableNames)) { + syncDBTables(conn, database, dataSource.getDialectType(), getSyncerByTableType(tableType)); + existTables = + dbObjectRepository.findByDatabaseIdAndTypeOrderByNameAsc(params.getDatabaseId(), + tableType); } - return entitiesToModels(tables, database, params.getIncludePermittedAction()); + tables.addAll(entitiesToModels(existTables, database, params.getIncludePermittedAction())); } } - @SkipAuthorize("permission check inside") - public void syncDBTables(Connection connection, Database database, DialectType dialectType) - throws InterruptedException { + private DBSchemaSyncer getSyncerByTableType(@NotNull DBObjectType tableType) { + switch (tableType) { + case TABLE: + return dbTableSyncer; + case EXTERNAL_TABLE: + return dbExternalTableSyncer; + case VIEW: + return dbViewSyncer; + default: + throw new IllegalArgumentException("Unsupported table type: " + tableType); + } + } + + // sync normal table + @SkipAuthorize("odc internal usage") + public void syncDBTables(@NotNull Connection connection, @NotNull Database database, + @NotNull DialectType dialectType) throws InterruptedException { + syncDBTables(connection, database, dialectType, dbTableSyncer); + } + + private void syncDBTables(@NotNull Connection connection, @NotNull Database database, + @NotNull DialectType dialectType, @NotNull DBSchemaSyncer syncer) throws InterruptedException { Lock lock = lockRegistry .obtain(dbSchemaSyncService.getSyncDBObjectLockKey(database.getDataSource().getId(), database.getId())); if (!lock.tryLock(3, TimeUnit.SECONDS)) { @@ -138,8 +224,8 @@ public void syncDBTables(Connection connection, Database database, DialectType d new Object[] {ResourceType.ODC_TABLE.getLocalizedMessage()}, "Can not acquire jdbc lock"); } try { - if (dbTableSyncer.supports(dialectType)) { - dbTableSyncer.sync(connection, database, dialectType); + if (syncer.supports(dialectType, connection)) { + syncer.sync(connection, database, dialectType); } else { throw new UnsupportedException("Unsupported dialect type: " + dialectType); } @@ -164,6 +250,7 @@ private List
entitiesToModels(Collection entities, Databa table.setCreateTime(entity.getCreateTime()); table.setUpdateTime(entity.getUpdateTime()); table.setOrganizationId(entity.getOrganizationId()); + table.setType(entity.getType()); if (includePermittedAction) { table.setAuthorizedPermissionTypes(id2Types.get(entity.getId())); } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/table/model/QueryTableParams.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/table/model/QueryTableParams.java index 78fe0e5eb7..441136438a 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/table/model/QueryTableParams.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/table/model/QueryTableParams.java @@ -15,8 +15,12 @@ */ package com.oceanbase.odc.service.connection.table.model; +import java.util.List; + import javax.validation.constraints.NotNull; +import com.oceanbase.tools.dbbrowser.model.DBObjectType; + import lombok.Builder; import lombok.Data; @@ -32,5 +36,11 @@ public class QueryTableParams { private Long databaseId; @NotNull private Boolean includePermittedAction; + /** + * table belonging to type in this collection needs to be fetched. if types is empty, return an + * empty collection. + */ + @NotNull + private List types; } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/table/model/Table.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/table/model/Table.java index 449946b7a6..fa5fd1d774 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/table/model/Table.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/table/model/Table.java @@ -26,6 +26,7 @@ import com.oceanbase.odc.core.shared.constant.ResourceType; import com.oceanbase.odc.service.connection.database.model.Database; import com.oceanbase.odc.service.permission.database.model.DatabasePermissionType; +import com.oceanbase.tools.dbbrowser.model.DBObjectType; import lombok.Data; @@ -61,6 +62,13 @@ public class Table implements SecurityResource, OrganizationIsolated, Serializab @JsonProperty(access = Access.READ_ONLY) private Set authorizedPermissionTypes; + /** + * Used to distinguish between external and basic tables, and additional table types may be added + * later. + */ + @JsonProperty(access = Access.READ_ONLY) + private DBObjectType type; + @Override public String resourceId() { return this.id == null ? null : this.id.toString(); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/util/ConnectionInfoUtil.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/util/ConnectionInfoUtil.java index 60a56cdaa6..0fce0f67c6 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/util/ConnectionInfoUtil.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/connection/util/ConnectionInfoUtil.java @@ -93,6 +93,19 @@ public static void initSessionVersion(@NonNull ConnectionSession connectionSessi log.debug("Init DB version completed."); } + public static void initOdpVersionIfExists(@NonNull ConnectionSession connectionSession) { + DialectType dialectType = connectionSession.getDialectType(); + if (dialectType != null && dialectType.isOceanbase()) { + String odpVersion = getSyncJdbcExecutor(connectionSession).execute(OBUtils::getODPVersion); + if (odpVersion == null) { + log.debug("OB Proxy does not exist or failed to obtain OB Proxy version."); + return; + } + connectionSession.setAttribute(ConnectionSessionConstants.ODP_VERSION, odpVersion); + log.debug("Init OB Proxy version completed."); + } + } + public static void killQuery(@NonNull String connectionId, @NonNull DataSourceFactory dataSourceFactory, DialectType dialectType) throws Exception { DataSource dataSource = dataSourceFactory.getDataSource(); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/datasecurity/DataMaskingService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/datasecurity/DataMaskingService.java index a7165a624a..a2af9edb97 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/datasecurity/DataMaskingService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/datasecurity/DataMaskingService.java @@ -61,8 +61,8 @@ import com.oceanbase.odc.service.datasecurity.util.MaskingAlgorithmUtil; import com.oceanbase.odc.service.iam.auth.AuthenticationFacade; import com.oceanbase.odc.service.session.model.SqlExecuteResult; -import com.oceanbase.odc.service.task.runtime.QuerySensitiveColumnReq; -import com.oceanbase.odc.service.task.runtime.QuerySensitiveColumnResp; +import com.oceanbase.odc.service.task.base.databasechange.QuerySensitiveColumnReq; +import com.oceanbase.odc.service.task.base.databasechange.QuerySensitiveColumnResp; import com.oceanbase.tools.sqlparser.statement.Statement; import lombok.NonNull; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/datasecurity/SensitiveColumnService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/datasecurity/SensitiveColumnService.java index 1be836cbd9..b6c41f248e 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/datasecurity/SensitiveColumnService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/datasecurity/SensitiveColumnService.java @@ -46,7 +46,9 @@ import com.oceanbase.odc.core.authority.util.PreAuthenticate; import com.oceanbase.odc.core.authority.util.SkipAuthorize; import com.oceanbase.odc.core.session.ConnectionSession; +import com.oceanbase.odc.core.session.ConnectionSessionUtil; import com.oceanbase.odc.core.shared.PreConditions; +import com.oceanbase.odc.core.shared.constant.DialectType; import com.oceanbase.odc.core.shared.constant.ErrorCodes; import com.oceanbase.odc.core.shared.constant.ResourceType; import com.oceanbase.odc.core.shared.exception.NotFoundException; @@ -115,7 +117,8 @@ public class SensitiveColumnService { private VersionDiffConfigService versionDiffConfigService; @Transactional(rollbackFor = Exception.class) - @PreAuthenticate(hasAnyResourceRole = {"OWNER, DBA, SECURITY_ADMINISTRATOR"}, resourceType = "ODC_PROJECT", + @PreAuthenticate(hasAnyResourceRole = {"OWNER, DBA, SECURITY_ADMINISTRATOR"}, + actions = {"OWNER", "DBA", "SECURITY_ADMINISTRATOR"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) public List listColumns(@NotNull Long projectId, @NotEmpty List databaseIds) { checkProjectDatabases(projectId, databaseIds); @@ -137,6 +140,12 @@ public List listColumns(@NotNull Long projectId, @NotEmp accessor.listBasicTableColumns(database.getName()), exists)); databaseColumn.setView2Columns(getFilteringExistColumns(database.getId(), accessor.listBasicViewColumns(database.getName()), exists)); + DialectType dialectType = session.getDialectType(); + String version = ConnectionSessionUtil.getVersion(session); + if (versionDiffConfigService.isExternalTableSupported(dialectType, version)) { + databaseColumn.setExternalTable2Columns(getFilteringExistColumns(database.getId(), + accessor.listBasicExternalTableColumns(database.getName()), exists)); + } databaseColumn.setDataTypeUnits(versionDiffConfigService.getDatatypeList(session)); if (!databaseColumn.getTable2Columns().isEmpty() || !databaseColumn.getView2Columns().isEmpty()) { databaseColumns.add(databaseColumn); @@ -149,7 +158,8 @@ public List listColumns(@NotNull Long projectId, @NotEmp } @Transactional(rollbackFor = Exception.class) - @PreAuthenticate(hasAnyResourceRole = {"OWNER, DBA, SECURITY_ADMINISTRATOR"}, resourceType = "ODC_PROJECT", + @PreAuthenticate(hasAnyResourceRole = {"OWNER, DBA, SECURITY_ADMINISTRATOR"}, + actions = {"OWNER", "DBA", "SECURITY_ADMINISTRATOR"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) public Boolean exists(@NotNull Long projectId, @NotNull SensitiveColumn column) { PreConditions.notNull(column.getDatabase(), "database"); @@ -172,7 +182,8 @@ public boolean existsInCurrentOrganization() { } @Transactional(rollbackFor = Exception.class) - @PreAuthenticate(hasAnyResourceRole = {"OWNER, DBA, SECURITY_ADMINISTRATOR"}, resourceType = "ODC_PROJECT", + @PreAuthenticate(hasAnyResourceRole = {"OWNER, DBA, SECURITY_ADMINISTRATOR"}, + actions = {"OWNER", "DBA", "SECURITY_ADMINISTRATOR"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) public List batchCreate(@NotNull Long projectId, @NotEmpty @Valid List columns) { @@ -204,7 +215,8 @@ public List batchCreate(@NotNull Long projectId, } @Transactional(rollbackFor = Exception.class) - @PreAuthenticate(hasAnyResourceRole = {"OWNER, DBA, SECURITY_ADMINISTRATOR"}, resourceType = "ODC_PROJECT", + @PreAuthenticate(hasAnyResourceRole = {"OWNER, DBA, SECURITY_ADMINISTRATOR"}, + actions = {"OWNER", "DBA", "SECURITY_ADMINISTRATOR"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) public SensitiveColumn detail(@NotNull Long projectId, @NotNull Long id) { SensitiveColumnEntity entity = nullSafeGet(id); @@ -231,7 +243,8 @@ public SensitiveColumn detail(@NotNull Long projectId, @NotNull Long id) { } @Transactional(rollbackFor = Exception.class) - @PreAuthenticate(hasAnyResourceRole = {"OWNER, DBA, SECURITY_ADMINISTRATOR"}, resourceType = "ODC_PROJECT", + @PreAuthenticate(hasAnyResourceRole = {"OWNER, DBA, SECURITY_ADMINISTRATOR"}, + actions = {"OWNER", "DBA", "SECURITY_ADMINISTRATOR"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) public List batchUpdate(@NotNull Long projectId, @NotEmpty List ids, @NotNull Long maskingAlgorithmId) { @@ -249,7 +262,8 @@ public List batchUpdate(@NotNull Long projectId, @NotEmpty List } @Transactional(rollbackFor = Exception.class) - @PreAuthenticate(hasAnyResourceRole = {"OWNER, DBA, SECURITY_ADMINISTRATOR"}, resourceType = "ODC_PROJECT", + @PreAuthenticate(hasAnyResourceRole = {"OWNER, DBA, SECURITY_ADMINISTRATOR"}, + actions = {"OWNER", "DBA", "SECURITY_ADMINISTRATOR"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) public List batchDelete(@NotNull Long projectId, @NotEmpty List ids) { List entities = batchNullSafeGet(new HashSet<>(ids)); @@ -264,7 +278,8 @@ public List batchDelete(@NotNull Long projectId, @NotEmpty List } @Transactional(rollbackFor = Exception.class) - @PreAuthenticate(hasAnyResourceRole = {"OWNER, DBA, SECURITY_ADMINISTRATOR"}, resourceType = "ODC_PROJECT", + @PreAuthenticate(hasAnyResourceRole = {"OWNER, DBA, SECURITY_ADMINISTRATOR"}, + actions = {"OWNER", "DBA", "SECURITY_ADMINISTRATOR"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) public Page list(@NotNull Long projectId, @NotNull QuerySensitiveColumnParams params, Pageable pageable) { @@ -306,7 +321,8 @@ public Page list(@NotNull Long projectId, @NotNull QuerySensiti } @Transactional(rollbackFor = Exception.class) - @PreAuthenticate(hasAnyResourceRole = {"OWNER, DBA, SECURITY_ADMINISTRATOR"}, resourceType = "ODC_PROJECT", + @PreAuthenticate(hasAnyResourceRole = {"OWNER, DBA, SECURITY_ADMINISTRATOR"}, + actions = {"OWNER", "DBA", "SECURITY_ADMINISTRATOR"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) public SensitiveColumnStats stats(@NotNull Long projectId) { SensitiveColumnStats stats = new SensitiveColumnStats(); @@ -332,7 +348,8 @@ public SensitiveColumnStats stats(@NotNull Long projectId) { } @Transactional(rollbackFor = Exception.class) - @PreAuthenticate(hasAnyResourceRole = {"OWNER, DBA, SECURITY_ADMINISTRATOR"}, resourceType = "ODC_PROJECT", + @PreAuthenticate(hasAnyResourceRole = {"OWNER, DBA, SECURITY_ADMINISTRATOR"}, + actions = {"OWNER", "DBA", "SECURITY_ADMINISTRATOR"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) public SensitiveColumn setEnabled(@NotNull Long projectId, @NotNull Long id, @NotNull Boolean enabled) { SensitiveColumnEntity entity = nullSafeGet(id); @@ -347,7 +364,8 @@ public SensitiveColumn setEnabled(@NotNull Long projectId, @NotNull Long id, @No } @Transactional(rollbackFor = Exception.class) - @PreAuthenticate(hasAnyResourceRole = {"OWNER, DBA, SECURITY_ADMINISTRATOR"}, resourceType = "ODC_PROJECT", + @PreAuthenticate(hasAnyResourceRole = {"OWNER, DBA, SECURITY_ADMINISTRATOR"}, + actions = {"OWNER", "DBA", "SECURITY_ADMINISTRATOR"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) public SensitiveColumnScanningTaskInfo startScanning(@NotNull Long projectId, @NotNull @Valid SensitiveColumnScanningReq req) { @@ -378,7 +396,8 @@ public SensitiveColumnScanningTaskInfo startScanning(@NotNull Long projectId, } @Transactional(rollbackFor = Exception.class) - @PreAuthenticate(hasAnyResourceRole = {"OWNER, DBA, SECURITY_ADMINISTRATOR"}, resourceType = "ODC_PROJECT", + @PreAuthenticate(hasAnyResourceRole = {"OWNER, DBA, SECURITY_ADMINISTRATOR"}, + actions = {"OWNER", "DBA", "SECURITY_ADMINISTRATOR"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) @StatefulRoute(stateName = StateName.UUID_STATEFUL_ID, stateIdExpression = "#taskId") public SensitiveColumnScanningTaskInfo getScanningResults(@NotNull Long projectId, @NotBlank String taskId) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/datasecurity/SensitiveRuleService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/datasecurity/SensitiveRuleService.java index 3174ca5ec1..0bc0fb9b05 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/datasecurity/SensitiveRuleService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/datasecurity/SensitiveRuleService.java @@ -85,7 +85,8 @@ public class SensitiveRuleService { private static final SensitiveRuleMapper ruleMapper = SensitiveRuleMapper.INSTANCE; @Transactional(rollbackFor = Exception.class) - @PreAuthenticate(hasAnyResourceRole = {"OWNER, DBA"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) + @PreAuthenticate(hasAnyResourceRole = {"OWNER, DBA"}, actions = {"OWNER", "DBA"}, resourceType = "ODC_PROJECT", + indexOfIdParam = 0) public Boolean exists(@NotNull Long projectId, @NotBlank String name) { SensitiveRuleEntity entity = new SensitiveRuleEntity(); entity.setName(name); @@ -95,7 +96,8 @@ public Boolean exists(@NotNull Long projectId, @NotBlank String name) { } @Transactional(rollbackFor = Exception.class) - @PreAuthenticate(hasAnyResourceRole = {"OWNER, DBA, SECURITY_ADMINISTRATOR"}, resourceType = "ODC_PROJECT", + @PreAuthenticate(hasAnyResourceRole = {"OWNER, DBA, SECURITY_ADMINISTRATOR"}, + actions = {"OWNER", "DBA", "SECURITY_ADMINISTRATOR"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) public SensitiveRule create(@NotNull Long projectId, @NotNull @Valid SensitiveRule rule) { Long organizationId = authenticationFacade.currentOrganizationId(); @@ -115,7 +117,8 @@ public SensitiveRule create(@NotNull Long projectId, @NotNull @Valid SensitiveRu } @Transactional(rollbackFor = Exception.class) - @PreAuthenticate(hasAnyResourceRole = {"OWNER, DBA, SECURITY_ADMINISTRATOR"}, resourceType = "ODC_PROJECT", + @PreAuthenticate(hasAnyResourceRole = {"OWNER, DBA, SECURITY_ADMINISTRATOR"}, + actions = {"OWNER", "DBA", "SECURITY_ADMINISTRATOR"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) public SensitiveRule detail(@NotNull Long projectId, @NotNull Long id) { SensitiveRuleEntity entity = nullSafeGet(id); @@ -130,7 +133,8 @@ public SensitiveRule detail(@NotNull Long projectId, @NotNull Long id) { } @Transactional(rollbackFor = Exception.class) - @PreAuthenticate(hasAnyResourceRole = {"OWNER, DBA, SECURITY_ADMINISTRATOR"}, resourceType = "ODC_PROJECT", + @PreAuthenticate(hasAnyResourceRole = {"OWNER, DBA, SECURITY_ADMINISTRATOR"}, + actions = {"OWNER", "DBA", "SECURITY_ADMINISTRATOR"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) public SensitiveRule update(@NotNull Long projectId, @NotNull Long id, @NotNull @Valid SensitiveRule rule) { SensitiveRuleEntity entity = nullSafeGet(id); @@ -162,7 +166,8 @@ public SensitiveRule update(@NotNull Long projectId, @NotNull Long id, @NotNull } @Transactional(rollbackFor = Exception.class) - @PreAuthenticate(hasAnyResourceRole = {"OWNER, DBA, SECURITY_ADMINISTRATOR"}, resourceType = "ODC_PROJECT", + @PreAuthenticate(hasAnyResourceRole = {"OWNER, DBA, SECURITY_ADMINISTRATOR"}, + actions = {"OWNER", "DBA", "SECURITY_ADMINISTRATOR"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) public SensitiveRule delete(@NotNull Long projectId, @NotNull Long id) { SensitiveRuleEntity entity = nullSafeGet(id); @@ -179,7 +184,8 @@ public SensitiveRule delete(@NotNull Long projectId, @NotNull Long id) { } @Transactional(rollbackFor = Exception.class) - @PreAuthenticate(hasAnyResourceRole = {"OWNER, DBA, SECURITY_ADMINISTRATOR"}, resourceType = "ODC_PROJECT", + @PreAuthenticate(hasAnyResourceRole = {"OWNER, DBA, SECURITY_ADMINISTRATOR"}, + actions = {"OWNER", "DBA", "SECURITY_ADMINISTRATOR"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) public Page list(@NotNull Long projectId, @NotNull @Valid QuerySensitiveRuleParams params, Pageable pageable) { @@ -195,7 +201,8 @@ public Page list(@NotNull Long projectId, @NotNull @Valid QuerySe } @Transactional(rollbackFor = Exception.class) - @PreAuthenticate(hasAnyResourceRole = {"OWNER, DBA, SECURITY_ADMINISTRATOR"}, resourceType = "ODC_PROJECT", + @PreAuthenticate(hasAnyResourceRole = {"OWNER, DBA, SECURITY_ADMINISTRATOR"}, + actions = {"OWNER", "DBA", "SECURITY_ADMINISTRATOR"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) public SensitiveRule setEnabled(@NotNull Long projectId, @NotNull Long id, @NotNull Boolean enabled) { SensitiveRuleEntity entity = nullSafeGet(id); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/datasecurity/model/DatabaseWithAllColumns.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/datasecurity/model/DatabaseWithAllColumns.java index 2cbd7e8444..580c193c39 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/datasecurity/model/DatabaseWithAllColumns.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/datasecurity/model/DatabaseWithAllColumns.java @@ -35,6 +35,7 @@ public class DatabaseWithAllColumns { private String databaseName; private Map> table2Columns; private Map> view2Columns; + private Map> externalTable2Columns; /** * Mapping from database type to show type, used for displaying column type icon */ diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/datatransfer/DBObjectNameAccessor.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/datatransfer/DBObjectNameAccessor.java index 733858cd77..b3038a86c8 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/datatransfer/DBObjectNameAccessor.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/datatransfer/DBObjectNameAccessor.java @@ -30,6 +30,7 @@ import com.oceanbase.odc.service.plugin.SchemaPluginUtil; import com.oceanbase.odc.service.session.factory.DefaultConnectSessionFactory; import com.oceanbase.tools.dbbrowser.model.DBObjectIdentity; +import com.oceanbase.tools.dbbrowser.model.DBObjectType; import com.oceanbase.tools.dbbrowser.model.DBSynonymType; import com.oceanbase.tools.loaddump.common.enums.ObjectType; @@ -93,7 +94,7 @@ public Set getObjectNames(@NonNull ObjectType objectType) { public Set getTableNames() { return queryNames(conn -> SchemaPluginUtil.getTableExtension(dialectType) - .list(conn, schema)).stream() + .list(conn, schema, DBObjectType.TABLE)).stream() .map(DBObjectIdentity::getName) .filter(name -> !StringUtils.endsWithIgnoreCase(name, OdcConstants.VALIDATE_DDL_TABLE_POSTFIX)) .collect(Collectors.toSet()); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/DBIdentitiesService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/DBIdentitiesService.java index cc91e234df..895c3fc320 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/DBIdentitiesService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/DBIdentitiesService.java @@ -47,6 +47,9 @@ public List list(ConnectionSession session, List if (types.contains(DBObjectType.TABLE)) { listTables(schemaAccessor, all); } + if (types.contains(DBObjectType.EXTERNAL_TABLE)) { + listExternalTables(schemaAccessor, all); + } schemaAccessor.showDatabases().forEach(db -> all.computeIfAbsent(db, SchemaIdentities::of)); return new ArrayList<>(all.values()); } @@ -63,4 +66,9 @@ void listViews(DBSchemaAccessor schemaAccessor, Map al .forEach(i -> all.computeIfAbsent(i.getSchemaName(), SchemaIdentities::of).add(i)); } + void listExternalTables(DBSchemaAccessor schemaAccessor, Map all) { + schemaAccessor.listExternalTables(null, null) + .forEach(i -> all.computeIfAbsent(i.getSchemaName(), SchemaIdentities::of).add(i)); + } + } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/DBPLModifyHelper.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/DBPLModifyHelper.java new file mode 100644 index 0000000000..a3e193d503 --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/DBPLModifyHelper.java @@ -0,0 +1,163 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.db; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.Lock; + +import javax.validation.Valid; +import javax.validation.constraints.NotNull; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.integration.jdbc.lock.JdbcLockRegistry; +import org.springframework.stereotype.Component; +import org.springframework.validation.annotation.Validated; + +import com.oceanbase.odc.common.util.StringUtils; +import com.oceanbase.odc.common.util.VersionUtils; +import com.oceanbase.odc.core.authority.util.SkipAuthorize; +import com.oceanbase.odc.core.session.ConnectionSession; +import com.oceanbase.odc.core.session.ConnectionSessionUtil; +import com.oceanbase.odc.core.shared.constant.ErrorCodes; +import com.oceanbase.odc.core.shared.exception.BadRequestException; +import com.oceanbase.odc.core.shared.exception.ConflictException; +import com.oceanbase.odc.core.sql.execute.model.SqlExecuteStatus; +import com.oceanbase.odc.core.sql.split.SqlCommentProcessor; +import com.oceanbase.odc.service.connection.model.ConnectionConfig; +import com.oceanbase.odc.service.db.model.EditPLReq; +import com.oceanbase.odc.service.db.model.EditPLResp; +import com.oceanbase.odc.service.session.ConnectConsoleService; +import com.oceanbase.odc.service.session.ConnectSessionService; +import com.oceanbase.odc.service.session.model.AsyncExecuteResultResp; +import com.oceanbase.odc.service.session.model.SqlAsyncExecuteReq; +import com.oceanbase.odc.service.session.model.SqlAsyncExecuteResp; +import com.oceanbase.odc.service.session.model.SqlExecuteResult; +import com.oceanbase.tools.dbbrowser.model.DBObjectType; +import com.oceanbase.tools.dbbrowser.util.MySQLSqlBuilder; + +import lombok.NonNull; + +/** + * @description: + * @author: zijia.cj + * @date: 2024/10/17 18:59 + * @since: 4.3.3 + */ +@Validated +@Component +@SkipAuthorize("permission check inside") +public class DBPLModifyHelper { + public static final String ODC_TEMPORARY_PROCEDURE = "_ODC_TEMPORARY_PROCEDURE"; + public static final String ODC_TEMPORARY_TRIGGER = "_ODC_TEMPORARY_TRIGGER"; + public static final String ODC_TEMPORARY_FUNCTION = "_ODC_TEMPORARY_FUNCTION"; + public static final String OB_VERSION_SUPPORT_MULTIPLE_SAME_TRIGGERS = "4.2"; + public static final int LOCK_TIMEOUT_SECONDS = 3; + @Autowired + private ConnectConsoleService connectConsoleService; + @Autowired + private JdbcLockRegistry jdbcLockRegistry; + @Autowired + private ConnectSessionService sessionService; + + public EditPLResp editPL(@NotNull String sessionId, @NotNull @Valid EditPLReq editPLReq) + throws Exception { + DBObjectType plType = editPLReq.getObjectType(); + switch (plType) { + case PROCEDURE: + return executeWrappedEditPL(sessionId, editPLReq, DBObjectType.PROCEDURE, ODC_TEMPORARY_PROCEDURE); + case FUNCTION: + return executeWrappedEditPL(sessionId, editPLReq, DBObjectType.FUNCTION, ODC_TEMPORARY_FUNCTION); + case TRIGGER: + if (VersionUtils.isLessThan( + ConnectionSessionUtil.getVersion(sessionService.nullSafeGet(sessionId, true)), + OB_VERSION_SUPPORT_MULTIPLE_SAME_TRIGGERS)) { + throw new BadRequestException( + "editing trigger in mysql mode is not supported in ob version less than " + + OB_VERSION_SUPPORT_MULTIPLE_SAME_TRIGGERS); + } + return executeWrappedEditPL(sessionId, editPLReq, DBObjectType.TRIGGER, ODC_TEMPORARY_TRIGGER); + default: + throw new IllegalArgumentException( + String.format("the pl type %s of editing procedure is not supported", plType)); + } + } + + private EditPLResp executeWrappedEditPL(String sessionId, EditPLReq editPLReq, + DBObjectType plType, String tempPlName) throws Exception { + String plName = editPLReq.getObjectName(); + String editPLSql = editPLReq.getSql(); + String escapeRegexPlName = StringUtils.escapeRegex(plName) + .orElseThrow(() -> new IllegalStateException(String.format("%s name cannot be null", plType))); + String tempPLSql = editPLSql.replaceFirst(escapeRegexPlName, tempPlName); + MySQLSqlBuilder wrappedSqlBuilder = new MySQLSqlBuilder(); + ConnectionSession connectionSession = sessionService.nullSafeGet(sessionId, true); + SqlCommentProcessor processor = ConnectionSessionUtil.getSqlCommentProcessor(connectionSession); + String delimiter = processor.getDelimiter(); + wrappedSqlBuilder.append("DELIMITER $$\n") + .append(tempPLSql).append(" $$\n") + .append("drop ").append(plType).append(" if exists ").identifier(tempPlName).append(" $$\n") + .append("drop ").append(plType).append(" if exists ").identifier(plName).append(" $$\n") + .append(editPLSql).append(" $$\n") + .append("DELIMITER " + delimiter); + String wrappedSql = wrappedSqlBuilder.toString(); + SqlAsyncExecuteReq sqlAsyncExecuteReq = new SqlAsyncExecuteReq(); + sqlAsyncExecuteReq.setSql(wrappedSql); + sqlAsyncExecuteReq.setSplit(true); + sqlAsyncExecuteReq.setContinueExecutionOnError(false); + Lock editPLLock = obtainEditPLLock(connectionSession, plType); + if (!editPLLock.tryLock(LOCK_TIMEOUT_SECONDS, TimeUnit.SECONDS)) { + throw new ConflictException(ErrorCodes.ResourceModifying, "Can not acquire jdbc lock"); + } + try { + SqlAsyncExecuteResp sqlAsyncExecuteResp = connectConsoleService.streamExecute(sessionId, sqlAsyncExecuteReq, + true); + EditPLResp editPLResp = new EditPLResp(); + editPLResp.setWrappedSql(wrappedSql); + editPLResp.setSqls(sqlAsyncExecuteResp.getSqls()); + editPLResp.setUnauthorizedDBResources(sqlAsyncExecuteResp.getUnauthorizedDBResources()); + editPLResp.setViolatedRules(sqlAsyncExecuteResp.getViolatedRules()); + editPLResp.setApprovalRequired(sqlAsyncExecuteResp.isApprovalRequired()); + if (editPLResp.isApprovalRequired()) { + return editPLResp; + } + AsyncExecuteResultResp moreResults; + List results = new ArrayList<>(); + do { + moreResults = connectConsoleService.getMoreResults(sessionId, + sqlAsyncExecuteResp.getRequestId()); + results.addAll(moreResults.getResults()); + } while (!moreResults.isFinished()); + for (SqlExecuteResult result : results) { + if (result.getStatus() != SqlExecuteStatus.SUCCESS) { + editPLResp.setErrorMessage(result.getTrack()); + return editPLResp; + } + } + return editPLResp; + } finally { + editPLLock.unlock(); + } + } + + private Lock obtainEditPLLock(@NotNull ConnectionSession connectionSession, @NonNull DBObjectType plType) { + ConnectionConfig connConfig = + (ConnectionConfig) ConnectionSessionUtil.getConnectionConfig(connectionSession); + Long dataSourceId = connConfig.getId(); + return jdbcLockRegistry.obtain(String.format("%s-%d", plType, dataSourceId)); + } +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/DBPLService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/DBPLService.java index 40d3570b96..ae23a61aeb 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/DBPLService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/DBPLService.java @@ -51,6 +51,7 @@ import com.oceanbase.odc.core.shared.exception.NotFoundException; import com.oceanbase.odc.core.shared.exception.UnsupportedException; import com.oceanbase.odc.core.sql.execute.SyncJdbcExecutor; +import com.oceanbase.odc.core.sql.execute.mapper.DefaultJdbcRowMapper; import com.oceanbase.odc.core.sql.execute.task.DefaultSqlExecuteTaskManager; import com.oceanbase.odc.core.sql.split.OffsetString; import com.oceanbase.odc.core.sql.split.SqlCommentProcessor; @@ -287,7 +288,8 @@ public String callFunction(@NonNull ConnectionSession session, @NonNull CallFunc if (dialectType.isOracle()) { callback = new OBOracleCallFunctionBlockCallBack(req, -1); } else if (dialectType.isMysql()) { - callback = new OBMysqlCallFunctionCallBack(req, -1); + DefaultJdbcRowMapper defaultJdbcRowMapper = new DefaultJdbcRowMapper(session); + callback = new OBMysqlCallFunctionCallBack(req, -1, defaultJdbcRowMapper); } else { throw new IllegalArgumentException("Illegal dialect type, " + dialectType); } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/DBTableService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/DBTableService.java index 342957a785..2ba8ba1d50 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/DBTableService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/DBTableService.java @@ -49,6 +49,7 @@ import com.oceanbase.odc.service.sqlcheck.SqlCheckUtil; import com.oceanbase.tools.dbbrowser.DBBrowser; import com.oceanbase.tools.dbbrowser.model.DBObjectIdentity; +import com.oceanbase.tools.dbbrowser.model.DBObjectType; import com.oceanbase.tools.dbbrowser.model.DBTable; import com.oceanbase.tools.dbbrowser.schema.DBSchemaAccessor; import com.oceanbase.tools.sqlparser.statement.Statement; @@ -88,11 +89,18 @@ public List showTablesLike(@NotNull ConnectionSession session, String sc } public DBTable getTable(@NotNull ConnectionSession connectionSession, String schemaName, - @NotBlank String tableName) { + @NotBlank String tableName, @NotNull DBObjectType type) { DBSchemaAccessor schemaAccessor = DBSchemaAccessors.create(connectionSession); - PreConditions.validExists(ResourceType.OB_TABLE, "tableName", tableName, - () -> schemaAccessor.showTables(schemaName).stream().filter(name -> name.equals(tableName)) - .collect(Collectors.toList()).size() > 0); + if (type == DBObjectType.TABLE) { + PreConditions.validExists(ResourceType.OB_TABLE, "tableName", tableName, + () -> schemaAccessor.showTables(schemaName).stream().filter(name -> name.equals(tableName)) + .collect(Collectors.toList()).size() > 0); + } + if (type == DBObjectType.EXTERNAL_TABLE) { + PreConditions.validExists(ResourceType.OB_TABLE, "tableName", tableName, + () -> schemaAccessor.showExternalTables(schemaName).stream().filter(name -> name.equals(tableName)) + .collect(Collectors.toList()).size() > 0); + } try { return connectionSession.getSyncJdbcExecutor( ConnectionSessionConstants.BACKEND_DS_KEY) @@ -116,7 +124,7 @@ public Map getTables(@NotNull ConnectionSession connectionSessi public List listTables(@NotNull ConnectionSession connectionSession, String schemaName) { return connectionSession.getSyncJdbcExecutor(ConnectionSessionConstants.BACKEND_DS_KEY) .execute((ConnectionCallback>) con -> getTableExtensionPoint(connectionSession) - .list(con, schemaName)) + .list(con, schemaName, DBObjectType.TABLE)) .stream().map(item -> { DBTable table = new DBTable(); table.setName(item.getName()); @@ -175,6 +183,18 @@ public GenerateTableDDLResp generateUpdateDDL(@NotNull ConnectionSession session .build(); } + public boolean syncExternalTableFiles(@NotNull ConnectionSession connectionSession, String schemaName, + @NotBlank String externalTableName) { + DBSchemaAccessor schemaAccessor = DBSchemaAccessors.create(connectionSession); + PreConditions.validExists(ResourceType.OB_TABLE, "tableName", externalTableName, + () -> schemaAccessor.showExternalTables(schemaName).stream() + .filter(name -> name.equals(externalTableName)) + .collect(Collectors.toList()).size() > 0); + return connectionSession.getSyncJdbcExecutor(ConnectionSessionConstants.BACKEND_DS_KEY) + .execute((ConnectionCallback) con -> getTableExtensionPoint(connectionSession) + .syncExternalTableFiles(con, schemaName, externalTableName)); + } + private String checkUpdateDDL(DialectType dialectType, String ddl) { boolean createIndex = false; boolean dropIndex = false; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/model/EditPLReq.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/model/EditPLReq.java new file mode 100644 index 0000000000..7c0fba2fa0 --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/model/EditPLReq.java @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.db.model; + +import javax.validation.constraints.NotBlank; +import javax.validation.constraints.NotNull; + +import com.oceanbase.tools.dbbrowser.model.DBObjectType; + +import lombok.Data; + +/** + * @description: + * @author: zijia.cj + * @date: 2024/10/18 13:00 + * @since: 4.3.3 + */ +@Data +public class EditPLReq { + @NotBlank + private String sql; + @NotBlank + private String objectName; + @NotNull + private DBObjectType objectType; +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/model/EditPLResp.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/model/EditPLResp.java new file mode 100644 index 0000000000..2bedcf9690 --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/model/EditPLResp.java @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.db.model; + +import java.util.List; + +import com.oceanbase.odc.service.connection.database.model.UnauthorizedDBResource; +import com.oceanbase.odc.service.regulation.ruleset.model.Rule; +import com.oceanbase.odc.service.session.model.SqlTuplesWithViolation; + +import lombok.Data; + +/** + * @description: + * @author: zijia.cj + * @date: 2024/10/18 14:11 + * @since: 4.3.3 + */ +@Data +public class EditPLResp { + // if pre-check is not successful ,sql confirmation window needs to display wrapped Sql instead of + // the original sql.Because what really executes is wrapped Sql + private String wrappedSql; + // if shouldIntercepted is true，it indicates sql pre-check does not pass. sql confirmation window + // needs to be opened + private boolean approvalRequired; + private List violatedRules; + private List sqls; + private List unauthorizedDBResources; + // if none, the modification succeeded.If not none, the modification failed.it indicates the error + // message + private String errorMessage; +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/DBSchemaSyncService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/DBSchemaSyncService.java index 2d00606786..466b6ee216 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/DBSchemaSyncService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/DBSchemaSyncService.java @@ -90,7 +90,7 @@ public boolean sync(@NonNull Database database) throws InterruptedException, SQL Connection conn = dataSource.getConnection()) { boolean success = true; for (DBSchemaSyncer syncer : syncers) { - if (syncer.supports(config.getDialectType())) { + if (syncer.supports(config.getDialectType(), conn)) { try { syncer.sync(conn, database, config.getDialectType()); } catch (UnsupportedOperationException | UnsupportedException | NotImplementedException e) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/DBSchemaSyncTaskManager.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/DBSchemaSyncTaskManager.java index a7440a5cb0..c79e860b05 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/DBSchemaSyncTaskManager.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/DBSchemaSyncTaskManager.java @@ -27,6 +27,7 @@ import org.apache.commons.collections4.CollectionUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Lazy; import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; import org.springframework.stereotype.Component; @@ -34,6 +35,7 @@ import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; +import com.oceanbase.odc.core.shared.constant.DialectType; import com.oceanbase.odc.core.shared.constant.ResourceType; import com.oceanbase.odc.core.shared.exception.ConflictException; import com.oceanbase.odc.core.shared.exception.NotFoundException; @@ -82,6 +84,10 @@ public class DBSchemaSyncTaskManager { @Autowired private GlobalSearchProperties globalSearchProperties; + @Value("${odc.integration.bastion.enabled:false}") + private boolean bastionEnabled; + + private final LoadingCache datasourceId2UserEntity = CacheBuilder.newBuilder().maximumSize(100) .expireAfterWrite(10, TimeUnit.MINUTES) .build(new CacheLoader() { @@ -113,9 +119,13 @@ public void submitTaskByDatabases(@NonNull Collection databases) { } public void submitTaskByDataSource(@NonNull ConnectionConfig dataSource) { + if (dataSource.getDialectType() == DialectType.FILE_SYSTEM) { + return; + } List databases = databaseService.listExistDatabasesByConnectionId(dataSource.getId()); databases.removeIf(e -> (syncProperties.isBlockExclusionsWhenSyncDbSchemas() - && syncProperties.getExcludeSchemas(dataSource.getDialectType()).contains(e.getName())) + && syncProperties.getExcludeSchemas(dataSource.getDialectType()).contains(e.getName()) + && !bastionEnabled) || e.getObjectSyncStatus() == DBObjectSyncStatus.PENDING); submitTaskByDatabases(databases); } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/GlobalSearchProperties.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/GlobalSearchProperties.java index 1024d4df08..8d581501b5 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/GlobalSearchProperties.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/GlobalSearchProperties.java @@ -40,12 +40,12 @@ public class GlobalSearchProperties { @Value("${odc.database.schema.global-search.max-pending-hours:1}") private long maxPendingHours; + public long getMaxPendingHours() { + return this.maxPendingHours <= 0 ? 1 : this.maxPendingHours; + } + public long getMaxPendingMillis() { - long maxPendingHours = this.maxPendingHours; - if (this.maxPendingHours <= 0) { - maxPendingHours = 1; - } - return TimeUnit.MILLISECONDS.convert(this.maxPendingHours, TimeUnit.HOURS); + return TimeUnit.MILLISECONDS.convert(getMaxPendingHours(), TimeUnit.HOURS); } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/syncer/DBSchemaSyncer.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/syncer/DBSchemaSyncer.java index b9750d0899..3033e32d62 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/syncer/DBSchemaSyncer.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/syncer/DBSchemaSyncer.java @@ -42,11 +42,12 @@ public interface DBSchemaSyncer extends Ordered { /** * Check if the synchronizer supports the dialect type - * + * * @param dialectType dialect type, refer to {@link DialectType} + * @param connection * @return true if the synchronizer supports the dialect type, otherwise false */ - boolean supports(@NonNull DialectType dialectType); + boolean supports(@NonNull DialectType dialectType, @NonNull Connection connection); /** * Get the object type that the synchronizer supports diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/syncer/column/AbstractDBColumnSyncer.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/syncer/column/AbstractDBColumnSyncer.java index 0d65905f4a..8e99f893b0 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/syncer/column/AbstractDBColumnSyncer.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/syncer/column/AbstractDBColumnSyncer.java @@ -36,6 +36,7 @@ import com.oceanbase.odc.metadb.dbobject.DBObjectEntity; import com.oceanbase.odc.metadb.dbobject.DBObjectRepository; import com.oceanbase.odc.service.connection.database.model.Database; +import com.oceanbase.odc.service.connection.database.model.DatabaseType; import com.oceanbase.odc.service.db.schema.syncer.DBSchemaSyncer; import com.oceanbase.odc.service.plugin.SchemaPluginUtil; import com.oceanbase.tools.dbbrowser.model.DBObjectType; @@ -58,6 +59,9 @@ public abstract class AbstractDBColumnSyncer implement @Override public void sync(@NonNull Connection connection, @NonNull Database database, @NonNull DialectType dialectType) { + if (database.getType() != DatabaseType.PHYSICAL) { + return; + } T extensionPoint = getExtensionPoint(dialectType); if (extensionPoint == null) { return; @@ -111,7 +115,7 @@ public void sync(@NonNull Connection connection, @NonNull Database database, @No } @Override - public boolean supports(@NonNull DialectType dialectType) { + public boolean supports(@NonNull DialectType dialectType, @NonNull Connection connection) { return getExtensionPoint(dialectType) != null; } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/syncer/column/DBTableAndViewColumnSyncer.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/syncer/column/DBTableAndViewAndExternalTableColumnSyncer.java similarity index 93% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/syncer/column/DBTableAndViewColumnSyncer.java rename to server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/syncer/column/DBTableAndViewAndExternalTableColumnSyncer.java index 7b63d5fc72..2ad6deb0e2 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/syncer/column/DBTableAndViewColumnSyncer.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/syncer/column/DBTableAndViewAndExternalTableColumnSyncer.java @@ -36,7 +36,7 @@ * @date 2024/4/10 20:13 */ @Component -public class DBTableAndViewColumnSyncer extends AbstractDBColumnSyncer { +public class DBTableAndViewAndExternalTableColumnSyncer extends AbstractDBColumnSyncer { @Override Map> getLatestObjectToColumns(@NonNull ColumnExtensionPoint extensionPoint, @@ -48,7 +48,7 @@ Map> getLatestObjectToColumns(@NonNull ColumnExtensionPoint @Override public Collection getColumnRelatedObjectTypes() { - return Arrays.asList(DBObjectType.TABLE, DBObjectType.VIEW); + return Arrays.asList(DBObjectType.TABLE, DBObjectType.VIEW, DBObjectType.EXTERNAL_TABLE); } @Override diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/syncer/object/AbstractDBObjectSyncer.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/syncer/object/AbstractDBObjectSyncer.java index 1007fc328b..294ec0a31d 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/syncer/object/AbstractDBObjectSyncer.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/syncer/object/AbstractDBObjectSyncer.java @@ -30,6 +30,7 @@ import com.oceanbase.odc.metadb.dbobject.DBObjectEntity; import com.oceanbase.odc.metadb.dbobject.DBObjectRepository; import com.oceanbase.odc.service.connection.database.model.Database; +import com.oceanbase.odc.service.connection.database.model.DatabaseType; import com.oceanbase.odc.service.db.schema.syncer.DBSchemaSyncer; import com.oceanbase.odc.service.plugin.SchemaPluginUtil; @@ -51,6 +52,9 @@ public abstract class AbstractDBObjectSyncer implement @Override public void sync(@NonNull Connection connection, @NonNull Database database, @NonNull DialectType dialectType) { + if (database.getType() != DatabaseType.PHYSICAL) { + return; + } T extensionPoint = getExtensionPoint(dialectType); if (extensionPoint == null) { return; @@ -85,7 +89,7 @@ public void sync(@NonNull Connection connection, @NonNull Database database, @No } @Override - public boolean supports(@NonNull DialectType dialectType) { + public boolean supports(@NonNull DialectType dialectType, @NonNull Connection connection) { return getExtensionPoint(dialectType) != null; } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/syncer/object/DBExternalTableSyncer.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/syncer/object/DBExternalTableSyncer.java new file mode 100644 index 0000000000..fc2ca837ae --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/syncer/object/DBExternalTableSyncer.java @@ -0,0 +1,100 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.db.schema.syncer.object; + +import java.sql.Connection; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import com.oceanbase.odc.core.shared.constant.DialectType; +import com.oceanbase.odc.core.shared.constant.ResourceType; +import com.oceanbase.odc.metadb.iam.PermissionEntity; +import com.oceanbase.odc.metadb.iam.PermissionRepository; +import com.oceanbase.odc.metadb.iam.UserPermissionRepository; +import com.oceanbase.odc.plugin.connect.api.InformationExtensionPoint; +import com.oceanbase.odc.plugin.schema.api.TableExtensionPoint; +import com.oceanbase.odc.service.connection.database.model.Database; +import com.oceanbase.odc.service.feature.VersionDiffConfigService; +import com.oceanbase.odc.service.plugin.ConnectionPluginUtil; +import com.oceanbase.tools.dbbrowser.model.DBObjectIdentity; +import com.oceanbase.tools.dbbrowser.model.DBObjectType; + +import lombok.NonNull; +import lombok.extern.slf4j.Slf4j; + +/** + * @description: + * @author: zijia.cj + * @date: 2024/8/23 14:33 + * @since: 4.3.3 + */ +@Component +@Slf4j +public class DBExternalTableSyncer extends AbstractDBObjectSyncer { + + @Autowired + private PermissionRepository permissionRepository; + + @Autowired + private UserPermissionRepository userPermissionRepository; + + @Autowired + private VersionDiffConfigService versionDiffConfigService; + + @Override + protected void preDelete(@NonNull Set toBeDeletedIds) { + List permissions = + permissionRepository.findByResourceTypeAndResourceIdIn(ResourceType.ODC_TABLE, toBeDeletedIds); + Set permissionIds = permissions.stream().map(PermissionEntity::getId).collect(Collectors.toSet()); + permissionRepository.deleteByIds(permissionIds); + userPermissionRepository.deleteByPermissionIds(permissionIds); + } + + @Override + public boolean supports(@NonNull DialectType dialectType, @NonNull Connection connection) { + try { + InformationExtensionPoint point = + ConnectionPluginUtil.getInformationExtension(dialectType); + String databaseProductVersion = point.getDBVersion(connection); + return versionDiffConfigService.isExternalTableSupported(dialectType, databaseProductVersion) + && getExtensionPoint(dialectType) != null; + } catch (Exception e) { + log.warn("check external table support failed", e); + return false; + } + } + + @Override + public DBObjectType getObjectType() { + return DBObjectType.EXTERNAL_TABLE; + } + + @Override + Set getLatestObjectNames(@NonNull TableExtensionPoint extensionPoint, + @NonNull Connection connection, @NonNull Database database) { + return extensionPoint.list(connection, database.getName(), DBObjectType.EXTERNAL_TABLE).stream() + .map(DBObjectIdentity::getName).collect(Collectors.toSet()); + } + + @Override + Class getExtensionPointClass() { + return TableExtensionPoint.class; + } +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/syncer/object/DBTableSyncer.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/syncer/object/DBTableSyncer.java index 5e43b29870..bf2980f5e8 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/syncer/object/DBTableSyncer.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/syncer/object/DBTableSyncer.java @@ -59,7 +59,8 @@ protected void preDelete(@NonNull Set toBeDeletedIds) { @Override protected Set getLatestObjectNames(@NonNull TableExtensionPoint extensionPoint, @NonNull Connection connection, @NonNull Database database) { - return extensionPoint.list(connection, database.getName()).stream().map(DBObjectIdentity::getName) + return extensionPoint.list(connection, database.getName(), DBObjectType.TABLE).stream() + .map(DBObjectIdentity::getName) .collect(Collectors.toSet()); } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/session/DBSessionService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/session/DBSessionService.java index 8cb42bc495..9b0e4b25d2 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/session/DBSessionService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/session/DBSessionService.java @@ -15,10 +15,7 @@ */ package com.oceanbase.odc.service.db.session; -import static com.oceanbase.odc.service.db.session.KillSessionOrQueryReq.KILL_QUERY_TYPE; - import java.util.List; -import java.util.Map; import java.util.stream.Collectors; import javax.annotation.Nullable; @@ -26,22 +23,17 @@ import org.springframework.jdbc.core.JdbcOperations; import org.springframework.stereotype.Service; -import com.google.common.base.MoreObjects; import com.oceanbase.odc.common.util.ExceptionUtils; import com.oceanbase.odc.core.authority.util.SkipAuthorize; import com.oceanbase.odc.core.session.ConnectionSession; import com.oceanbase.odc.core.session.ConnectionSessionConstants; import com.oceanbase.odc.core.session.ConnectionSessionUtil; -import com.oceanbase.odc.core.shared.PreConditions; import com.oceanbase.odc.core.shared.model.OdcDBSession; import com.oceanbase.odc.core.sql.util.OdcDBSessionRowMapper; import com.oceanbase.odc.service.db.browser.DBStatsAccessors; import com.oceanbase.tools.dbbrowser.model.DBSession; import com.oceanbase.tools.dbbrowser.stats.DBStatsAccessor; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; @@ -73,36 +65,4 @@ public List list(@NonNull ConnectionSession session) { JdbcOperations jdbcOperations = session.getSyncJdbcExecutor(ConnectionSessionConstants.BACKEND_DS_KEY); return jdbcOperations.query("SHOW FULL PROCESSLIST", new OdcDBSessionRowMapper()); } - - public List getKillSql(@NonNull ConnectionSession session, @NonNull List sessionIds, - String closeType) { - List allSession = list(session); - Map sessionId2SvrpIp = - allSession.stream().collect( - Collectors.toMap(OdcDBSession::getSessionId, - s -> MoreObjects.firstNonNull(s.getSvrIp(), ""))); - return sessionIds.stream().map(sid -> { - PreConditions.notNegative(Long.parseLong(sid), "sessionId"); - StringBuilder sqlBuilder = new StringBuilder(); - sqlBuilder.append("kill "); - if (KILL_QUERY_TYPE.equalsIgnoreCase(closeType)) { - sqlBuilder.append("query "); - } - sqlBuilder.append(sid); - if (sessionId2SvrpIp.get(sid) != null) { - sqlBuilder.append(" /*").append(sessionId2SvrpIp.get(sid)).append("*/"); - } - return new SessionIdKillSql(sid, sqlBuilder.append(";").toString()); - }).collect(Collectors.toList()); - } - - - @AllArgsConstructor - @NoArgsConstructor - @Data - public static class SessionIdKillSql { - private String sessionId; - private String killSql; - } - } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/session/DefaultDBSessionManage.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/session/DefaultDBSessionManage.java index 1555b8bfa4..3cd1bd16b6 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/session/DefaultDBSessionManage.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/session/DefaultDBSessionManage.java @@ -18,8 +18,10 @@ import static com.oceanbase.odc.core.shared.constant.DialectType.OB_MYSQL; import java.sql.Connection; +import java.sql.SQLException; import java.sql.Statement; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; @@ -33,15 +35,17 @@ import java.util.concurrent.TimeoutException; import java.util.function.Predicate; import java.util.function.Supplier; -import java.util.regex.Matcher; -import java.util.regex.Pattern; import java.util.stream.Collectors; import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.collections4.SetUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; +import com.google.common.base.MoreObjects; import com.google.common.collect.Lists; +import com.oceanbase.odc.common.util.HostUtils; +import com.oceanbase.odc.common.util.HostUtils.ServerAddress; import com.oceanbase.odc.common.util.StringUtils; import com.oceanbase.odc.common.util.VersionUtils; import com.oceanbase.odc.core.authority.util.SkipAuthorize; @@ -55,22 +59,20 @@ import com.oceanbase.odc.core.shared.model.OdcDBSession; import com.oceanbase.odc.core.sql.execute.model.JdbcGeneralResult; import com.oceanbase.odc.core.sql.execute.model.SqlTuple; +import com.oceanbase.odc.plugin.connect.api.SessionExtensionPoint; import com.oceanbase.odc.service.connection.ConnectionService; import com.oceanbase.odc.service.connection.model.ConnectionConfig; -import com.oceanbase.odc.service.connection.util.ConnectionInfoUtil; import com.oceanbase.odc.service.connection.util.ConnectionMapper; import com.oceanbase.odc.service.db.browser.DBStatsAccessors; -import com.oceanbase.odc.service.db.session.DBSessionService.SessionIdKillSql; +import com.oceanbase.odc.service.plugin.ConnectionPluginUtil; import com.oceanbase.odc.service.session.DBSessionManageFacade; import com.oceanbase.odc.service.session.OdcStatementCallBack; import com.oceanbase.odc.service.session.factory.DefaultConnectSessionFactory; -import com.oceanbase.odc.service.session.factory.DruidDataSourceFactory; import com.oceanbase.odc.service.session.factory.OBConsoleDataSourceFactory; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; -import lombok.NonNull; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; @@ -78,16 +80,10 @@ @Slf4j public class DefaultDBSessionManage implements DBSessionManageFacade { - private static final String SERVER_REGEX = ".*/\\*(?([0-9]{1,3}.){1,3}([0-9]{1,3})):" - + "(?[0-9]{1,5})\\*/.*"; - private static final Pattern SERVER_PATTERN = Pattern.compile(SERVER_REGEX); private static final ConnectionMapper CONNECTION_MAPPER = ConnectionMapper.INSTANCE; private static final String GLOBAL_CLIENT_SESSION_OB_PROXY_VERSION_NUMBER = "4.2.3"; private static final String GLOBAL_CLIENT_SESSION_OB_VERSION_NUMBER = "4.2.5"; private static final String ORACLE_MODEL_KILL_SESSION_WITH_BLOCK_OB_VERSION_NUMBER = "4.2.1.0"; - private static final byte GLOBAL_CLIENT_SESSION_PROXY_ID_MIN = 0; - private static final short GLOBAL_CLIENT_SESSION_PROXY_ID_MAX = 8191; - private static final byte GLOBAL_CLIENT_SESSION_ID_VERSION = 2; @Autowired private DBSessionService dbSessionService; @@ -97,7 +93,7 @@ public class DefaultDBSessionManage implements DBSessionManageFacade { @Override @SkipAuthorize - public List killSessionOrQuery(KillSessionOrQueryReq request) { + public List killSessionOrQuery(KillSessionOrQueryReq request) { Verify.notNull(request.getSessionIds(), "session can not be null"); Verify.notNull(request.getDatasourceId(), "connection session id can not be null"); ConnectionConfig connectionConfig = connectionService.getForConnect(Long.valueOf(request.getDatasourceId())); @@ -105,9 +101,15 @@ public List killSessionOrQuery(KillSessionOrQueryReq request) ConnectionSession connectionSession = null; try { connectionSession = factory.generateSession(); - List session = - dbSessionService.getKillSql(connectionSession, request.getSessionIds(), request.getKillType()); - return doKillSessionOrQuery(connectionSession, session); + Map connectionId2KillSql; + SessionExtensionPoint sessionExtension = + ConnectionPluginUtil.getSessionExtension(connectionConfig.getDialectType()); + if (KillSessionOrQueryReq.KILL_QUERY_TYPE.equals(request.getKillType())) { + connectionId2KillSql = sessionExtension.getKillQuerySqls(new HashSet<>(request.getSessionIds())); + } else { + connectionId2KillSql = sessionExtension.getKillSessionSqls(new HashSet<>(request.getSessionIds())); + } + return doKill(connectionSession, connectionId2KillSql); } catch (Exception e) { log.info("kill session failed,datasourceId#{}", request.getDatasourceId(), e); throw e; @@ -118,8 +120,8 @@ public List killSessionOrQuery(KillSessionOrQueryReq request) } } - @SkipAuthorize @Override + @SkipAuthorize("odc internal usage") public boolean supportKillConsoleQuery(ConnectionSession session) { if (Objects.nonNull(session) && ConnectionSessionUtil.isLogicalSession(session)) { return false; @@ -135,19 +137,22 @@ public boolean killConsoleQuery(ConnectionSession session) { Verify.notNull(connectionId, "ConnectionId"); ConnectionConfig conn = (ConnectionConfig) ConnectionSessionUtil.getConnectionConfig(session); Verify.notNull(conn, "ConnectionConfig"); - DruidDataSourceFactory factory = new DruidDataSourceFactory(conn); + SessionExtensionPoint sessionExtension = + ConnectionPluginUtil.getSessionExtension(conn.getDialectType()); + DefaultConnectSessionFactory factory = new DefaultConnectSessionFactory(conn); + ConnectionSession copiedSession = null; try { - ConnectionInfoUtil.killQuery(connectionId, factory, session.getDialectType()); - } catch (Exception e) { - if (session.getDialectType().isOceanbase()) { - ConnectionSessionUtil.killQueryByDirectConnect(connectionId, factory); - log.info("Kill query by direct connect succeed, connectionId={}", connectionId); - } else { - log.warn("Kill query occur error, connectionId={}", connectionId, e); - return false; + copiedSession = factory.generateSession(); + Map connectionId2KillSql = sessionExtension.getKillQuerySqls( + SetUtils.hashSet(connectionId)); + List results = doKill(copiedSession, connectionId2KillSql); + Verify.singleton(results, "killResults"); + return results.get(0).isKilled(); + } finally { + if (copiedSession != null) { + copiedSession.expire(); } } - return true; } @Override @@ -165,12 +170,7 @@ public void killAllSessions(ConnectionSession connectionSession, Predicate doKillAllSessions( - list, - connectionSession, - executor), - lockTableTimeOutSeconds); + waitingForResult(() -> doKillAllSessions(list, connectionSession, executor), lockTableTimeOutSeconds); } finally { try { executor.shutdownNow(); @@ -178,57 +178,61 @@ public void killAllSessions(ConnectionSession connectionSession, Predicate checkedList = getSessionList(connectionSession, filter); - if (CollectionUtils.isNotEmpty(checkedList)) { - throw new IllegalStateException("kill session failed, has session reserved"); + long startTimeMs = System.currentTimeMillis(); + long endTimeMs = startTimeMs + 2000L; + while (System.currentTimeMillis() < endTimeMs) { + List checkedList = getSessionList(connectionSession, filter); + if (CollectionUtils.isEmpty(getSessionList(connectionSession, filter))) { + log.info("All sessions killed, elapsed time: {}ms", System.currentTimeMillis() - startTimeMs); + return; + } + log.info("Has sessions reserved:{}", checkedList); + try { + Thread.sleep(100); + } catch (InterruptedException e) { + log.warn("Thread sleep interrupted", e); + } } + throw new IllegalStateException("kill session failed, has session reserved"); } - private List getSessionList(ConnectionSession connectionSession, Predicate filter) { - return DBStatsAccessors.create(connectionSession) - .listAllSessions() - .stream() - .map(OdcDBSession::from) - .filter(filter == null ? a -> true : filter) - .collect(Collectors.toList()); - } + private List doKill(ConnectionSession session, Map connectionId2KillSqls) { + List sqlTupleSessionIds = connectionId2KillSqls.entrySet().stream().map(entry -> { + String connectionId = entry.getKey(); + String killSql = entry.getValue(); + return new SqlTupleSessionId(SqlTuple.newTuple(killSql), connectionId); + }).collect(Collectors.toList()); - private List doKillSessionOrQuery( - ConnectionSession connectionSession, List killSessionSqls) { - List sqlTupleSessionIds = killSessionSqls.stream().map( - s -> new SqlTupleSessionId(SqlTuple.newTuple(s.getKillSql()), s.getSessionId())) - .collect(Collectors.toList()); Map sqlId2SessionId = sqlTupleSessionIds.stream().collect( Collectors.toMap(s -> s.getSqlTuple().getSqlId(), SqlTupleSessionId::getSessionId)); - List sqlTuples = - sqlTupleSessionIds.stream().map(SqlTupleSessionId::getSqlTuple).collect(Collectors.toList()); - List jdbcGeneralResults = - executeKillSession(connectionSession, sqlTuples, sqlTuples.toString()); + List jdbcGeneralResults = executeSqls(session, sqlTupleSessionIds.stream() + .map(SqlTupleSessionId::getSqlTuple) + .collect(Collectors.toList())); + if (session.getDialectType().isOceanbase()) { + jdbcGeneralResults = additionalKillIfNecessary(session, jdbcGeneralResults, sqlTupleSessionIds); + } return jdbcGeneralResults.stream() - .map(res -> new KillSessionResult(res, sqlId2SessionId.get(res.getSqlTuple().getSqlId()))) + .map(res -> new KillResult(res, sqlId2SessionId.get(res.getSqlTuple().getSqlId()))) .collect(Collectors.toList()); } - @SkipAuthorize("odc internal usage") - public List executeKillSession(ConnectionSession connectionSession, List sqlTuples, - String sqlScript) { - List results = executeKillCommands(connectionSession, sqlTuples, sqlScript); - if (connectionSession.getDialectType() == DialectType.OB_MYSQL - || connectionSession.getDialectType() == DialectType.OB_ORACLE) { - return processResults(connectionSession, results); - } - return results; + // Will reuse the ConnectionSession to get the session list + private List getSessionList(ConnectionSession connectionSession, Predicate filter) { + return DBStatsAccessors.create(connectionSession) + .listAllSessions() + .stream() + .map(OdcDBSession::from) + .filter(filter == null ? a -> true : filter) + .collect(Collectors.toList()); } - private List executeKillCommands(ConnectionSession connectionSession, List sqlTuples, - String sqlScript) { + private List executeSqls(ConnectionSession connectionSession, List sqlTuples) { List results = connectionSession.getSyncJdbcExecutor(ConnectionSessionConstants.BACKEND_DS_KEY) .execute(new OdcStatementCallBack(sqlTuples, connectionSession, true, null, false)); if (results == null) { - log.warn("Execution of the kill session command failed with unknown error, sql={}", sqlScript); + log.warn("Execution of the kill session command failed with unknown error, sql={}", sqlTuples); throw new InternalServerError("Unknown error"); } return results; @@ -242,10 +246,20 @@ private List executeKillCommands(ConnectionSession connection * @param results * @return */ - private List processResults(ConnectionSession connectionSession, - List results) { + private List additionalKillIfNecessary(ConnectionSession connectionSession, + List results, List sqlTupleSessionIds) { + Map sessionId2SvrAddr = + getSessionList(connectionSession, s -> s.getSvrIp() != null) + .stream().collect(Collectors.toMap(OdcDBSession::getSessionId, + s -> HostUtils.extractServerAddress(MoreObjects.firstNonNull(s.getSvrIp(), "")))); + Map sqlId2SessionId = sqlTupleSessionIds.stream().collect( + Collectors.toMap(s -> s.getSqlTuple().getSqlId(), SqlTupleSessionId::getSessionId)); + Boolean isDirectedOBServer = isObServerDirected(connectionSession); - String obProxyVersion = getObProxyVersion(connectionSession, isDirectedOBServer); + String obProxyVersion = null; + if (Boolean.FALSE.equals(isDirectedOBServer)) { + obProxyVersion = ConnectionSessionUtil.getObProxyVersion(connectionSession); + } String obVersion = ConnectionSessionUtil.getVersion(connectionSession); boolean isEnabledGlobalClientSession = isGlobalClientSessionEnabled(connectionSession, obProxyVersion, obVersion); @@ -258,7 +272,9 @@ private List processResults(ConnectionSession connectionSessi if (isUnknownThreadIdError(e)) { jdbcGeneralResult = handleUnknownThreadIdError(connectionSession, jdbcGeneralResult, isDirectedOBServer, - isEnabledGlobalClientSession, isSupportedOracleModeKillSession); + isEnabledGlobalClientSession, isSupportedOracleModeKillSession, + sessionId2SvrAddr.getOrDefault( + sqlId2SessionId.get(jdbcGeneralResult.getSqlTuple().getSqlId()), null)); } else { log.warn("Failed to execute sql in kill session scenario, sqlTuple={}", jdbcGeneralResult.getSqlTuple(), e); @@ -293,27 +309,6 @@ private Boolean isObServerDirected(ConnectionSession connectionSession) { return null; } - /** - * Get the OBProxy version number. If an exception occurs or the version does not support, return - * null. - * - * @param connectionSession - * @param isDirectedOBServer - * @return - */ - private String getObProxyVersion(ConnectionSession connectionSession, Boolean isDirectedOBServer) { - if (Boolean.TRUE.equals(isDirectedOBServer)) { - return null; - } - try { - return connectionSession.getSyncJdbcExecutor(ConnectionSessionConstants.BACKEND_DS_KEY) - .queryForObject("select proxy_version()", String.class); - } catch (Exception e) { - log.warn("Failed to obtain the OBProxy version number: {}", e.getMessage()); - return null; - } - } - private boolean isGlobalClientSessionEnabled(ConnectionSession connectionSession, String obProxyVersion, String obVersion) { // verification version requirement @@ -323,38 +318,19 @@ private boolean isGlobalClientSessionEnabled(ConnectionSession connectionSession || VersionUtils.isLessThan(obVersion, GLOBAL_CLIENT_SESSION_OB_VERSION_NUMBER)) { return false; } - try { - Integer proxyId = getOBProxyConfig(connectionSession, "proxy_id"); - Integer clientSessionIdVersion = getOBProxyConfig(connectionSession, "client_session_id_version"); - - return proxyId != null - && proxyId >= GLOBAL_CLIENT_SESSION_PROXY_ID_MIN - && proxyId <= GLOBAL_CLIENT_SESSION_PROXY_ID_MAX - && clientSessionIdVersion != null - && clientSessionIdVersion == GLOBAL_CLIENT_SESSION_ID_VERSION; - } catch (Exception e) { - log.warn("Failed to determine if global client session is enabled: {}", e.getMessage()); - return false; - } - } - - /** - * Gets the value of OBProxy's configuration variable If an exception occurs or the version does not - * support, return null. - * - * @param connectionSession - * @param configName - * @return - */ - private Integer getOBProxyConfig(ConnectionSession connectionSession, String configName) { - try { - return connectionSession.getSyncJdbcExecutor(ConnectionSessionConstants.BACKEND_DS_KEY) - .query("show proxyconfig like '" + configName + "';", - rs -> rs.next() ? rs.getInt("value") : null); - } catch (Exception e) { - log.warn("Failed to obtain the value of OBProxy's configuration variable: {}", e.getMessage()); - return null; - } + // Check whether the global session is open + // If the global session is open, the "time" column will be displayed in the result set after + // executing the sql statement of "show processlist" + return connectionSession.getSyncJdbcExecutor(ConnectionSessionConstants.BACKEND_DS_KEY) + .query("show processlist", rs -> { + try { + int columnIndex = rs.findColumn("time"); + return columnIndex > 0; + } catch (SQLException e) { + log.warn("Failed to find the column 'time' in the result set", e); + return false; + } + }); } private boolean isUnknownThreadIdError(Exception e) { @@ -363,7 +339,8 @@ private boolean isUnknownThreadIdError(Exception e) { private JdbcGeneralResult handleUnknownThreadIdError(ConnectionSession connectionSession, JdbcGeneralResult jdbcGeneralResult, Boolean isDirectedOBServer, - boolean isEnabledGlobalClientSession, boolean isSupportedOracleModeKillSession) { + boolean isEnabledGlobalClientSession, boolean isSupportedOracleModeKillSession, + ServerAddress directServerAddress) { if (Boolean.TRUE.equals(isDirectedOBServer)) { log.info("The current connection mode is directing observer, return error result directly"); return jdbcGeneralResult; @@ -377,7 +354,7 @@ private JdbcGeneralResult handleUnknownThreadIdError(ConnectionSession connectio jdbcGeneralResult.getSqlTuple()); } return tryKillSessionViaDirectConnectObServer(connectionSession, jdbcGeneralResult, - jdbcGeneralResult.getSqlTuple()); + jdbcGeneralResult.getSqlTuple(), directServerAddress); } private boolean isOracleModeKillSessionSupported(String obVersion, ConnectionSession connectionSession) { @@ -429,10 +406,10 @@ private JdbcGeneralResult tryKillSessionByAnonymousBlock(ConnectionSession conne * @return */ private JdbcGeneralResult tryKillSessionViaDirectConnectObServer(ConnectionSession connectionSession, - JdbcGeneralResult jdbcGeneralResult, SqlTuple sqlTuple) { + JdbcGeneralResult jdbcGeneralResult, SqlTuple sqlTuple, ServerAddress serverAddress) { try { log.info("Kill query/session Unknown thread id error, try direct connect observer"); - directLinkServerAndExecute(sqlTuple.getExecutedSql(), connectionSession); + directLinkServerAndExecute(sqlTuple.getExecutedSql(), connectionSession, serverAddress); return JdbcGeneralResult.successResult(sqlTuple); } catch (Exception e) { log.warn("Failed to direct connect observer {}", e.getMessage()); @@ -440,9 +417,8 @@ private JdbcGeneralResult tryKillSessionViaDirectConnectObServer(ConnectionSessi } } - private void directLinkServerAndExecute(String sql, ConnectionSession session) + private void directLinkServerAndExecute(String sql, ConnectionSession session, ServerAddress serverAddress) throws Exception { - ServerAddress serverAddress = extractServerAddress(sql); if (Objects.isNull(serverAddress)) { throw new Exception("ServerAddress not found"); } @@ -462,46 +438,22 @@ private void directLinkServerAndExecute(String sql, ConnectionSession session) } } - ServerAddress extractServerAddress(String sql) { - String removeBlank = org.apache.commons.lang3.StringUtils.replace(sql, "\\s+", ""); - if (org.apache.commons.lang3.StringUtils.isBlank(removeBlank)) { - log.debug("unable to extract server address, sql was empty"); - return null; - } - int startPos = org.apache.commons.lang3.StringUtils.indexOf(removeBlank, "/*"); - if (-1 == startPos) { - log.debug("unable to extract server address, no comment found"); - return null; - } - String subStr = org.apache.commons.lang3.StringUtils.substring(removeBlank, startPos); - Matcher matcher = SERVER_PATTERN.matcher(subStr); - if (!matcher.matches()) { - log.info("unable to extract server address, does not match pattern"); - return null; - } - String ipAddress = matcher.group("ip"); - String port = matcher.group("port"); - if (org.apache.commons.lang3.StringUtils.isEmpty(ipAddress) - || org.apache.commons.lang3.StringUtils.isEmpty(port)) { - log.info("unable to extract server address, ipAddress={}, port={}", ipAddress, port); - return null; - } - return new ServerAddress(ipAddress, port); - } private CompletableFuture doKillAllSessions(List list, ConnectionSession connectionSession, Executor executor) { - return CompletableFuture.supplyAsync((Supplier) () -> { Lists.partition(list, 1024) .forEach(sessionList -> { - doKillSessionOrQuery(connectionSession, getKillSql(sessionList)); + SessionExtensionPoint sessionExtension = + ConnectionPluginUtil.getSessionExtension(connectionSession.getDialectType()); + Map connectionId2KillSql = sessionExtension.getKillSessionSqls( + sessionList.stream().map(OdcDBSession::getSessionId).collect(Collectors.toSet())); + doKill(connectionSession, connectionId2KillSql); }); return null; }, executor).exceptionally(ex -> { throw new CompletionException(ex); }); - } private void waitingForResult(Supplier> completableFutureSupplier, @@ -517,30 +469,6 @@ private void waitingForResult(Supplier> completableFutu } } - private List getKillSql(@NonNull List allSession) { - return allSession.stream() - .map(dbSession -> { - StringBuilder sqlBuilder = new StringBuilder(); - sqlBuilder.append("kill "); - sqlBuilder.append(dbSession.getSessionId()); - if (dbSession.getSvrIp() != null) { - sqlBuilder.append(" /*").append(dbSession.getSvrIp()).append("*/"); - } - return new SessionIdKillSql(dbSession.getSessionId(), sqlBuilder.append(";").toString()); - }).collect(Collectors.toList()); - } - - @Data - static class ServerAddress { - String ipAddress; - String port; - - public ServerAddress(String ipAddress, String port) { - this.ipAddress = ipAddress; - this.port = port; - } - } - @AllArgsConstructor @Data @NoArgsConstructor diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/session/KillSessionResult.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/session/KillResult.java similarity index 92% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/db/session/KillSessionResult.java rename to server/odc-service/src/main/java/com/oceanbase/odc/service/db/session/KillResult.java index 8c38cd9ec2..26bb2c966c 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/session/KillSessionResult.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/session/KillResult.java @@ -24,12 +24,12 @@ @Data @AllArgsConstructor -public class KillSessionResult { +public class KillResult { private String sessionId; private boolean killed; private String errorMessage; - public KillSessionResult(JdbcGeneralResult jdbcGeneralResult, String sessionId) { + public KillResult(JdbcGeneralResult jdbcGeneralResult, String sessionId) { this.sessionId = sessionId; this.killed = jdbcGeneralResult.getStatus().equals(SqlExecuteStatus.SUCCESS); if (!killed) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/util/OBMysqlCallFunctionCallBack.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/util/OBMysqlCallFunctionCallBack.java index 51219453ad..84914deed9 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/db/util/OBMysqlCallFunctionCallBack.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/db/util/OBMysqlCallFunctionCallBack.java @@ -24,12 +24,14 @@ import java.util.Objects; import java.util.stream.Collectors; +import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.Validate; import org.springframework.dao.DataAccessException; import org.springframework.jdbc.core.ConnectionCallback; import com.oceanbase.odc.common.util.StringUtils; -import com.oceanbase.odc.core.sql.util.JdbcDataTypeUtil; +import com.oceanbase.odc.core.sql.execute.mapper.JdbcRowMapper; +import com.oceanbase.odc.core.sql.execute.model.JdbcQueryResult; import com.oceanbase.odc.service.db.model.CallFunctionReq; import com.oceanbase.odc.service.db.model.CallFunctionResp; import com.oceanbase.odc.service.db.model.PLOutParam; @@ -39,21 +41,25 @@ import com.oceanbase.tools.dbbrowser.util.SqlBuilder; import lombok.NonNull; +import lombok.extern.slf4j.Slf4j; +@Slf4j public class OBMysqlCallFunctionCallBack implements ConnectionCallback { private final DBFunction function; private final int timeoutSeconds; + private final JdbcRowMapper rowDataMapper; - public OBMysqlCallFunctionCallBack(@NonNull CallFunctionReq callFunctionReq, int timeoutSeconds) { + public OBMysqlCallFunctionCallBack(@NonNull CallFunctionReq callFunctionReq, int timeoutSeconds, + @NonNull JdbcRowMapper rowDataMapper) { Validate.notBlank(callFunctionReq.getFunction().getFunName(), "Function name can not be blank"); this.function = callFunctionReq.getFunction(); this.timeoutSeconds = timeoutSeconds; + this.rowDataMapper = rowDataMapper; } @Override public CallFunctionResp doInConnection(Connection con) throws SQLException, DataAccessException { - CallFunctionResp callFunctionResp = new CallFunctionResp(); List params = new ArrayList<>(); if (function.getParams() != null) { params = function.getParams(); @@ -76,23 +82,38 @@ public CallFunctionResp doInConnection(Connection con) throws SQLException, Data if (this.timeoutSeconds > 0) { stmt.setQueryTimeout(this.timeoutSeconds); } - PLOutParam plOutParam = new PLOutParam(); try (ResultSet res = stmt.executeQuery(sqlBuilder.toString())) { if (!res.next()) { - plOutParam.setValue(function.getReturnValue()); + return generateDefaultReturnValue(); + } + JdbcQueryResult jdbcQueryResult = new JdbcQueryResult(res.getMetaData(), rowDataMapper); + jdbcQueryResult.addLine(res); + List> rows = jdbcQueryResult.getRows(); + if (CollectionUtils.size(rows) == 1 && CollectionUtils.size(rows.get(0)) == 1) { + CallFunctionResp callFunctionResp = new CallFunctionResp(); + PLOutParam plOutParam = new PLOutParam(); + plOutParam.setValue(String.valueOf(rows.get(0).get(0))); plOutParam.setDataType(function.getReturnType()); callFunctionResp.setReturnValue(plOutParam); + callFunctionResp.setOutParams(null); return callFunctionResp; } - Object value = JdbcDataTypeUtil.getValueFromResultSet( - res, 1, function.getReturnType()); - plOutParam.setValue(String.valueOf(value)); - plOutParam.setDataType(function.getReturnType()); - callFunctionResp.setReturnValue(plOutParam); + throw new IllegalStateException("The return value of a function must be unique"); } + } catch (Exception e) { + log.warn("Failed to call function {}", function.getFunName(), e); + CallFunctionResp callFunctionResp = generateDefaultReturnValue(); + callFunctionResp.setErrorMessage(e.getMessage()); + return callFunctionResp; } - callFunctionResp.setOutParams(null); - return callFunctionResp; } + private CallFunctionResp generateDefaultReturnValue() { + CallFunctionResp callFunctionResp = new CallFunctionResp(); + PLOutParam plOutParam = new PLOutParam(); + plOutParam.setValue(function.getReturnValue()); + plOutParam.setDataType(function.getReturnType()); + callFunctionResp.setReturnValue(plOutParam); + return callFunctionResp; + } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMConfiguration.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMConfiguration.java index 475793c5e3..c79c3dd3fe 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMConfiguration.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMConfiguration.java @@ -16,11 +16,9 @@ package com.oceanbase.odc.service.dlm; import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import com.oceanbase.tools.migrator.common.enums.ShardingStrategy; -import com.oceanbase.tools.migrator.core.IJobStore; import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -36,9 +34,6 @@ @Configuration public class DLMConfiguration { - @Value("${odc.task.dlm.thread-pool-size:15}") - private int dlmThreadPoolSize; - @Value("${odc.task.dlm.single-task-read-write-ratio:0.5}") private double readWriteRatio; @@ -54,9 +49,10 @@ public class DLMConfiguration { @Value("${odc.task.dlm.default-scan-batch-size:10000}") private int defaultScanBatchSize; - @Bean - public DLMJobFactory dlmJobFactory(IJobStore jobStore) { - return new DLMJobFactory(jobStore); - } + @Value("${odc.task.dlm.session-limiting.enabled:true}") + private boolean sessionLimitingEnabled; + + @Value("${odc.task.dlm.session-limiting-ratio:25}") + private int sessionLimitingRatio; } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMJobFactory.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMJobFactory.java index e11e83d500..2829ddb2ab 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMJobFactory.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMJobFactory.java @@ -18,9 +18,9 @@ import com.oceanbase.odc.service.dlm.model.DlmTableUnit; import com.oceanbase.odc.service.dlm.model.DlmTableUnitParameters; import com.oceanbase.tools.migrator.common.dto.HistoryJob; -import com.oceanbase.tools.migrator.core.IJobStore; import com.oceanbase.tools.migrator.core.JobFactory; import com.oceanbase.tools.migrator.core.JobReq; +import com.oceanbase.tools.migrator.core.store.IJobStore; import com.oceanbase.tools.migrator.job.Job; import lombok.extern.slf4j.Slf4j; @@ -41,7 +41,6 @@ public Job createJob(DlmTableUnit parameters) { HistoryJob historyJob = new HistoryJob(); historyJob.setId(parameters.getDlmTableUnitId()); historyJob.setJobType(parameters.getType()); - historyJob.setTableId(-1L); historyJob.setPrintSqlTrace(false); historyJob.setSourceTable(parameters.getTableName()); historyJob.setTargetTable(parameters.getTargetTableName()); @@ -56,6 +55,8 @@ public Job createJob(DlmTableUnit parameters) { req.setHistoryJob(historyJob); req.setSourceDs(parameters.getSourceDatasourceInfo()); req.setTargetDs(parameters.getTargetDatasourceInfo()); + req.setSourceLimitConfig(parameters.getSourceLimitConfig()); + req.setTargetLimitConfig(parameters.getTargetLimitConfig()); return super.createJob(req); } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMJobStore.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMJobStore.java index ba5d1a4be9..3d31cf1c77 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMJobStore.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMJobStore.java @@ -16,37 +16,24 @@ package com.oceanbase.odc.service.dlm; import java.sql.Connection; -import java.sql.Date; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; import java.util.LinkedList; import java.util.List; -import java.util.Map; import com.alibaba.druid.pool.DruidDataSource; -import com.oceanbase.odc.common.json.JsonUtils; -import com.oceanbase.odc.core.shared.constant.TaskStatus; import com.oceanbase.odc.service.connection.model.ConnectionConfig; import com.oceanbase.odc.service.dlm.model.DlmTableUnit; -import com.oceanbase.odc.service.dlm.model.RateLimitConfiguration; -import com.oceanbase.odc.service.schedule.job.DLMJobReq; -import com.oceanbase.odc.service.task.constants.JobParametersKeyConstants; +import com.oceanbase.odc.service.session.factory.DruidDataSourceFactory; import com.oceanbase.tools.migrator.common.dto.JobStatistic; -import com.oceanbase.tools.migrator.common.dto.TableSizeInfo; import com.oceanbase.tools.migrator.common.dto.TaskGenerator; import com.oceanbase.tools.migrator.common.element.PrimaryKey; -import com.oceanbase.tools.migrator.common.exception.JobException; -import com.oceanbase.tools.migrator.common.exception.JobSqlException; -import com.oceanbase.tools.migrator.common.meta.TableMeta; -import com.oceanbase.tools.migrator.core.IJobStore; import com.oceanbase.tools.migrator.core.handler.genarator.GeneratorStatus; -import com.oceanbase.tools.migrator.core.handler.genarator.GeneratorType; -import com.oceanbase.tools.migrator.core.meta.ClusterMeta; -import com.oceanbase.tools.migrator.core.meta.JobMeta; import com.oceanbase.tools.migrator.core.meta.TaskMeta; -import com.oceanbase.tools.migrator.core.meta.TenantMeta; +import com.oceanbase.tools.migrator.core.store.IJobStore; +import lombok.Setter; import lombok.extern.slf4j.Slf4j; /** @@ -58,18 +45,21 @@ public class DLMJobStore implements IJobStore { private DruidDataSource dataSource; - private boolean enableBreakpointRecovery = false; - private Map dlmTableUnits; - private Map jobParameters; + private boolean enableBreakpointRecovery = true; + @Setter + private DlmTableUnit dlmTableUnit; public DLMJobStore(ConnectionConfig metaDBConfig) { + try { + DruidDataSourceFactory druidDataSourceFactory = new DruidDataSourceFactory(metaDBConfig); + dataSource = (DruidDataSource) druidDataSourceFactory.getDataSource(); + } catch (Exception e) { + log.warn("Failed to connect to the meta database; closing save point."); + enableBreakpointRecovery = false; + } } - public void setDlmTableUnits(Map dlmTableUnits) { - this.dlmTableUnits = dlmTableUnits; - } - public void destroy() { if (dataSource == null) { return; @@ -82,7 +72,7 @@ public void destroy() { } @Override - public TaskGenerator getTaskGenerator(String generatorId, String jobId) throws SQLException { + public TaskGenerator getTaskGenerator(String jobId) throws SQLException { if (enableBreakpointRecovery) { try (Connection conn = dataSource.getConnection(); PreparedStatement ps = conn.prepareStatement( @@ -92,15 +82,14 @@ public TaskGenerator getTaskGenerator(String generatorId, String jobId) throws S if (resultSet.next()) { TaskGenerator taskGenerator = new TaskGenerator(); taskGenerator.setId(resultSet.getString("generator_id")); - taskGenerator.setGeneratorType(GeneratorType.valueOf(resultSet.getString("type"))); taskGenerator.setGeneratorStatus(GeneratorStatus.valueOf(resultSet.getString("status"))); taskGenerator.setJobId(jobId); taskGenerator.setTaskCount(resultSet.getInt("task_count")); taskGenerator - .setGeneratorSavePoint(PrimaryKey.valuesOf(resultSet.getString("primary_key_save_point"))); + .setPrimaryKeySavePoint(PrimaryKey.valuesOf(resultSet.getString("primary_key_save_point"))); taskGenerator.setProcessedDataSize(resultSet.getLong("processed_row_count")); taskGenerator.setProcessedDataSize(resultSet.getLong("processed_data_size")); - taskGenerator.setGeneratorPartitionSavepoint(resultSet.getString("partition_save_point")); + taskGenerator.setPartitionSavePoint(resultSet.getString("partition_save_point")); log.info("Load task generator success.jobId={}", jobId); return taskGenerator; } @@ -112,6 +101,16 @@ public TaskGenerator getTaskGenerator(String generatorId, String jobId) throws S @Override public void storeTaskGenerator(TaskGenerator taskGenerator) throws SQLException { + taskGenerator.getPartName2MaxKey() + .forEach((k, v) -> dlmTableUnit.getStatistic().getPartName2MaxKey().put(k, v.getSqlString())); + taskGenerator.getPartName2MinKey() + .forEach((k, v) -> dlmTableUnit.getStatistic().getPartName2MinKey().put(k, v.getSqlString())); + if (taskGenerator.getGlobalMaxKey() != null) { + dlmTableUnit.getStatistic().setGlobalMaxKey(taskGenerator.getGlobalMaxKey().getSqlString()); + } + if (taskGenerator.getGlobalMinKey() != null) { + dlmTableUnit.getStatistic().setGlobalMinKey(taskGenerator.getGlobalMinKey().getSqlString()); + } if (enableBreakpointRecovery) { StringBuilder sb = new StringBuilder(); sb.append("INSERT INTO dlm_task_generator "); @@ -130,11 +129,11 @@ public void storeTaskGenerator(TaskGenerator taskGenerator) throws SQLException ps.setLong(3, taskGenerator.getProcessedDataSize()); ps.setLong(4, taskGenerator.getProcessedRowCount()); ps.setString(5, taskGenerator.getGeneratorStatus().name()); - ps.setString(6, GeneratorType.AUTO.name()); + ps.setString(6, ""); ps.setLong(7, taskGenerator.getTaskCount()); - ps.setString(8, taskGenerator.getGeneratorSavePoint() == null ? "" - : taskGenerator.getGeneratorSavePoint().toSqlString()); - ps.setString(9, taskGenerator.getGeneratorPartitionSavepoint()); + ps.setString(8, taskGenerator.getPrimaryKeySavePoint() == null ? "" + : taskGenerator.getPrimaryKeySavePoint().toSqlString()); + ps.setString(9, taskGenerator.getPartitionSavePoint()); if (ps.executeUpdate() == 1) { log.info("Update task generator success.jobId={}", taskGenerator.getJobId()); } else { @@ -145,39 +144,34 @@ public void storeTaskGenerator(TaskGenerator taskGenerator) throws SQLException } @Override - public void bindGeneratorToJob(String s, TaskGenerator taskGenerator) throws SQLException { - - } - - @Override - public JobStatistic getJobStatistic(String s) throws JobException { + public JobStatistic getJobStatistic(String s) throws SQLException { return new JobStatistic(); } @Override - public void storeJobStatistic(JobMeta jobMeta) throws JobSqlException { - dlmTableUnits.get(jobMeta.getJobId()).getStatistic().setProcessedRowCount(jobMeta.getJobStat().getRowCount()); - dlmTableUnits.get(jobMeta.getJobId()).getStatistic() - .setProcessedRowsPerSecond(jobMeta.getJobStat().getAvgRowCount()); + public void storeJobStatistic(JobStatistic jobStatistic) throws SQLException { + dlmTableUnit.getStatistic() + .setProcessedRowCount(jobStatistic.getRowCount().get()); + dlmTableUnit.getStatistic() + .setProcessedRowsPerSecond(jobStatistic.getRowCountPerSeconds()); - dlmTableUnits.get(jobMeta.getJobId()).getStatistic().setReadRowCount(jobMeta.getJobStat().getReadRowCount()); - dlmTableUnits.get(jobMeta.getJobId()).getStatistic() - .setReadRowsPerSecond(jobMeta.getJobStat().getAvgReadRowCount()); + dlmTableUnit.getStatistic().setReadRowCount(jobStatistic.getReadRowCount().get()); + dlmTableUnit.getStatistic() + .setReadRowsPerSecond(jobStatistic.getReadRowCountPerSeconds()); } @Override - public List getTaskMeta(JobMeta jobMeta) throws SQLException { + public List loadUnfinishedTask(String generatorId) throws SQLException { if (enableBreakpointRecovery) { try (Connection conn = dataSource.getConnection(); PreparedStatement ps = conn.prepareStatement( "select * from dlm_task_unit where generator_id = ? AND status !='SUCCESS'")) { - ps.setString(1, jobMeta.getGenerator().getId()); + ps.setString(1, generatorId); ResultSet resultSet = ps.executeQuery(); List taskMetas = new LinkedList<>(); while (resultSet.next()) { TaskMeta taskMeta = new TaskMeta(); taskMeta.setTaskIndex(resultSet.getLong("task_index")); - taskMeta.setJobMeta(jobMeta); taskMeta.setGeneratorId(resultSet.getString("generator_id")); taskMeta.setTaskStatus(com.oceanbase.tools.migrator.common.enums.TaskStatus .valueOf(resultSet.getString("status"))); @@ -197,7 +191,6 @@ public List getTaskMeta(JobMeta jobMeta) throws SQLException { @Override public void storeTaskMeta(TaskMeta taskMeta) throws SQLException { if (enableBreakpointRecovery) { - log.info("start to store taskMeta:{}", taskMeta); StringBuilder sb = new StringBuilder(); sb.append("INSERT INTO dlm_task_unit "); sb.append( @@ -228,7 +221,8 @@ public void storeTaskMeta(TaskMeta taskMeta) throws SQLException { } @Override - public Long getAbnormalTaskIndex(String jobId) { + public long getAbnormalTaskCount(String jobId) { + long count = 0; if (enableBreakpointRecovery) { try (Connection conn = dataSource.getConnection(); PreparedStatement ps = conn.prepareStatement( @@ -236,78 +230,14 @@ public Long getAbnormalTaskIndex(String jobId) { ps.setString(1, jobId); ResultSet resultSet = ps.executeQuery(); if (resultSet.next()) { - long count = resultSet.getLong(1); - return count > 0 ? count : null; + count = resultSet.getLong(1); } } catch (Exception ignored) { log.warn("Get abnormal task failed.jobId={}", jobId); } } - return null; + return count; } - @Override - public void updateTableSizeInfo(TableSizeInfo tableSizeInfo, long l) { - - } - @Override - public void updateLimiter(JobMeta jobMeta) { - try { - RateLimitConfiguration params; - if (jobParameters.containsKey(JobParametersKeyConstants.DLM_RATE_LIMIT_CONFIG)) { - params = JsonUtils.fromJson( - jobParameters.get(JobParametersKeyConstants.DLM_RATE_LIMIT_CONFIG), - RateLimitConfiguration.class); - } else { - DLMJobReq dlmJobReq = JsonUtils.fromJson( - jobParameters.get(JobParametersKeyConstants.META_TASK_PARAMETER_JSON), - DLMJobReq.class); - params = dlmJobReq.getRateLimit(); - } - if (params.getDataSizeLimit() != null) { - setClusterLimitConfig(jobMeta.getSourceCluster(), params.getDataSizeLimit()); - setClusterLimitConfig(jobMeta.getTargetCluster(), params.getDataSizeLimit()); - setTenantLimitConfig(jobMeta.getSourceTenant(), params.getDataSizeLimit()); - setTenantLimitConfig(jobMeta.getTargetTenant(), params.getDataSizeLimit()); - log.info("Update rate limit success,dataSizeLimit={}", params.getDataSizeLimit()); - } - if (params.getRowLimit() != null) { - setTableLimitConfig(jobMeta.getTargetTableMeta(), params.getRowLimit()); - setTableLimitConfig(jobMeta.getSourceTableMeta(), params.getRowLimit()); - log.info("Update rate limit success,rowLimit={}", params.getRowLimit()); - } - } catch (Exception e) { - log.warn("Update rate limit failed,errorMsg={}", e.getMessage()); - setClusterLimitConfig(jobMeta.getSourceCluster(), 1024); - setClusterLimitConfig(jobMeta.getTargetCluster(), 1024); - setTenantLimitConfig(jobMeta.getSourceTenant(), 1024); - setTenantLimitConfig(jobMeta.getTargetTenant(), 1024); - setTableLimitConfig(jobMeta.getTargetTableMeta(), 1000); - setTableLimitConfig(jobMeta.getSourceTableMeta(), 1000); - } - } - - public void setJobParameters(Map jobParameters) { - this.jobParameters = jobParameters; - } - - private void setClusterLimitConfig(ClusterMeta clusterMeta, long dataSizeLimit) { - clusterMeta.setReadSizeLimit(dataSizeLimit); - clusterMeta.setWriteSizeLimit(dataSizeLimit); - clusterMeta.setWriteUsedQuota(1); - clusterMeta.setReadUsedQuota(1); - } - - private void setTenantLimitConfig(TenantMeta tenantMeta, long dataSizeLimit) { - tenantMeta.setReadSizeLimit(dataSizeLimit); - tenantMeta.setWriteSizeLimit(dataSizeLimit); - tenantMeta.setWriteUsedQuota(1); - tenantMeta.setReadUsedQuota(1); - } - - private void setTableLimitConfig(TableMeta tableMeta, int rowLimit) { - tableMeta.setReadRowCountLimit(rowLimit); - tableMeta.setWriteRowCountLimit(rowLimit); - } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMService.java index fe02913217..9862737d42 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMService.java @@ -97,6 +97,9 @@ public void createOrUpdateDlmTableUnits(List dlmTableUnits) { DlmTableUnitEntity entity; if (entityOptional.isPresent()) { entity = entityOptional.get(); + if (entity.getStatus() == TaskStatus.DONE) { + return; + } entity.setStatistic(JsonUtils.toJson(o.getStatistic())); entity.setStatus(o.getStatus()); entity.setStartTime(o.getStartTime()); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMTableStructureSynchronizer.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMTableStructureSynchronizer.java index eee02f6a3d..c772f46ac9 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMTableStructureSynchronizer.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DLMTableStructureSynchronizer.java @@ -94,11 +94,12 @@ public static void sync(ConnectionConfig srcConfig, ConnectionConfig tgtConfig, DBTableStructureComparator comparator = new DBTableStructureComparator(tgtTableEditor, tgtConfig.getType().getDialectType(), srcConfig.getDefaultSchema(), tgtConfig.getDefaultSchema()); List changeSqlScript = new LinkedList<>(); + targetType.remove(DBObjectType.TABLE); if (tgtTable == null) { srcTable.setSchemaName(tgtConfig.getDefaultSchema()); srcTable.setName(tgtTableName); changeSqlScript.add(tgtTableEditor.generateCreateObjectDDL(srcTable)); - } else { + } else if (!targetType.isEmpty()) { DBObjectComparisonResult result = comparator.compare(srcTable, tgtTable); if (result.getComparisonResult() == ComparisonResult.INCONSISTENT) { changeSqlScript = result.getSubDBObjectComparisonResult().stream() @@ -128,11 +129,10 @@ public static void sync(ConnectionConfig srcConfig, ConnectionConfig tgtConfig, public static boolean isSupportedSyncTableStructure(DialectType srcType, String srcVersion, DialectType tgtType, String tgtVersion) { - // only supports MySQL or OBMySQL - if (!srcType.isMysql() || !tgtType.isMysql()) { + if (srcType != tgtType) { return false; } - if (srcType != tgtType) { + if (!srcType.isOceanbase() && !srcType.isMysql()) { return false; } // unsupported MySQL versions below 5.7.0 diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DataArchiveJobStore.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DataArchiveJobStore.java deleted file mode 100644 index f373059cb0..0000000000 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DataArchiveJobStore.java +++ /dev/null @@ -1,214 +0,0 @@ -/* - * Copyright (c) 2023 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.oceanbase.odc.service.dlm; - -import java.util.List; -import java.util.Optional; -import java.util.stream.Collectors; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; - -import com.oceanbase.odc.common.json.JsonUtils; -import com.oceanbase.odc.metadb.dlm.DlmTableUnitRepository; -import com.oceanbase.odc.metadb.dlm.TaskGeneratorEntity; -import com.oceanbase.odc.metadb.dlm.TaskGeneratorRepository; -import com.oceanbase.odc.metadb.dlm.TaskUnitEntity; -import com.oceanbase.odc.metadb.dlm.TaskUnitRepository; -import com.oceanbase.odc.service.dlm.model.RateLimitConfiguration; -import com.oceanbase.odc.service.dlm.utils.DlmJobIdUtil; -import com.oceanbase.odc.service.dlm.utils.TaskGeneratorMapper; -import com.oceanbase.odc.service.dlm.utils.TaskUnitMapper; -import com.oceanbase.odc.service.schedule.model.DlmTableUnitStatistic; -import com.oceanbase.tools.migrator.common.dto.JobStatistic; -import com.oceanbase.tools.migrator.common.dto.TableSizeInfo; -import com.oceanbase.tools.migrator.common.dto.TaskGenerator; -import com.oceanbase.tools.migrator.common.meta.TableMeta; -import com.oceanbase.tools.migrator.core.IJobStore; -import com.oceanbase.tools.migrator.core.meta.ClusterMeta; -import com.oceanbase.tools.migrator.core.meta.JobMeta; -import com.oceanbase.tools.migrator.core.meta.TaskMeta; -import com.oceanbase.tools.migrator.core.meta.TenantMeta; - -import lombok.extern.slf4j.Slf4j; - -/** - * @Author：tinker - * @Date: 2023/5/8 19:27 - * @Descripition: TODO Store runtime data and use it to resume execution from a breakpoint. - */ -@Component -@Slf4j -public class DataArchiveJobStore implements IJobStore { - - @Value("${odc.task.dlm.support-breakpoint-recovery:false}") - private boolean supportBreakpointRecovery; - @Autowired - private DlmLimiterService limiterService; - @Autowired - private TaskGeneratorRepository taskGeneratorRepository; - @Autowired - private TaskUnitRepository taskUnitRepository; - @Autowired - private DlmTableUnitRepository dlmTableUnitRepository; - - private final TaskGeneratorMapper taskGeneratorMapper = TaskGeneratorMapper.INSTANCE; - private final TaskUnitMapper taskUnitMapper = TaskUnitMapper.INSTANCE; - - @Override - public TaskGenerator getTaskGenerator(String generatorId, String jobId) { - if (supportBreakpointRecovery) { - return taskGeneratorRepository.findByJobId(jobId).map(taskGeneratorMapper::entityToModel) - .orElse(null); - } - return null; - } - - @Override - public void storeTaskGenerator(TaskGenerator taskGenerator) { - if (supportBreakpointRecovery) { - Optional optional = taskGeneratorRepository.findByGeneratorId(taskGenerator.getId()); - TaskGeneratorEntity entity; - if (optional.isPresent()) { - entity = optional.get(); - entity.setStatus(taskGenerator.getGeneratorStatus().name()); - entity.setTaskCount(taskGenerator.getTaskCount()); - entity.setPartitionSavePoint(taskGenerator.getGeneratorPartitionSavepoint()); - entity.setProcessedRowCount(taskGenerator.getProcessedRowCount()); - entity.setProcessedDataSize(taskGenerator.getProcessedDataSize()); - if (taskGenerator.getGeneratorSavePoint() != null) { - entity.setPrimaryKeySavePoint(taskGenerator.getGeneratorSavePoint().toSqlString()); - } - } else { - entity = taskGeneratorMapper.modelToEntity(taskGenerator); - } - taskGeneratorRepository.save(entity); - } - } - - @Override - public void bindGeneratorToJob(String jobId, TaskGenerator taskGenerator) {} - - @Override - public JobStatistic getJobStatistic(String s) { - return new JobStatistic(); - } - - @Override - public void storeJobStatistic(JobMeta jobMeta) { - DlmTableUnitStatistic dlmExecutionDetail = new DlmTableUnitStatistic(); - dlmExecutionDetail.setProcessedRowCount(jobMeta.getJobStat().getRowCount()); - dlmExecutionDetail.setProcessedRowsPerSecond(jobMeta.getJobStat().getAvgRowCount()); - dlmExecutionDetail.setReadRowCount(jobMeta.getJobStat().getReadRowCount()); - dlmExecutionDetail.setReadRowsPerSecond(jobMeta.getJobStat().getAvgReadRowCount()); - dlmTableUnitRepository.updateStatisticByDlmTableUnitId(jobMeta.getJobId(), - JsonUtils.toJson(dlmExecutionDetail)); - } - - @Override - public List getTaskMeta(JobMeta jobMeta) { - if (supportBreakpointRecovery) { - List tasks = taskUnitRepository.findByGeneratorId(jobMeta.getGenerator().getId()).stream().map( - taskUnitMapper::entityToModel).collect( - Collectors.toList()); - tasks.forEach(o -> o.setJobMeta(jobMeta)); - return tasks; - } - return null; - } - - @Override - public void storeTaskMeta(TaskMeta taskMeta) { - if (supportBreakpointRecovery) { - Optional optional = taskUnitRepository.findByJobIdAndGeneratorIdAndTaskIndex( - taskMeta.getJobMeta().getJobId(), taskMeta.getGeneratorId(), taskMeta.getTaskIndex()); - TaskUnitEntity entity; - if (optional.isPresent()) { - entity = optional.get(); - entity.setStatus(taskMeta.getTaskStatus().name()); - entity.setPartitionName(taskMeta.getPartitionName()); - if (taskMeta.getMinPrimaryKey() != null) { - entity.setLowerBoundPrimaryKey(taskMeta.getMinPrimaryKey().toSqlString()); - } - if (taskMeta.getMaxPrimaryKey() != null) { - entity.setUpperBoundPrimaryKey(taskMeta.getMaxPrimaryKey().toSqlString()); - } - if (taskMeta.getCursorPrimaryKey() != null) { - entity.setPrimaryKeyCursor(taskMeta.getCursorPrimaryKey().toSqlString()); - } - } else { - entity = taskUnitMapper.modelToEntity(taskMeta); - } - taskUnitRepository.save(entity); - } - } - - @Override - public Long getAbnormalTaskIndex(String jobId) { - if (supportBreakpointRecovery) { - Long abnormalTaskCount = taskUnitRepository.findAbnormalTaskByJobId(jobId); - if (abnormalTaskCount != 0) { - return abnormalTaskCount; - } - } - return null; - } - - @Override - public void updateTableSizeInfo(TableSizeInfo tableSizeInfo, long l) { - - } - - @Override - public void updateLimiter(JobMeta jobMeta) { - RateLimitConfiguration rateLimit; - try { - rateLimit = limiterService - .getByOrderIdOrElseDefaultConfig(Long.parseLong(DlmJobIdUtil.getJobName(jobMeta.getJobId()))); - } catch (Exception e) { - log.warn("Update limiter failed,jobId={},error={}", - jobMeta.getJobId(), e); - return; - } - setClusterLimitConfig(jobMeta.getSourceCluster(), rateLimit.getDataSizeLimit()); - setClusterLimitConfig(jobMeta.getTargetCluster(), rateLimit.getDataSizeLimit()); - setTenantLimitConfig(jobMeta.getSourceTenant(), rateLimit.getDataSizeLimit()); - setTenantLimitConfig(jobMeta.getTargetTenant(), rateLimit.getDataSizeLimit()); - setTableLimitConfig(jobMeta.getSourceTableMeta(), rateLimit.getRowLimit()); - setTableLimitConfig(jobMeta.getTargetTableMeta(), rateLimit.getRowLimit()); - } - - private void setClusterLimitConfig(ClusterMeta clusterMeta, long dataSizeLimit) { - clusterMeta.setReadSizeLimit(dataSizeLimit); - clusterMeta.setWriteSizeLimit(dataSizeLimit); - clusterMeta.setWriteUsedQuota(1); - clusterMeta.setReadUsedQuota(1); - } - - private void setTenantLimitConfig(TenantMeta tenantMeta, long dataSizeLimit) { - tenantMeta.setReadSizeLimit(dataSizeLimit); - tenantMeta.setWriteSizeLimit(dataSizeLimit); - tenantMeta.setWriteUsedQuota(1); - tenantMeta.setReadUsedQuota(1); - } - - private void setTableLimitConfig(TableMeta tableMeta, int rowLimit) { - tableMeta.setReadRowCountLimit(rowLimit); - tableMeta.setWriteRowCountLimit(rowLimit); - } - -} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DataSourceInfoMapper.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DataSourceInfoMapper.java index 56e8a383be..e33817c7f0 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DataSourceInfoMapper.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/DataSourceInfoMapper.java @@ -27,7 +27,8 @@ import com.oceanbase.odc.service.plugin.ConnectionPluginUtil; import com.oceanbase.odc.service.session.factory.OBConsoleDataSourceFactory; import com.oceanbase.tools.migrator.common.configure.DataSourceInfo; -import com.oceanbase.tools.migrator.common.enums.DataBaseType; +import com.oceanbase.tools.migrator.common.enums.DatasourceType; +import com.oceanbase.tools.migrator.datasource.fs.FileFormat; import lombok.extern.slf4j.Slf4j; @@ -43,29 +44,14 @@ public static ConnectionConfig toConnectionConfig(DataSourceInfo dataSourceInfo) ConnectionConfig connectionConfig = new ConnectionConfig(); connectionConfig.setDefaultSchema(dataSourceInfo.getDatabaseName()); connectionConfig.setPassword(dataSourceInfo.getPassword()); - connectionConfig.setHost(dataSourceInfo.getIp()); + connectionConfig.setHost(dataSourceInfo.getHost()); connectionConfig.setPort(dataSourceInfo.getPort()); - connectionConfig.setUsername(dataSourceInfo.getFullUserName()); - connectionConfig.setType(ConnectType.valueOf(dataSourceInfo.getDatabaseType().name())); - // convert full username to native user name - if (dataSourceInfo.getDatabaseType() == DataBaseType.OB_ORACLE) { - String userName = connectionConfig.getUsername(); - if (userName.contains("#")) { - userName = userName.split("#")[0]; - } - if (userName.contains("@")) { - userName = userName.split("@")[0]; - } - if (userName.contains("\"")) { - userName = userName.replace("\"", ""); - } - connectionConfig.setUsername(userName); - connectionConfig.setTenantName(dataSourceInfo.getTenantName()); - connectionConfig.setClusterName(dataSourceInfo.getClusterName()); - } + connectionConfig.setUsername(dataSourceInfo.getUsername()); + connectionConfig.setType(ConnectType.valueOf(dataSourceInfo.getType().name())); return connectionConfig; } + public static DataSourceInfo toDataSourceInfo(ConnectionConfig connectionConfig, String schemaName) { DataSourceInfo dataSourceInfo = new DataSourceInfo(); dataSourceInfo.setDatabaseName(connectionConfig.getDefaultSchema()); @@ -73,40 +59,45 @@ public static DataSourceInfo toDataSourceInfo(ConnectionConfig connectionConfig, if (StringUtils.isNotEmpty(connectionConfig.getPassword())) { dataSourceInfo.setPassword(connectionConfig.getPassword()); } - dataSourceInfo.setIp(connectionConfig.getHost()); + dataSourceInfo.setHost(connectionConfig.getHost()); dataSourceInfo.setPort(connectionConfig.getPort()); switch (connectionConfig.getDialectType()) { case DORIS: case MYSQL: { - dataSourceInfo.setFullUserName(connectionConfig.getUsername()); - dataSourceInfo.setDatabaseType(DataBaseType.MYSQL); + dataSourceInfo.setUsername(connectionConfig.getUsername()); + dataSourceInfo.setType(DatasourceType.MYSQL); break; } case OB_MYSQL: { dataSourceInfo - .setFullUserName(OBConsoleDataSourceFactory.getUsername(connectionConfig)); - dataSourceInfo.setDatabaseType(DataBaseType.OB_MYSQL); - dataSourceInfo.setClusterName(connectionConfig.getClusterName()); - dataSourceInfo.setSysDatabaseName("oceanbase"); + .setUsername(OBConsoleDataSourceFactory.getUsername(connectionConfig)); + dataSourceInfo.setType(DatasourceType.OB_MYSQL); break; } case OB_ORACLE: - dataSourceInfo.setFullUserName(OBConsoleDataSourceFactory.getUsername(connectionConfig)); - dataSourceInfo.setClusterName(connectionConfig.getClusterName()); - dataSourceInfo.setTenantName(connectionConfig.getTenantName()); - dataSourceInfo.setDatabaseType(DataBaseType.OB_ORACLE); + dataSourceInfo.setUsername(OBConsoleDataSourceFactory.getUsername(connectionConfig)); + dataSourceInfo.setType(DatasourceType.OB_ORACLE); break; case POSTGRESQL: - dataSourceInfo.setFullUserName(connectionConfig.getUsername()); + dataSourceInfo.setUsername(connectionConfig.getUsername()); connectionConfig.setDefaultSchema(schemaName); String jdbcUrl = getJdbcUrl(connectionConfig) + "&stringtype=unspecified"; - dataSourceInfo.setJdbcUrl(jdbcUrl); - dataSourceInfo.setDatabaseType(DataBaseType.POSTGRESQL); + dataSourceInfo.setUrl(jdbcUrl); + dataSourceInfo.setType(DatasourceType.POSTGRESQL); break; case ORACLE: - dataSourceInfo.setJdbcUrl(getJdbcUrl(connectionConfig)); - dataSourceInfo.setDatabaseType(DataBaseType.ORACLE); - dataSourceInfo.setFullUserName(getOracleUsername(connectionConfig)); + dataSourceInfo.setUrl(getJdbcUrl(connectionConfig)); + dataSourceInfo.setType(DatasourceType.ORACLE); + dataSourceInfo.setUsername(getOracleUsername(connectionConfig)); + break; + case FILE_SYSTEM: + dataSourceInfo.setHost(connectionConfig.getHost()); + dataSourceInfo.setType(DatasourceType.valueOf(connectionConfig.getType().name())); + dataSourceInfo.setUsername(connectionConfig.getUsername()); + dataSourceInfo.setPassword(connectionConfig.getPassword()); + dataSourceInfo.setFileFormat(FileFormat.CSV); + dataSourceInfo.setRegion(connectionConfig.getRegion()); + dataSourceInfo.setDefaultCharset("UTF-8"); break; default: log.warn(String.format("Unsupported datasource type:%s", connectionConfig.getDialectType())); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/model/DataArchiveTableConfig.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/model/DataArchiveTableConfig.java index b852f42cdb..06103cadc9 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/model/DataArchiveTableConfig.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/model/DataArchiveTableConfig.java @@ -15,8 +15,10 @@ */ package com.oceanbase.odc.service.dlm.model; +import java.util.HashMap; import java.util.LinkedList; import java.util.List; +import java.util.Map; import com.oceanbase.odc.common.util.StringUtils; @@ -40,6 +42,14 @@ public class DataArchiveTableConfig { // the sql condition such as "gmt_create < '2023-01-01'" private String conditionExpression; + private String minKey; + + private String maxKey; + + private Map partName2MinKey = new HashMap<>(); + + private Map partName2MaxKey = new HashMap<>(); + public String getTargetTableName() { return StringUtils.isEmpty(targetTableName) ? tableName : targetTableName; } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/model/DlmTableUnit.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/model/DlmTableUnit.java index 2370ed086d..03a0b03183 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/model/DlmTableUnit.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/model/DlmTableUnit.java @@ -16,11 +16,14 @@ package com.oceanbase.odc.service.dlm.model; import java.util.Date; +import java.util.Set; import com.oceanbase.odc.core.shared.constant.TaskStatus; import com.oceanbase.odc.service.schedule.model.DlmTableUnitStatistic; +import com.oceanbase.tools.dbbrowser.model.DBObjectType; import com.oceanbase.tools.migrator.common.configure.DataSourceInfo; import com.oceanbase.tools.migrator.common.enums.JobType; +import com.oceanbase.tools.migrator.limiter.LimiterConfig; import lombok.Data; @@ -48,6 +51,10 @@ public class DlmTableUnit { private DataSourceInfo targetDatasourceInfo; + private LimiterConfig sourceLimitConfig; + + private LimiterConfig targetLimitConfig; + private DlmTableUnitStatistic statistic; private DlmTableUnitParameters parameters; @@ -60,4 +67,6 @@ public class DlmTableUnit { private Date endTime; + private Set syncTableStructure; + } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/utils/DataArchiveConditionUtil.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/utils/DataArchiveConditionUtil.java index 1ee1f976b9..02babab202 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/utils/DataArchiveConditionUtil.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/utils/DataArchiveConditionUtil.java @@ -18,7 +18,6 @@ import java.time.LocalDateTime; import java.time.ZoneId; import java.time.format.DateTimeFormatter; -import java.time.temporal.ChronoUnit; import java.util.Collections; import java.util.Date; import java.util.HashMap; @@ -30,6 +29,7 @@ import org.apache.commons.collections4.CollectionUtils; import com.oceanbase.odc.common.util.StringUtils; +import com.oceanbase.odc.core.shared.exception.UnsupportedException; import com.oceanbase.odc.service.dlm.model.OffsetConfig; import com.oceanbase.odc.service.dlm.model.Operator; @@ -82,21 +82,18 @@ private static String calculateDateTime(Date baseDate, OffsetConfig config) { if (StringUtils.isNotEmpty(config.getPattern())) { String[] parts = config.getPattern().split("\\|"); String offsetString = parts[1].substring(1); - ChronoUnit unit = parseUnit(offsetString.substring(offsetString.length() - 1)); - long offsetSeconds = parseValue(offsetString) * unit.getDuration().getSeconds(); + long offsetValue = parseValue(offsetString); if (parts[1].startsWith("-")) { - offsetSeconds = -offsetSeconds; + offsetValue = -offsetValue; } - localDateTime = baseDate.toInstant().atZone(ZoneId.systemDefault()).toLocalDateTime() - .plusSeconds(offsetSeconds); + localDateTime = calculateDateTime(baseDate, offsetValue, offsetString.substring(offsetString.length() - 1)); return localDateTime.format(DateTimeFormatter.ofPattern(parts[0])); } else { - long offsetSeconds = config.getValue() * parseUnit(config.getUnit()).getDuration().getSeconds(); + long offsetValue = config.getValue(); if (config.getOperator() == Operator.MINUS) { - offsetSeconds = -offsetSeconds; + offsetValue = -offsetValue; } - localDateTime = baseDate.toInstant().atZone(ZoneId.systemDefault()).toLocalDateTime() - .plusSeconds(offsetSeconds); + localDateTime = calculateDateTime(baseDate, offsetValue, config.getUnit()); return localDateTime.format(DateTimeFormatter.ofPattern(config.getDateFormatPattern())); } @@ -106,24 +103,18 @@ private static int parseValue(String offsetString) { return Integer.parseInt(offsetString.substring(0, offsetString.length() - 1)); } - private static ChronoUnit parseUnit(String unit) { + private static LocalDateTime calculateDateTime(Date baseDate, long offsetValue, String unit) { switch (unit) { case "y": - return ChronoUnit.YEARS; + return baseDate.toInstant().atZone(ZoneId.systemDefault()).toLocalDateTime().plusYears(offsetValue); case "M": - return ChronoUnit.MONTHS; + return baseDate.toInstant().atZone(ZoneId.systemDefault()).toLocalDateTime().plusMonths(offsetValue); case "d": - return ChronoUnit.DAYS; - case "h": - return ChronoUnit.HOURS; - case "m": - return ChronoUnit.MINUTES; - case "s": - return ChronoUnit.SECONDS; + return baseDate.toInstant().atZone(ZoneId.systemDefault()).toLocalDateTime().plusDays(offsetValue); case "w": - return ChronoUnit.WEEKS; + return baseDate.toInstant().atZone(ZoneId.systemDefault()).toLocalDateTime().plusWeeks(offsetValue); default: - throw new IllegalArgumentException("Unknown unit: " + unit); + throw new UnsupportedException("Unsupported unit: " + unit); } } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/utils/TaskGeneratorMapper.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/utils/TaskGeneratorMapper.java index 9c61909e5a..8e124d27e7 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/utils/TaskGeneratorMapper.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/dlm/utils/TaskGeneratorMapper.java @@ -36,16 +36,15 @@ public interface TaskGeneratorMapper { @Mapping(source = "generatorId", target = "id") @Mapping(source = "status", target = "generatorStatus") - @Mapping(target = "generatorType", constant = "AUTO") - @Mapping(source = "partitionSavePoint", target = "generatorPartitionSavepoint") - @Mapping(target = "generatorSavePoint", + @Mapping(source = "partitionSavePoint", target = "partitionSavePoint") + @Mapping(target = "primaryKeySavePoint", expression = "java(com.oceanbase.tools.migrator.common.element.PrimaryKey.valuesOf(entity.getPrimaryKeySavePoint()))") TaskGenerator entityToModel(TaskGeneratorEntity entity); @InheritInverseConfiguration @Mapping(target = "type", constant = "AUTO") @Mapping(target = "primaryKeySavePoint", - expression = "java(model.getGeneratorSavePoint() != null ?model.getGeneratorSavePoint().toSqlString():null)") + expression = "java(model.getPrimaryKeySavePoint() != null ?model.getPrimaryKeySavePoint().toSqlString():null)") @Mapping(target = "id", ignore = true) TaskGeneratorEntity modelToEntity(TaskGenerator model); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/feature/VersionDiffConfigService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/feature/VersionDiffConfigService.java index 180fd20975..c3947af58a 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/feature/VersionDiffConfigService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/feature/VersionDiffConfigService.java @@ -19,6 +19,7 @@ import java.util.List; import java.util.Optional; +import org.apache.commons.collections4.CollectionUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.jdbc.core.JdbcOperations; import org.springframework.stereotype.Service; @@ -53,6 +54,7 @@ public class VersionDiffConfigService { private static final String SUPPORT_KILL_SESSION = "support_kill_session"; private static final String SUPPORT_KILL_QUERY = "support_kill_query"; private static final String SUPPORT_PL_DEBUG = "support_pl_debug"; + private static final String SUPPORT_EXTERNAL_TABLE = "support_external_table"; private static final String COLUMN_DATA_TYPE = "column_data_type"; private static final String ARM_OB_PREFIX = "aarch64"; private static final String ARM_OB_SUPPORT_PL_DEBUG_MIN_VERSION = "3.2.3"; @@ -134,8 +136,10 @@ public List getSupportFeatures(ConnectionSession connectionSession) { } // killSession that is greater than the specified version is currently not supported - if (SUPPORT_KILL_SESSION.equalsIgnoreCase(configKey) - || SUPPORT_KILL_QUERY.equalsIgnoreCase(configKey)) { + // Only effect OB mode dialect + if (connectionSession.getDialectType().isOceanbase() && + (SUPPORT_KILL_SESSION.equalsIgnoreCase(configKey) + || SUPPORT_KILL_QUERY.equalsIgnoreCase(configKey))) { Optional nonSupport = systemConfigs.stream().filter( c -> c.getKey().equalsIgnoreCase(MAX_SUPPORT_KILL_OB_VERSION)).findFirst(); if (nonSupport.isPresent()) { @@ -156,6 +160,20 @@ public List getSupportFeatures(ConnectionSession connectionSession) { return obSupportList; } + public boolean isExternalTableSupported(@NonNull DialectType dialectType, @NonNull String versionNumber) { + VersionDiffConfig config = new VersionDiffConfig(); + config.setDbMode(dialectType.name()); + config.setConfigKey(SUPPORT_EXTERNAL_TABLE); + List list = versionDiffConfigDAO.query(config); + String minVersion = CollectionUtils.isNotEmpty(list) ? list.get(0).getMinVersion() : null; + if ((dialectType == DialectType.OB_MYSQL || dialectType == DialectType.OB_ORACLE) + && minVersion != null + && VersionUtils.isGreaterThanOrEqualsTo(versionNumber, minVersion)) { + return true; + } + return false; + } + private boolean isHourFormat(ConnectionSession connectionSession) { JdbcOperations jdbcOperations = connectionSession.getSyncJdbcExecutor(ConnectionSessionConstants.CONSOLE_DS_KEY); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/ApprovalPermissionService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/ApprovalPermissionService.java index 070c3bb365..b173078eed 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/ApprovalPermissionService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/ApprovalPermissionService.java @@ -39,12 +39,12 @@ import com.oceanbase.odc.metadb.iam.UserEntity; import com.oceanbase.odc.metadb.iam.UserRepository; import com.oceanbase.odc.metadb.iam.UserRoleRepository; -import com.oceanbase.odc.metadb.iam.resourcerole.UserResourceRoleEntity; import com.oceanbase.odc.metadb.iam.resourcerole.UserResourceRoleRepository; import com.oceanbase.odc.service.flow.model.FlowNodeStatus; import com.oceanbase.odc.service.iam.ResourceRoleService; import com.oceanbase.odc.service.iam.UserService; import com.oceanbase.odc.service.iam.auth.AuthenticationFacade; +import com.oceanbase.odc.service.iam.model.UserResourceRole; import lombok.NonNull; @@ -163,12 +163,12 @@ private Map> getUsersByFlowInstanceIdsAndStatus(@NonNull C if (resourceRoleIdentifiers.isEmpty()) { return new HashMap<>(); } - Map> identifier2UserIds = userResourceRoleRepository.findByResourceIdsAndResourceRoleIdsIn( - resourceRoleIdentifiers).stream() + Map> identifier2UserIds = resourceRoleService + .listByResourceIdentifierIn(resourceRoleIdentifiers).stream() .collect(Collectors.groupingBy(o -> String.format("%s:%s", o.getResourceId(), o.getResourceRoleId()))) .entrySet() .stream().collect(Collectors.toMap(entry -> entry.getKey(), - entry -> entry.getValue().stream().map(UserResourceRoleEntity::getUserId).collect( + entry -> entry.getValue().stream().map(UserResourceRole::getUserId).collect( Collectors.toSet()))); // map approval instance ids to users ids Map> instanceId2UserIds = diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowInstanceService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowInstanceService.java index 8b595d28be..3fa3f737da 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowInstanceService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowInstanceService.java @@ -262,6 +262,8 @@ public class FlowInstanceService { @Autowired private EnvironmentService environmentService; @Autowired + private FlowPermissionHelper flowPermissionHelper; + @Autowired private MeterManager meterManager; private static final long MAX_EXPORT_OBJECT_COUNT = 10000; @@ -412,9 +414,22 @@ public FlowMetaInfo getMetaInfo() { return FlowMetaInfo.of(entities); } - public Page listAll(@NotNull Pageable pageable, @NotNull QueryFlowInstanceParams params) { - if (Objects.nonNull(params.getProjectId())) { - projectPermissionValidator.checkProjectRole(params.getProjectId(), ResourceRoleName.all()); + public Page listUnfinishedFlowInstances(@NotNull Pageable pageable, + @NonNull Long projectId) { + QueryFlowInstanceParams.builder().projectIds(Collections.singleton(projectId)).containsAll(true).statuses( + Arrays.asList(FlowStatus.APPROVING, FlowStatus.CREATED, FlowStatus.EXECUTING, FlowStatus.ROLLBACKING, + FlowStatus.WAIT_FOR_EXECUTION, FlowStatus.WAIT_FOR_CONFIRM)) + .build(); + return list(pageable, + QueryFlowInstanceParams.builder().projectIds(Collections.singleton(projectId)).containsAll(true) + .statuses( + FlowStatus.listUnfinishedStatus()) + .build()); + } + + private Page listAll(@NotNull Pageable pageable, @NotNull QueryFlowInstanceParams params) { + if (Objects.nonNull(params.getProjectIds())) { + projectPermissionValidator.checkProjectRole(params.getProjectIds(), ResourceRoleName.all()); } if (params.getParentInstanceId() != null) { // TODO 4.1.3 自动运行模块改造完成后剥离 @@ -438,9 +453,8 @@ public Page listAll(@NotNull Pageable pageable, @NotNull Que Specification specification = Specification.where(FlowInstanceSpecs.idIn(flowInstanceIds)) .and(FlowInstanceSpecs.organizationIdEquals(authenticationFacade.currentOrganizationId())); - // TODO Remove the checker after the SQL console development is completed - if (params.getProjectId() != null) { - specification = specification.and(FlowInstanceSpecs.projectIdEquals(params.getProjectId())); + if (CollectionUtils.isNotEmpty(params.getProjectIds())) { + specification = specification.and(FlowInstanceSpecs.projectIdIn(params.getProjectIds())); } return flowInstanceRepository.findAll(specification, pageable); } @@ -488,75 +502,36 @@ public Page listAll(@NotNull Pageable pageable, @NotNull Que specification = specification.and(FlowInstanceViewSpecs.taskTypeIn(types)); } - Set resourceRoleIdentifiers = userService.getCurrentUserResourceRoleIdentifiers(); - if (params.getContainsAll()) { - // does not join any project - if (CollectionUtils.isEmpty(resourceRoleIdentifiers)) { - specification = - specification.and(FlowInstanceViewSpecs.creatorIdEquals(authenticationFacade.currentUserId())); - return flowInstanceViewRepository.findAll(specification, pageable).map(FlowInstanceEntity::from); - } - // find by project id - if (Objects.nonNull(params.getProjectId())) { - specification = specification.and(FlowInstanceViewSpecs.projectIdEquals(params.getProjectId())); - // if other project roles, show current user's created, waiting for approval and approved/rejected - // tickets - if (!projectPermissionValidator.hasProjectRole(params.getProjectId(), - Collections.singletonList(ResourceRoleName.OWNER))) { - specification = specification.and(FlowInstanceViewSpecs.leftJoinFlowInstanceApprovalView( - resourceRoleIdentifiers, authenticationFacade.currentUserId(), - FlowNodeStatus.getExecutingAndFinalStatuses())); - } - // if project owner, show all tickets of the project - } else { - // find tickets related to all projects that the current user joins in - Map> currentUserProjectId2ResourceRoleNames = - resourceRoleService.getProjectId2ResourceRoleNames(); - Set ownerProjectIds = currentUserProjectId2ResourceRoleNames.entrySet().stream() - .filter(entry -> entry.getValue().contains(ResourceRoleName.OWNER)) - .map(Entry::getKey) - .collect(Collectors.toSet()); - Set otherRoleProjectIds = new HashSet<>(currentUserProjectId2ResourceRoleNames.keySet()); - otherRoleProjectIds.removeAll(ownerProjectIds); - - - Specification ownerSpecification = - Specification.where(FlowInstanceViewSpecs.projectIdIn(ownerProjectIds)); - - Specification otherRoleSpecification = - Specification.where(FlowInstanceViewSpecs.projectIdIn(otherRoleProjectIds)) - .and(FlowInstanceViewSpecs.leftJoinFlowInstanceApprovalView( - resourceRoleIdentifiers, authenticationFacade.currentUserId(), - FlowNodeStatus.getExecutingAndFinalStatuses())); - - if (CollectionUtils.isEmpty(ownerProjectIds)) { - specification = specification.and(otherRoleSpecification); - } else if (CollectionUtils.isEmpty(otherRoleProjectIds)) { - specification = specification.and(ownerSpecification); - } else { - specification = specification.and(ownerSpecification.or(otherRoleSpecification)); - } + if (CollectionUtils.isNotEmpty(params.getProjectIds())) { + specification = specification.and(FlowInstanceViewSpecs.projectIdIn(params.getProjectIds())); + } else { + Set joinedProjectIds = userService.getCurrentUserJoinedProjectIds(); + // if the user does not join any projects in team space, then return empty directly + if (CollectionUtils.isEmpty(joinedProjectIds) + && authenticationFacade.currentOrganization().getType() == OrganizationType.TEAM) { + return Page.empty(); } + specification = + specification.and(FlowInstanceViewSpecs.projectIdIn(userService.getCurrentUserJoinedProjectIds())); + } + if (params.getContainsAll()) { return flowInstanceViewRepository.findAll(specification, pageable).map(FlowInstanceEntity::from); } - if (!params.getApproveByCurrentUser() && params.getCreatedByCurrentUser()) { + if (params.getCreatedByCurrentUser()) { // created by current user - specification = specification.and(FlowInstanceViewSpecs.projectIdEquals(params.getProjectId())) - .and(FlowInstanceViewSpecs.creatorIdEquals(authenticationFacade.currentUserId())); - return flowInstanceViewRepository.findAll(specification, pageable).map(FlowInstanceEntity::from); - } else if (params.getApproveByCurrentUser() && !params.getCreatedByCurrentUser()) { + specification = + specification.and(FlowInstanceViewSpecs.creatorIdEquals(authenticationFacade.currentUserId())); + } + if (params.getApproveByCurrentUser()) { + Set resourceRoleIdentifiers = userService.getCurrentUserResourceRoleIdentifiers(); + // does not join any project, so there does not exist any tickets to approve if (CollectionUtils.isEmpty(resourceRoleIdentifiers)) { return Page.empty(); } - // approving by current user - specification = - specification.and(FlowInstanceViewSpecs.projectIdEquals(params.getProjectId())) - .and(FlowInstanceViewSpecs.leftJoinFlowInstanceApprovalView( - resourceRoleIdentifiers, null, FlowNodeStatus.getExecutingStatuses())); - return flowInstanceViewRepository.findAll(specification, pageable).map(FlowInstanceEntity::from); - } else { - throw new UnsupportedOperationException("Unsupported list flow instance query"); + specification = specification.and(FlowInstanceViewSpecs.leftJoinFlowInstanceApprovalView( + resourceRoleIdentifiers, null, FlowNodeStatus.getExecutingStatuses())); } + return flowInstanceViewRepository.findAll(specification, pageable).map(FlowInstanceEntity::from); } public List listByIds(@NonNull Collection ids) { @@ -564,24 +539,19 @@ public List listByIds(@NonNull Collection ids) { } public FlowInstanceDetailResp detail(@NotNull Long id) { - return mapFlowInstance(id, flowInstance -> { + return mapFlowInstanceWithReadPermission(id, flowInstance -> { FlowInstanceMapper instanceMapper = mapperFactory.generateMapperByInstance(flowInstance, false); FlowNodeInstanceMapper nodeInstanceMapper = mapperFactory.generateNodeMapperByInstance(flowInstance, false); return instanceMapper.map(flowInstance, nodeInstanceMapper); - }, false); + }); } @Transactional(rollbackFor = Exception.class) public FlowInstanceDetailResp cancel(@NotNull Long id, Boolean skipAuth) { - FlowInstance flowInstance = mapFlowInstance(id, flowInst -> flowInst, skipAuth); + FlowInstance flowInstance = mapFlowInstanceWithWritePermission(id, flowInst -> flowInst); return cancel(flowInstance, skipAuth); } - public FlowInstanceDetailResp cancelNotCheckPermission(@NotNull Long id) { - FlowInstance flowInstance = mapFlowInstance(id, flowInst -> flowInst, false); - return cancel(flowInstance, false); - } - public Map getStatus(Set ids) { Specification specification = Specification.where(FlowInstanceSpecs.idIn(ids)) .and(FlowInstanceSpecs.organizationIdEquals(authenticationFacade.currentOrganizationId())); @@ -737,30 +707,37 @@ public FlowInstanceDetailResp reject(@NotNull Long id, return FlowInstanceDetailResp.withIdAndType(id, getTaskByFlowInstanceId(id).getTaskType()); } - public T mapFlowInstance(@NonNull Long flowInstanceId, Function function, Boolean skipAuth) { + public T mapFlowInstance(@NonNull Long flowInstanceId, Function mapper, + Consumer checkAuth) { Optional optional = flowFactory.getFlowInstance(flowInstanceId); FlowInstance flowInstance = optional.orElseThrow(() -> new NotFoundException(ResourceType.ODC_FLOW_INSTANCE, "id", flowInstanceId)); try { - if (!skipAuth) { - boolean isProjectOwner = flowInstance.getProjectId() != null && projectPermissionValidator - .hasProjectRole(flowInstance.getProjectId(), Collections.singletonList(ResourceRoleName.OWNER)); - if (!Objects.equals(authenticationFacade.currentUserId(), flowInstance.getCreatorId()) - && !isProjectOwner) { - List entities = approvalPermissionService.getApprovableApprovalInstances(); - Set flowInstanceIds = entities.stream().map(UserTaskInstanceEntity::getFlowInstanceId) - .collect(Collectors.toSet()); - PreConditions.validExists(ResourceType.ODC_FLOW_INSTANCE, "id", flowInstanceId, - () -> flowInstanceIds.contains(flowInstanceId)); - } - permissionValidator.checkCurrentOrganization(flowInstance); + if (checkAuth != null) { + checkAuth.accept(flowInstance); } - return function.apply(flowInstance); + return mapper.apply(flowInstance); } finally { flowInstance.dealloc(); } } + public T mapFlowInstanceWithReadPermission(@NonNull Long flowInstanceId, Function mapper) { + return mapFlowInstance(flowInstanceId, mapper, flowPermissionHelper.withProjectMemberCheck()); + } + + public T mapFlowInstanceWithWritePermission(@NonNull Long flowInstanceId, Function mapper) { + return mapFlowInstance(flowInstanceId, mapper, flowPermissionHelper.withExecutableCheck()); + } + + public T mapFlowInstanceWithApprovalPermission(@NonNull Long flowInstanceId, Function mapper) { + return mapFlowInstance(flowInstanceId, mapper, flowPermissionHelper.withApprovableCheck()); + } + + public T mapFlowInstanceWithoutPermissionCheck(@NonNull Long flowInstanceId, Function mapper) { + return mapFlowInstance(flowInstanceId, mapper, flowPermissionHelper.skipCheck()); + } + public TaskEntity getTaskByFlowInstanceId(Long id) { List entities = serviceTaskRepository .findAll(ServiceTaskInstanceSpecs.flowInstanceIdEquals(id)) @@ -792,7 +769,7 @@ private void checkCreateFlowInstancePermission(CreateFlowInstanceReq req) { } else if (CollectionUtils.isNotEmpty(parameters.getExportDbObjects())) { ConnectionConfig config = connectionService.getBasicWithoutPermissionCheck(req.getConnectionId()); parameters.getExportDbObjects().forEach(item -> { - if (item.getDbObjectType() == ObjectType.TABLE) { + if (item.getDbObjectType() == ObjectType.TABLE || item.getDbObjectType() == ObjectType.VIEW) { resource2Types.put( DBResource.from(config, req.getDatabaseName(), item.getObjectName(), ResourceType.ODC_TABLE), @@ -1095,28 +1072,31 @@ private FlowInstanceConfigurer buildConfigurer( private void completeApprovalInstance(@NonNull Long flowInstanceId, @NonNull Consumer consumer, Boolean skipAuth) { List instances = - mapFlowInstance(flowInstanceId, flowInstance -> flowInstance.filterInstanceNode(instance -> { - if (instance.getNodeType() != FlowNodeType.APPROVAL_TASK) { - return false; - } - return instance.getStatus() == FlowNodeStatus.EXECUTING - || instance.getStatus() == FlowNodeStatus.WAIT_FOR_CONFIRM; - }).stream().map(instance -> { - Verify.verify(instance instanceof FlowApprovalInstance, "FlowApprovalInstance's type is illegal"); - return (FlowApprovalInstance) instance; - }).collect(Collectors.toList()), skipAuth); + skipAuth ? mapFlowInstanceWithoutPermissionCheck(flowInstanceId, + generateApprovalMapper()) + : mapFlowInstanceWithApprovalPermission(flowInstanceId, generateApprovalMapper()); PreConditions.validExists(ResourceType.ODC_FLOW_APPROVAL_INSTANCE, "flowInstanceId", flowInstanceId, () -> instances.size() > 0); Verify.singleton(instances, "ApprovalInstance"); FlowApprovalInstance target = instances.get(0); - if (!skipAuth) { - PreConditions.validExists(ResourceType.ODC_FLOW_INSTANCE, "id", flowInstanceId, - () -> approvalPermissionService.isApprovable(target.getId())); - } Verify.verify(target.isPresentOnThisMachine(), "Approval instance is not on this machine"); consumer.accept(target); } + private Function> generateApprovalMapper() { + return flowInstance -> flowInstance.filterInstanceNode(instance -> { + if (instance.getNodeType() != FlowNodeType.APPROVAL_TASK) { + return false; + } + return instance.getStatus() == FlowNodeStatus.EXECUTING + || instance.getStatus() == FlowNodeStatus.WAIT_FOR_CONFIRM; + }).stream().map(instance -> { + Verify.verify(instance instanceof FlowApprovalInstance, + "FlowApprovalInstance's type is illegal"); + return (FlowApprovalInstance) instance; + }).collect(Collectors.toList()); + } + private void initVariables(Map variables, TaskEntity taskEntity, TaskEntity preCheckTaskEntity, List configList, RiskLevelDescriber riskLevelDescriber) { FlowTaskUtil.setTaskId(variables, taskEntity.getId()); @@ -1198,7 +1178,7 @@ private TemplateVariables buildTemplateVariables(CreateFlowInstanceReq flowInsta } List databaseOwners = new ArrayList<>(); List userResourceRoles = - resourceRoleService.listByResourceTypeAndId(ResourceType.ODC_DATABASE, database.getId()); + resourceRoleService.listByResourceTypeAndResourceId(ResourceType.ODC_DATABASE, database.getId()); if (CollectionUtils.isNotEmpty(userResourceRoles)) { Set userIds = userResourceRoles.stream().map(UserResourceRole::getUserId).collect(Collectors.toSet()); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowPermissionHelper.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowPermissionHelper.java new file mode 100644 index 0000000000..f4a7c1b58d --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowPermissionHelper.java @@ -0,0 +1,116 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.flow; + +import java.util.Arrays; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.function.Consumer; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import com.oceanbase.odc.core.shared.PreConditions; +import com.oceanbase.odc.core.shared.constant.ResourceRoleName; +import com.oceanbase.odc.core.shared.constant.ResourceType; +import com.oceanbase.odc.core.shared.exception.AccessDeniedException; +import com.oceanbase.odc.metadb.flow.UserTaskInstanceEntity; +import com.oceanbase.odc.service.flow.instance.FlowInstance; +import com.oceanbase.odc.service.iam.HorizontalDataPermissionValidator; +import com.oceanbase.odc.service.iam.ProjectPermissionValidator; +import com.oceanbase.odc.service.iam.auth.AuthenticationFacade; + +/** + * @Author: Lebie + * @Date: 2024/10/25 15:29 + * @Description: [] + */ +@Component +public class FlowPermissionHelper { + @Autowired + private ProjectPermissionValidator projectPermissionValidator; + + @Autowired + private ApprovalPermissionService approvalPermissionService; + + @Autowired + private AuthenticationFacade authenticationFacade; + + @Autowired + private HorizontalDataPermissionValidator horizontalDataPermissionValidator; + + + public Consumer withProjectMemberCheck() { + return withProjectPermissionCheck( + flowInstance -> flowInstance.getProjectId() != null && projectPermissionValidator + .hasProjectRole(flowInstance.getProjectId(), ResourceRoleName.all())); + } + + public Consumer withProjectOwnerOrDBACheck() { + return withProjectPermissionCheck( + flowInstance -> flowInstance.getProjectId() != null && projectPermissionValidator + .hasProjectRole(flowInstance.getProjectId(), + Arrays.asList(ResourceRoleName.OWNER, ResourceRoleName.DBA))); + } + + public Consumer withApprovableCheck() { + return flowInstance -> { + List entities = approvalPermissionService.getApprovableApprovalInstances(); + Set flowInstanceIds = entities.stream().map(UserTaskInstanceEntity::getFlowInstanceId) + .collect(Collectors.toSet()); + PreConditions.validExists(ResourceType.ODC_FLOW_INSTANCE, "id", flowInstance.getId(), + () -> flowInstanceIds.contains(flowInstance.getId())); + horizontalDataPermissionValidator.checkCurrentOrganization(flowInstance); + }; + } + + public Consumer withExecutableCheck() { + return flowInstance -> { + try { + withCreatorCheck().accept(flowInstance); + } catch (Exception ex) { + withProjectOwnerOrDBACheck().accept(flowInstance); + } + }; + } + + public Consumer withCreatorCheck() { + return flowInstance -> { + if (!Objects.equals(authenticationFacade.currentUserId(), flowInstance.getCreatorId())) { + throw new AccessDeniedException(); + } + horizontalDataPermissionValidator.checkCurrentOrganization(flowInstance); + }; + } + + public Consumer skipCheck() { + return flowInstance -> { + }; + } + + private Consumer withProjectPermissionCheck(Predicate predicate) { + return flowInstance -> { + if (!Objects.equals(authenticationFacade.currentUserId(), flowInstance.getCreatorId()) + && !predicate.test(flowInstance)) { + throw new AccessDeniedException(); + } + horizontalDataPermissionValidator.checkCurrentOrganization(flowInstance); + }; + } +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowTaskInstanceLoggerService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowTaskInstanceLoggerService.java index 8dc2eed412..f5e73bf95a 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowTaskInstanceLoggerService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowTaskInstanceLoggerService.java @@ -68,11 +68,15 @@ public class FlowTaskInstanceLoggerService { @Autowired private ScheduleLogProperties loggerProperty; + @Autowired + private FlowPermissionHelper flowPermissionHelper; + @SneakyThrows public String getLogContent(OdcTaskLogLevel level, Long flowInstanceId) { try { Optional taskEntityOptional = - flowTaskInstanceService.getLogDownloadableTaskEntity(flowInstanceId, false); + flowTaskInstanceService.getLogDownloadableTaskEntity(flowInstanceId, + flowPermissionHelper.withProjectMemberCheck()); return getLogContent(taskEntityOptional, level, flowInstanceId); } catch (Exception e) { log.warn("Task log file not found, flowInstanceId={}", flowInstanceId, e); @@ -84,7 +88,7 @@ public String getLogContent(OdcTaskLogLevel level, Long flowInstanceId) { public String getLogContentWithoutPermission(OdcTaskLogLevel level, Long flowInstanceId) { try { Optional taskEntityOptional = - flowTaskInstanceService.getLogDownloadableTaskEntity(flowInstanceId, true); + flowTaskInstanceService.getLogDownloadableTaskEntity(flowInstanceId, null); return getLogContent(taskEntityOptional, level, flowInstanceId); } catch (Exception e) { log.warn("get log failed, task log file not found, flowInstanceId={}", flowInstanceId); @@ -129,7 +133,8 @@ public String getLogContent(Optional taskEntityOptional, OdcTaskLogL @SneakyThrows private InputStream downloadLog(Long flowInstanceId) { Optional taskEntityOptional = - flowTaskInstanceService.getLogDownloadableTaskEntity(flowInstanceId, false); + flowTaskInstanceService.getLogDownloadableTaskEntity(flowInstanceId, + flowPermissionHelper.withProjectMemberCheck()); TaskEntity taskEntity = taskEntityOptional .orElseThrow(() -> new NotFoundException(ErrorCodes.NotFound, new Object[] {flowInstanceId}, ErrorCodes.TaskLogNotFound.getLocalizedMessage(new Object[] {"Id", flowInstanceId}))); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowTaskInstanceService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowTaskInstanceService.java index 5a1642c3f0..617e28a50e 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowTaskInstanceService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/FlowTaskInstanceService.java @@ -30,6 +30,7 @@ import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Consumer; import java.util.function.Predicate; import java.util.stream.Collectors; @@ -77,6 +78,7 @@ import com.oceanbase.odc.service.dispatch.DispatchResponse; import com.oceanbase.odc.service.dispatch.RequestDispatcher; import com.oceanbase.odc.service.dispatch.TaskDispatchChecker; +import com.oceanbase.odc.service.flow.instance.FlowInstance; import com.oceanbase.odc.service.flow.instance.FlowTaskInstance; import com.oceanbase.odc.service.flow.model.BinaryDataResult; import com.oceanbase.odc.service.flow.model.ByteArrayDataResult; @@ -160,6 +162,8 @@ public class FlowTaskInstanceService { private TaskFrameworkEnabledProperties taskFrameworkProperties; @Autowired private FlowTaskInstanceLoggerService flowTaskInstanceLoggerService; + @Autowired + private FlowPermissionHelper flowPermissionHelper; @Value("${odc.task.async.result-preview-max-size-bytes:5242880}") private long resultPreviewMaxSizeBytes; @@ -169,7 +173,8 @@ public class FlowTaskInstanceService { @Transactional(rollbackFor = Exception.class) public FlowInstanceDetailResp executeTask(@NotNull Long id) throws IOException { List instances = - filterTaskInstance(id, instance -> instance.getStatus() == FlowNodeStatus.PENDING, false); + filterTaskInstance(id, instance -> instance.getStatus() == FlowNodeStatus.PENDING, + flowPermissionHelper.withExecutableCheck()); PreConditions.validExists(ResourceType.ODC_FLOW_TASK_INSTANCE, "flowInstanceId", id, () -> instances.size() > 0); Verify.singleton(instances, "FlowTaskInstance"); @@ -205,18 +210,12 @@ public InputStreamResource downloadLog(@NotNull Long flowInstanceId) { return flowTaskInstanceLoggerService.downloadLogFile(flowInstanceId); } - public List getResult(@NotNull Long id, boolean skipAuth) throws IOException { - TaskEntity task = flowInstanceService.getTaskByFlowInstanceId(id); - if (task.getTaskType() == TaskType.ONLINE_SCHEMA_CHANGE || task.getTaskType() == TaskType.EXPORT - || task.getTaskType() == TaskType.MULTIPLE_ASYNC) { - return getTaskResultFromEntity(task, true); - } - Optional taskEntityOptional = getCompleteTaskEntity(id, skipAuth); - if (!taskEntityOptional.isPresent()) { - return Collections.emptyList(); - } - TaskEntity taskEntity = taskEntityOptional.get(); - return getTaskResultFromEntity(taskEntity, true); + public List getResult(@NotNull Long id) throws IOException { + return getResult(id, flowPermissionHelper.withProjectMemberCheck()); + } + + public List getResultSkipPermissionCheck(@NotNull Long id) throws IOException { + return getResult(id, flowPermissionHelper.skipCheck()); } public List getTaskResultFromEntity(@NotNull TaskEntity taskEntity, @@ -261,11 +260,10 @@ public List getTaskResultFromEntity(@NotNull TaskEntit } public List getResult( - @NotNull Long flowInstanceId, @NotNull Long nodeInstanceId, boolean skipAuth) throws IOException { - List taskInstances = this.flowInstanceService.mapFlowInstance( + @NotNull Long flowInstanceId, @NotNull Long nodeInstanceId) throws IOException { + List taskInstances = this.flowInstanceService.mapFlowInstanceWithReadPermission( flowInstanceId, i -> i.filterInstanceNode(f -> f instanceof FlowTaskInstance) - .stream().map(f -> (FlowTaskInstance) f).collect(Collectors.toList()), - skipAuth); + .stream().map(f -> (FlowTaskInstance) f).collect(Collectors.toList())); Optional target = taskInstances.stream() .filter(f -> f.getId().equals(nodeInstanceId)).findFirst(); if (!target.isPresent()) { @@ -340,7 +338,8 @@ public List download(@NonNull Long flowInstanceId, String targ public List downRollbackPlanResult(@NonNull Long flowInstanceId) throws IOException { Optional taskEntityOptional = getTaskEntity(flowInstanceId, - instance -> instance.getStatus().isFinalStatus() && instance.getTaskType() == TaskType.ASYNC, false); + instance -> instance.getStatus().isFinalStatus() && instance.getTaskType() == TaskType.ASYNC, + flowPermissionHelper.withProjectMemberCheck()); PreConditions.validExists(ResourceType.ODC_FILE, "flowInstanceId", flowInstanceId, taskEntityOptional::isPresent); TaskEntity taskEntity = taskEntityOptional.get(); @@ -536,7 +535,8 @@ public List getAsyncDownloadUrl(Long id, List objectIds, String } public List getExecuteResult(Long flowInstanceId) throws IOException { - Optional taskEntityOptional = getCompleteTaskEntity(flowInstanceId, false); + Optional taskEntityOptional = getCompleteTaskEntity(flowInstanceId, + flowPermissionHelper.withProjectMemberCheck()); if (!taskEntityOptional.isPresent()) { return Collections.emptyList(); } @@ -565,6 +565,22 @@ public List getExecuteResult(Long flowInstanceId) throws IOExc } } + + private List getResult(@NotNull Long id, Consumer checkAuth) + throws IOException { + TaskEntity task = flowInstanceService.getTaskByFlowInstanceId(id); + if (task.getTaskType() == TaskType.ONLINE_SCHEMA_CHANGE || task.getTaskType() == TaskType.EXPORT + || task.getTaskType() == TaskType.MULTIPLE_ASYNC) { + return getTaskResultFromEntity(task, true); + } + Optional taskEntityOptional = getCompleteTaskEntity(id, checkAuth); + if (!taskEntityOptional.isPresent()) { + return Collections.emptyList(); + } + TaskEntity taskEntity = taskEntityOptional.get(); + return getTaskResultFromEntity(taskEntity, true); + } + private Set getDownloadImportFileNames(@NonNull TaskEntity taskEntity, String targetFileName) { DataTransferConfig config = JsonUtils.fromJson( taskEntity.getParametersJson(), DataTransferConfig.class); @@ -585,7 +601,7 @@ private Set getDownloadImportFileNames(@NonNull TaskEntity taskEntity, S } private List filterTaskInstance(@NonNull Long flowInstanceId, - @NonNull Predicate predicate, boolean skipAuth) { + @NonNull Predicate predicate, Consumer checkAuth) { return flowInstanceService.mapFlowInstance(flowInstanceId, flowInstance -> flowInstance.filterInstanceNode(instance -> { if (instance.getNodeType() != FlowNodeType.SERVICE_TASK) { @@ -595,7 +611,7 @@ private List filterTaskInstance(@NonNull Long flowInstanceId, }).stream().map(instance -> { Verify.verify(instance instanceof FlowTaskInstance, "FlowTaskInstance's type is illegal"); return (FlowTaskInstance) instance; - }).collect(Collectors.toList()), skipAuth); + }).collect(Collectors.toList()), checkAuth); } private List getMultipleAsyncResult(@NonNull TaskEntity taskEntity) { @@ -702,11 +718,11 @@ private List innerGetResult(@NonNull TaskEntity ta return Collections.singletonList(detail); } - private Optional getCompleteTaskEntity(@NonNull Long flowInstanceId, boolean skipAuth) { + private Optional getCompleteTaskEntity(@NonNull Long flowInstanceId, Consumer checkAuth) { return getTaskEntity(flowInstanceId, i -> i.getStatus().isFinalStatus() && i.getTaskType() != TaskType.SQL_CHECK && i.getTaskType() != TaskType.PRE_CHECK - && i.getTaskType() != TaskType.GENERATE_ROLLBACK, skipAuth); + && i.getTaskType() != TaskType.GENERATE_ROLLBACK, checkAuth); } private Optional getDownloadableTaskEntity(@NonNull Long flowInstanceId) { @@ -725,20 +741,21 @@ private Optional getDownloadableTaskEntity(@NonNull Long flowInstanc && instance.getTaskType() != TaskType.APPLY_DATABASE_PERMISSION && instance.getTaskType() != TaskType.APPLY_TABLE_PERMISSION; } - }, false); + }, flowPermissionHelper.withProjectMemberCheck()); } - public Optional getLogDownloadableTaskEntity(@NotNull Long flowInstanceId, boolean skipAuth) { + public Optional getLogDownloadableTaskEntity(@NotNull Long flowInstanceId, + Consumer checkAuth) { return getTaskEntity(flowInstanceId, instance -> (instance.getStatus().isFinalStatus() || instance.getStatus() == FlowNodeStatus.EXECUTING) && instance.getTaskType() != TaskType.SQL_CHECK && instance.getTaskType() != TaskType.PRE_CHECK && instance.getTaskType() != TaskType.GENERATE_ROLLBACK, - skipAuth); + checkAuth); } private Optional getTaskEntity(@NonNull Long flowInstanceId, - @NonNull Predicate predicate, boolean skipAuth) { - List taskInstances = filterTaskInstance(flowInstanceId, predicate, skipAuth); + @NonNull Predicate predicate, Consumer checkAuth) { + List taskInstances = filterTaskInstance(flowInstanceId, predicate, checkAuth); if (CollectionUtils.isEmpty(taskInstances)) { return Optional.empty(); } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/factory/FlowResponseMapperFactory.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/factory/FlowResponseMapperFactory.java index aac09f1535..55f8a3a72e 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/factory/FlowResponseMapperFactory.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/factory/FlowResponseMapperFactory.java @@ -26,7 +26,6 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Objects; -import java.util.Optional; import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; @@ -41,11 +40,7 @@ import com.google.common.collect.Sets; import com.oceanbase.odc.common.json.JsonUtils; import com.oceanbase.odc.common.util.StringUtils; -import com.oceanbase.odc.core.shared.constant.ResourceType; import com.oceanbase.odc.core.shared.constant.TaskType; -import com.oceanbase.odc.core.shared.exception.NotFoundException; -import com.oceanbase.odc.metadb.collaboration.ProjectEntity; -import com.oceanbase.odc.metadb.collaboration.ProjectRepository; import com.oceanbase.odc.metadb.connection.ConnectionConfigRepository; import com.oceanbase.odc.metadb.connection.ConnectionEntity; import com.oceanbase.odc.metadb.connection.ConnectionSpecs; @@ -66,17 +61,18 @@ import com.oceanbase.odc.metadb.iam.UserRepository; import com.oceanbase.odc.metadb.iam.UserRoleEntity; import com.oceanbase.odc.metadb.iam.UserRoleRepository; -import com.oceanbase.odc.metadb.iam.resourcerole.UserResourceRoleEntity; -import com.oceanbase.odc.metadb.iam.resourcerole.UserResourceRoleRepository; import com.oceanbase.odc.metadb.integration.IntegrationEntity; import com.oceanbase.odc.metadb.regulation.risklevel.RiskLevelRepository; import com.oceanbase.odc.metadb.task.TaskEntity; import com.oceanbase.odc.metadb.task.TaskRepository; import com.oceanbase.odc.metadb.task.TaskSpecs; +import com.oceanbase.odc.service.collaboration.project.ProjectService; +import com.oceanbase.odc.service.collaboration.project.model.Project; import com.oceanbase.odc.service.connection.database.DatabaseService; import com.oceanbase.odc.service.connection.database.model.Database; import com.oceanbase.odc.service.connection.model.ConnectionConfig; import com.oceanbase.odc.service.connection.util.ConnectionMapper; +import com.oceanbase.odc.service.databasechange.model.DatabaseChangeDatabase; import com.oceanbase.odc.service.flow.ApprovalPermissionService; import com.oceanbase.odc.service.flow.instance.FlowInstance; import com.oceanbase.odc.service.flow.model.FlowInstanceDetailResp; @@ -86,13 +82,17 @@ import com.oceanbase.odc.service.flow.model.FlowNodeStatus; import com.oceanbase.odc.service.flow.model.FlowTaskExecutionStrategy; import com.oceanbase.odc.service.flow.task.model.DBStructureComparisonParameter; +import com.oceanbase.odc.service.flow.task.model.MultipleDatabaseChangeParameters; +import com.oceanbase.odc.service.iam.ResourceRoleService; import com.oceanbase.odc.service.iam.auth.AuthenticationFacade; +import com.oceanbase.odc.service.iam.model.UserResourceRole; import com.oceanbase.odc.service.integration.IntegrationService; import com.oceanbase.odc.service.integration.client.ApprovalClient; import com.oceanbase.odc.service.integration.model.ApprovalProperties; import com.oceanbase.odc.service.integration.model.IntegrationConfig; import com.oceanbase.odc.service.integration.model.TemplateVariables; import com.oceanbase.odc.service.integration.model.TemplateVariables.Variable; +import com.oceanbase.odc.service.permission.project.ApplyProjectParameter; import com.oceanbase.odc.service.regulation.risklevel.RiskLevelMapper; import lombok.NonNull; @@ -136,13 +136,14 @@ public class FlowResponseMapperFactory { @Autowired private FlowInstanceRepository flowInstanceRepository; @Autowired - private UserResourceRoleRepository userResourceRoleRepository; + private ResourceRoleService resourceRoleService; @Autowired private RiskLevelRepository riskLevelRepository; @Autowired private AuthenticationFacade authenticationFacade; @Autowired - private ProjectRepository projectRepository; + private ProjectService projectService; + private final ConnectionMapper connectionMapper = ConnectionMapper.INSTANCE; private final RiskLevelMapper riskLevelMapper = RiskLevelMapper.INSTANCE; @@ -227,9 +228,9 @@ private FlowNodeInstanceMapper generateNodeMapper(@NonNull Collection flow && candidateResourceRoleIdentifiers.isEmpty()) { return Collections.emptyList(); } else if (!candidateResourceRoleIdentifiers.isEmpty()) { - Set resourceRoleUserIds = userResourceRoleRepository - .findByResourceIdsAndResourceRoleIdsIn(candidateResourceRoleIdentifiers) - .stream().map(UserResourceRoleEntity::getUserId).collect(Collectors.toSet()); + Set resourceRoleUserIds = + resourceRoleService.listByResourceIdentifierIn(candidateResourceRoleIdentifiers) + .stream().map(UserResourceRole::getUserId).collect(Collectors.toSet()); return CollectionUtils.isEmpty(resourceRoleUserIds) ? Collections.emptyList() : userRepository.findByUserIdsAndEnabled(resourceRoleUserIds, true); } else if (candidateUserIds.isEmpty()) { @@ -267,7 +268,8 @@ private FlowNodeInstanceMapper generateNodeMapper(@NonNull Collection flow IntegrationConfig config = integrationService.detailWithoutPermissionCheck(externalApproval.getApprovalId()); ApprovalProperties properties = ApprovalProperties.from(config); - if (StringUtils.isEmpty(properties.getAdvanced().getHyperlinkExpression())) { + if (Objects.isNull(properties.getAdvanced()) + || StringUtils.isEmpty(properties.getAdvanced().getHyperlinkExpression())) { return null; } TemplateVariables variables = new TemplateVariables(); @@ -328,43 +330,22 @@ private FlowInstanceMapper generateMapper(@NonNull Collection flowInstance Set databaseIds = taskId2TaskEntity.values().stream() .map(TaskEntity::getDatabaseId) .filter(Objects::nonNull).collect(Collectors.toSet()); - Set sourceDatabaseIdsInComparisonTask = new HashSet<>(); - Set targetDatabaseIdsInComparisonTask = new HashSet<>(); - taskId2TaskEntity.values().stream() - .filter(task -> task.getTaskType().equals(TaskType.STRUCTURE_COMPARISON)) - .forEach(task -> { - DBStructureComparisonParameter parameter = JsonUtils.fromJson( - task.getParametersJson(), DBStructureComparisonParameter.class); - sourceDatabaseIdsInComparisonTask.add(parameter.getSourceDatabaseId()); - targetDatabaseIdsInComparisonTask.add(parameter.getTargetDatabaseId()); - }); - databaseIds.addAll(targetDatabaseIdsInComparisonTask); + databaseIds.addAll(collectMultiDatabaseChangeDatabaseIds(taskId2TaskEntity)); + databaseIds.addAll(collectDBStructureComparisonDatabaseIds(taskId2TaskEntity)); + Set projectIds = new HashSet<>(); + Map id2Project = new HashMap<>(); if (CollectionUtils.isNotEmpty(databaseIds)) { id2Database = databaseService.listDatabasesByIds(databaseIds).stream() .collect(Collectors.toMap(Database::getId, database -> database)); - - // set project name for structure comparison task - Set projectIds = sourceDatabaseIdsInComparisonTask.stream() - .map(id2Database::get) - .filter(Objects::nonNull) - .map(database -> database.getProject().getId()) - .collect(Collectors.toSet()); - Map id2ProjectEntity = projectRepository.findByIdIn(projectIds).stream() - .collect(Collectors.toMap(ProjectEntity::getId, Function.identity())); - - for (Long id : sourceDatabaseIdsInComparisonTask) { - Database database = id2Database.get(id); - if (Objects.nonNull(database) && Objects.nonNull(database.getProject())) { - Long projectId = database.getProject().getId(); - if (Objects.nonNull(projectId)) { - ProjectEntity projectEntity = Optional.ofNullable(id2ProjectEntity.get(projectId)).orElseThrow( - () -> new NotFoundException(ResourceType.ODC_PROJECT, "projectId", projectId)); - database.getProject().setName(projectEntity.getName()); - } - } - } + projectIds.addAll(id2Database.values().stream().map(db -> db.getProject().getId()) + .filter(Objects::nonNull).collect(Collectors.toSet())); + } + projectIds.addAll(collectApplyProjectIds(taskId2TaskEntity)); + if (CollectionUtils.isNotEmpty(projectIds)) { + id2Project = projectService.listByIds(projectIds).stream() + .collect(Collectors.toMap(Project::getId, project -> project, (a, b) -> a)); } /** * find the ConnectionConfig associated with each Database @@ -417,7 +398,9 @@ private FlowInstanceMapper generateMapper(@NonNull Collection flowInstance .getRiskLevelByRiskLevelId( id -> riskLevelRepository.findById(id).map(riskLevelMapper::entityToModel).orElse(null)) .getCandidatesByFlowInstanceId(candidatesByFlowInstanceIds::get) - .getDatabaseById(id2Database::get).build(); + .getDatabaseById(id2Database::get) + .getProjectById(id2Project::get) + .build(); } public Map> getUserId2Roles(@NonNull Collection userIds, boolean skipAuth) { @@ -473,4 +456,44 @@ private List listConnectionsByConnectionIdsWithoutPermissionCh return connectionRepository.findAll(specification); } + private Set collectDBStructureComparisonDatabaseIds(Map taskId2TaskEntity) { + Set targetDatabaseIdsInComparisonTask = new HashSet<>(); + taskId2TaskEntity.values().stream() + .filter(task -> task.getTaskType().equals(TaskType.STRUCTURE_COMPARISON)) + .forEach(task -> { + DBStructureComparisonParameter parameter = JsonUtils.fromJson( + task.getParametersJson(), DBStructureComparisonParameter.class); + targetDatabaseIdsInComparisonTask.add(parameter.getTargetDatabaseId()); + }); + return targetDatabaseIdsInComparisonTask; + } + + private Set collectApplyProjectIds(Map taskId2TaskEntity) { + Set applyProjectIds = taskId2TaskEntity.values().stream() + .filter(task -> task.getTaskType() == TaskType.APPLY_PROJECT_PERMISSION) + .map(task -> { + ApplyProjectParameter parameter = + JsonUtils.fromJson(task.getParametersJson(), ApplyProjectParameter.class); + if (Objects.nonNull(parameter) && Objects.nonNull(parameter.getProject())) { + return parameter.getProject().getId(); + } + return null; + }) + .filter(Objects::nonNull) + .collect(Collectors.toSet()); + return applyProjectIds; + } + + private Set collectMultiDatabaseChangeDatabaseIds(Map taskId2TaskEntity) { + Set databaseIds = new HashSet<>(); + taskId2TaskEntity.values().stream() + .filter(task -> task.getTaskType().equals(TaskType.MULTIPLE_ASYNC)) + .forEach(task -> { + MultipleDatabaseChangeParameters parameter = JsonUtils.fromJson( + task.getParametersJson(), MultipleDatabaseChangeParameters.class); + databaseIds.addAll(parameter.getDatabases().stream().map(DatabaseChangeDatabase::getId) + .collect(Collectors.toSet())); + }); + return databaseIds; + } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/listener/ApprovalStatusNotifyListener.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/listener/ApprovalStatusNotifyListener.java index dc63c66be1..74cfc729b6 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/listener/ApprovalStatusNotifyListener.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/listener/ApprovalStatusNotifyListener.java @@ -28,12 +28,12 @@ import com.oceanbase.odc.core.flow.util.EmptyExecutionListener; import com.oceanbase.odc.metadb.flow.FlowInstanceApprovalViewEntity; import com.oceanbase.odc.metadb.flow.FlowInstanceApprovalViewRepository; -import com.oceanbase.odc.metadb.iam.resourcerole.UserResourceRoleEntity; -import com.oceanbase.odc.metadb.iam.resourcerole.UserResourceRoleRepository; import com.oceanbase.odc.metadb.task.TaskEntity; import com.oceanbase.odc.service.flow.FlowInstanceService; import com.oceanbase.odc.service.flow.FlowableAdaptor; import com.oceanbase.odc.service.flow.instance.FlowApprovalInstance; +import com.oceanbase.odc.service.iam.ResourceRoleService; +import com.oceanbase.odc.service.iam.model.UserResourceRole; import com.oceanbase.odc.service.notification.Broker; import com.oceanbase.odc.service.notification.NotificationProperties; import com.oceanbase.odc.service.notification.helper.EventBuilder; @@ -59,9 +59,9 @@ public class ApprovalStatusNotifyListener extends EmptyExecutionListener { @Autowired private FlowableAdaptor flowableAdaptor; @Autowired - FlowInstanceApprovalViewRepository flowInstanceApprovalViewRepository; + private FlowInstanceApprovalViewRepository flowInstanceApprovalViewRepository; @Autowired - UserResourceRoleRepository userResourceRoleRepository; + private ResourceRoleService resourceRoleService; @Override protected void onExecutiuonStart(DelegateExecution execution) { @@ -79,12 +79,12 @@ protected void onExecutiuonStart(DelegateExecution execution) { List approvals = flowInstanceApprovalViewRepository.findByIdIn(Collections.singletonList(target.getId())); if (CollectionUtils.isNotEmpty(approvals)) { - List userResourceRoles = - userResourceRoleRepository.findByResourceIdsAndResourceRoleIdsIn( + List userResourceRoles = + resourceRoleService.listByResourceIdentifierIn( approvals.stream().map(FlowInstanceApprovalViewEntity::getResourceRoleIdentifier) .collect(Collectors.toSet())); approverIds = CollectionUtils.isEmpty(userResourceRoles) ? null - : userResourceRoles.stream().map(UserResourceRoleEntity::getUserId).collect(Collectors.toSet()); + : userResourceRoles.stream().map(UserResourceRole::getUserId).collect(Collectors.toSet()); } Event event = eventBuilder.ofPendingApprovalTask(taskEntity, approverIds); broker.enqueueEvent(event); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/model/FlowInstanceDetailResp.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/model/FlowInstanceDetailResp.java index 04497a9690..e9f18dd262 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/model/FlowInstanceDetailResp.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/model/FlowInstanceDetailResp.java @@ -41,6 +41,7 @@ import com.oceanbase.odc.metadb.iam.UserEntity; import com.oceanbase.odc.metadb.task.TaskEntity; import com.oceanbase.odc.plugin.task.api.datatransfer.model.DataTransferConfig; +import com.oceanbase.odc.service.collaboration.project.model.Project; import com.oceanbase.odc.service.common.model.InnerUser; import com.oceanbase.odc.service.connection.database.model.Database; import com.oceanbase.odc.service.flow.instance.BaseFlowNodeInstance; @@ -99,6 +100,7 @@ public class FlowInstanceDetailResp { private boolean rollbackable; private Date completeTime; private List nodeList; + private Project project; @Getter @Builder @@ -113,6 +115,7 @@ public static class FlowInstanceMapper { private final Function> getExecutionStrategyByFlowInstanceId; private final Function getRiskLevelByRiskLevelId; private final Function> getCandidatesByFlowInstanceId; + private final Function getProjectById; public FlowInstanceDetailResp map(@NonNull FlowInstanceEntity entity) { FlowInstanceDetailResp resp = FlowInstanceDetailResp.withId(entity.getId()); @@ -120,6 +123,7 @@ public FlowInstanceDetailResp map(@NonNull FlowInstanceEntity entity) { resp.setCreateTime(entity.getCreateTime()); resp.setStatus(entity.getStatus()); resp.setProjectId(entity.getProjectId()); + resp.setProject(getProjectById.apply(entity.getProjectId())); resp.setOrganizationId(entity.getOrganizationId()); Set candidates = getCandidatesByFlowInstanceId.apply(entity.getId()); if (candidates != null) { @@ -172,6 +176,7 @@ public FlowInstanceDetailResp map(@NonNull FlowInstance flowInstance, @NonNull F resp.setStatus(flowInstance.getStatus()); resp.setDescription(flowInstance.getDescription()); resp.setProjectId(flowInstance.getProjectId()); + resp.setProject(getProjectById.apply(flowInstance.getProjectId())); List instances = flowInstance.filterInstanceNode(instance -> FlowNodeType.SERVICE_TASK == instance.getNodeType() diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/model/QueryFlowInstanceParams.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/model/QueryFlowInstanceParams.java index 3d02e2714f..e8f309f09e 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/model/QueryFlowInstanceParams.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/model/QueryFlowInstanceParams.java @@ -17,6 +17,7 @@ import java.util.Date; import java.util.List; +import java.util.Set; import com.oceanbase.odc.core.shared.constant.FlowStatus; import com.oceanbase.odc.core.shared.constant.TaskType; @@ -49,5 +50,5 @@ public class QueryFlowInstanceParams { private Boolean containsAll; private Long parentInstanceId; - private Long projectId; + private Set projectIds; } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/processor/CreateFlowInstanceProcessAspect.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/processor/CreateFlowInstanceProcessAspect.java index 0fa3896d45..256fef7270 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/processor/CreateFlowInstanceProcessAspect.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/processor/CreateFlowInstanceProcessAspect.java @@ -44,9 +44,6 @@ import com.oceanbase.odc.service.iam.auth.AuthenticationFacade; import com.oceanbase.odc.service.partitionplan.model.PartitionPlanConfig; import com.oceanbase.odc.service.quartz.util.QuartzCronExpressionUtils; -import com.oceanbase.odc.service.schedule.flowtask.AlterScheduleParameters; -import com.oceanbase.odc.service.schedule.model.JobType; -import com.oceanbase.odc.service.schedule.model.OperationType; import com.oceanbase.odc.service.schedule.model.TriggerConfig; import com.oceanbase.odc.service.schedule.model.TriggerStrategy; @@ -74,8 +71,6 @@ public class CreateFlowInstanceProcessAspect implements InitializingBean { @Value("${odc.task.trigger.minimum-interval:600}") private Long triggerMinimumIntervalSeconds; - private final Map scheduleTaskPreprocessors = new HashMap<>(); - private final Map flowTaskPreprocessors = new HashMap<>(); @Pointcut("@annotation(com.oceanbase.odc.service.flow.processor.EnablePreprocess) && args(com.oceanbase.odc.service.flow.model.CreateFlowInstanceReq)") @@ -148,14 +143,6 @@ private void validateTriggerConfig(CreateFlowInstanceReq req) { if (parameters.getDroppingTrigger() != null) { validateTriggerConfig(parameters.getDroppingTrigger()); } - return; - } - if (req.getParameters() instanceof AlterScheduleParameters) { - AlterScheduleParameters parameters = (AlterScheduleParameters) req.getParameters(); - if (parameters.getOperationType() == OperationType.CREATE - || parameters.getOperationType() == OperationType.UPDATE) { - validateTriggerConfig(parameters.getTriggerConfig()); - } } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/DatabaseChangeRuntimeFlowableTaskCopied.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/DatabaseChangeRuntimeFlowableTaskCopied.java index be2f01c11b..41ec4c55a2 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/DatabaseChangeRuntimeFlowableTaskCopied.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/DatabaseChangeRuntimeFlowableTaskCopied.java @@ -52,12 +52,12 @@ import com.oceanbase.odc.service.objectstorage.model.ObjectMetadata; import com.oceanbase.odc.service.sqlcheck.SqlCheckUtil; import com.oceanbase.odc.service.task.TaskService; +import com.oceanbase.odc.service.task.base.databasechange.DatabaseChangeTask; +import com.oceanbase.odc.service.task.base.databasechange.DatabaseChangeTaskParameters; import com.oceanbase.odc.service.task.config.TaskFrameworkProperties; import com.oceanbase.odc.service.task.constants.JobParametersKeyConstants; import com.oceanbase.odc.service.task.enums.JobStatus; import com.oceanbase.odc.service.task.exception.JobException; -import com.oceanbase.odc.service.task.runtime.DatabaseChangeTask; -import com.oceanbase.odc.service.task.runtime.DatabaseChangeTaskParameters; import com.oceanbase.odc.service.task.schedule.DefaultJobDefinition; import com.oceanbase.odc.service.task.schedule.JobDefinition; import com.oceanbase.odc.service.task.schedule.JobScheduler; @@ -148,7 +148,7 @@ protected DatabaseChangeResult start(Long taskId, TaskService taskService, Deleg TimeUnit.MILLISECONDS); JobEntity jobEntity = taskFrameworkService.find(this.jobId); - result = JsonUtils.fromJson(jobEntity.getResultJson(), DatabaseChangeResult.class); + result = JsonUtils.fromJson(JobUtils.retrieveJobResultStr(jobEntity), DatabaseChangeResult.class); result.setRollbackPlanResult(rollbackPlanTaskResult); if (jobEntity.getStatus() == JobStatus.DONE) { isSuccessful = true; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/PreCheckRuntimeFlowableTaskCopied.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/PreCheckRuntimeFlowableTaskCopied.java index 25fac88b4c..c4d731ccfd 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/PreCheckRuntimeFlowableTaskCopied.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/PreCheckRuntimeFlowableTaskCopied.java @@ -66,11 +66,11 @@ import com.oceanbase.odc.service.sqlcheck.model.CheckResult; import com.oceanbase.odc.service.sqlcheck.model.CheckViolation; import com.oceanbase.odc.service.task.TaskService; +import com.oceanbase.odc.service.task.base.precheck.PreCheckTask; +import com.oceanbase.odc.service.task.base.precheck.PreCheckTaskParameters; import com.oceanbase.odc.service.task.constants.JobParametersKeyConstants; import com.oceanbase.odc.service.task.enums.JobStatus; import com.oceanbase.odc.service.task.model.ExecutorInfo; -import com.oceanbase.odc.service.task.runtime.PreCheckTask; -import com.oceanbase.odc.service.task.runtime.PreCheckTaskParameters; import com.oceanbase.odc.service.task.schedule.DefaultJobDefinition; import com.oceanbase.odc.service.task.schedule.JobDefinition; import com.oceanbase.odc.service.task.schedule.JobScheduler; @@ -141,7 +141,8 @@ protected Void start(Long taskId, TaskService taskService, DelegateExecution exe if (jobEntity.getStatus() != JobStatus.DONE) { throw new ServiceTaskError(new RuntimeException("Pre-check task failed")); } - this.preCheckResult = JsonUtils.fromJson(jobEntity.getResultJson(), PreCheckTaskResult.class); + this.preCheckResult = + JsonUtils.fromJson(JobUtils.retrieveJobResultStr(jobEntity), PreCheckTaskResult.class); if (Objects.nonNull(this.preCheckResult)) { this.preCheckResult.setExecutorInfo(new ExecutorInfo(this.hostProperties)); storeTaskResultToFile(this.preCheckResult.getSqlCheckResult()); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/RollbackPlanRuntimeFlowableTaskCopied.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/RollbackPlanRuntimeFlowableTaskCopied.java index d42a909700..d9b9797c16 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/RollbackPlanRuntimeFlowableTaskCopied.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/flow/task/RollbackPlanRuntimeFlowableTaskCopied.java @@ -43,10 +43,10 @@ import com.oceanbase.odc.service.objectstorage.model.ObjectMetadata; import com.oceanbase.odc.service.rollbackplan.model.RollbackProperties; import com.oceanbase.odc.service.task.TaskService; +import com.oceanbase.odc.service.task.base.rollback.RollbackPlanTask; +import com.oceanbase.odc.service.task.base.rollback.RollbackPlanTaskParameters; import com.oceanbase.odc.service.task.constants.JobParametersKeyConstants; import com.oceanbase.odc.service.task.enums.JobStatus; -import com.oceanbase.odc.service.task.runtime.RollbackPlanTask; -import com.oceanbase.odc.service.task.runtime.RollbackPlanTaskParameters; import com.oceanbase.odc.service.task.schedule.DefaultJobDefinition; import com.oceanbase.odc.service.task.schedule.JobDefinition; import com.oceanbase.odc.service.task.schedule.JobScheduler; @@ -94,7 +94,8 @@ protected RollbackPlanTaskResult start(Long taskId, TaskService taskService, Del if (Objects.isNull(jobEntity) || jobEntity.getStatus() != JobStatus.DONE) { throw new ServiceTaskError(new RuntimeException("Generate rollback plan task failed")); } - RollbackPlanTaskResult result = JsonUtils.fromJson(jobEntity.getResultJson(), RollbackPlanTaskResult.class); + RollbackPlanTaskResult result = + JsonUtils.fromJson(JobUtils.retrieveJobResultStr(jobEntity), RollbackPlanTaskResult.class); DatabaseChangeResult databaseChangeResult = new DatabaseChangeResult(); databaseChangeResult.setRollbackPlanResult(result); taskEntity.setResultJson(JsonUtils.toJson(databaseChangeResult)); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/git/model/FileChangeType.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/git/model/FileChangeType.java index 830b415b40..eb13618555 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/git/model/FileChangeType.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/git/model/FileChangeType.java @@ -31,5 +31,4 @@ public enum FileChangeType { R, // conflict C - } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/GlobalResourceRoleService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/GlobalResourceRoleService.java new file mode 100644 index 0000000000..949b34a5fe --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/GlobalResourceRoleService.java @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.iam; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Set; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.springframework.util.CollectionUtils; + +import com.oceanbase.odc.core.authority.util.SkipAuthorize; +import com.oceanbase.odc.core.shared.constant.ResourceRoleName; +import com.oceanbase.odc.core.shared.constant.ResourceType; +import com.oceanbase.odc.metadb.iam.UserRoleRepository; +import com.oceanbase.odc.service.iam.model.UserGlobalResourceRole; +import com.oceanbase.odc.service.iam.util.GlobalResourceRoleUtil; + +/** + * @Author: Lebie + * @Date: 2024/11/19 15:47 + * @Description: [] + */ +@Service +public class GlobalResourceRoleService { + + @Autowired + private UserRoleRepository userRoleRepository; + + @SkipAuthorize("odc internal usage") + public List findGlobalResourceRoleUsersByOrganizationId(Long organizationId) { + return userRoleRepository.findByOrganizationIdAndNameIn( + organizationId, + Arrays.asList(GlobalResourceRoleUtil.GLOBAL_PROJECT_OWNER, GlobalResourceRoleUtil.GLOBAL_PROJECT_DBA, + GlobalResourceRoleUtil.GLOBAL_PROJECT_SECURITY_ADMINISTRATOR)); + } + + @SkipAuthorize("odc internal usage") + public List findGlobalResourceRoleUsersByOrganizationIdAndUserId(Long organizationId, + Long userId) { + return userRoleRepository.findByOrganizationIdAndUserIdAndNameIn(organizationId, userId, + Arrays.asList(GlobalResourceRoleUtil.GLOBAL_PROJECT_OWNER, GlobalResourceRoleUtil.GLOBAL_PROJECT_DBA, + GlobalResourceRoleUtil.GLOBAL_PROJECT_SECURITY_ADMINISTRATOR)); + } + + @SkipAuthorize("odc internal usage") + public List findGlobalResourceRoleUsersByOrganizationIdAndRole(Long organizationId, + ResourceType resourceType, ResourceRoleName resourceRoleName) { + if (resourceType != ResourceType.ODC_PROJECT) { + return Collections.emptyList(); + } + return userRoleRepository.findByOrganizationIdAndNameIn( + organizationId, Arrays.asList(GlobalResourceRoleUtil.getGlobalRoleName(resourceRoleName))); + } + + @SkipAuthorize("odc internal usage") + public List findGlobalResourceRoleUsersByOrganizationIdAndRoleIn(Long organizationId, + Set resourceRoleNames) { + if (CollectionUtils.isEmpty(resourceRoleNames)) { + return Collections.emptyList(); + } + return userRoleRepository.findByOrganizationIdAndNameIn(organizationId, + GlobalResourceRoleUtil.getGlobalRoleName(resourceRoleNames)); + } +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/PermissionService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/PermissionService.java index 24d42461f9..681a1f06b1 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/PermissionService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/PermissionService.java @@ -120,5 +120,4 @@ public void deleteExpiredPermission(Date expiredTime) { log.info("Clear expired permission, count: {}, expired time: {}", count, expiredTime); } } - } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/ProjectPermissionValidator.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/ProjectPermissionValidator.java index 7aed0759ab..42299ac19b 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/ProjectPermissionValidator.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/ProjectPermissionValidator.java @@ -15,6 +15,7 @@ */ package com.oceanbase.odc.service.iam; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.List; @@ -25,7 +26,9 @@ import org.springframework.stereotype.Component; import com.oceanbase.odc.core.authority.model.DefaultSecurityResource; +import com.oceanbase.odc.core.authority.permission.ComposedPermission; import com.oceanbase.odc.core.authority.permission.Permission; +import com.oceanbase.odc.core.authority.permission.ProjectPermission; import com.oceanbase.odc.core.authority.permission.ResourceRoleBasedPermission; import com.oceanbase.odc.core.authority.util.SkipAuthorize; import com.oceanbase.odc.core.shared.constant.ResourceRoleName; @@ -73,9 +76,15 @@ public boolean hasProjectRole(@NonNull Collection projectIds, @NonNull Lis return false; } List permissions = projectIds.stream().filter(Objects::nonNull) - .map(projectId -> new ResourceRoleBasedPermission( - new DefaultSecurityResource(projectId.toString(), "ODC_PROJECT"), roleNames)) - .collect(Collectors.toList()); + .map(projectId -> { + Permission resourceRolePermission = new ResourceRoleBasedPermission( + new DefaultSecurityResource(projectId.toString(), "ODC_PROJECT"), roleNames); + Permission projectResourcePermission = + new ProjectPermission(new DefaultSecurityResource(projectId.toString(), "ODC_PROJECT"), + roleNames.stream().map(ResourceRoleName::name).collect( + Collectors.toList())); + return new ComposedPermission(Arrays.asList(resourceRolePermission, projectResourcePermission)); + }).collect(Collectors.toList()); return authorizationFacade.isImpliesPermissions(authenticationFacade.currentUser(), permissions); } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/ResourceRoleBasedPermissionExtractor.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/ResourceRoleBasedPermissionExtractor.java index 61e5f2bbd1..418ad3f632 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/ResourceRoleBasedPermissionExtractor.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/ResourceRoleBasedPermissionExtractor.java @@ -49,11 +49,11 @@ public class ResourceRoleBasedPermissionExtractor { @Autowired private ResourceRoleRepository repository; - public List getResourcePermissions(List entities) { + public List getResourcePermissions(List entities) { if (CollectionUtils.isEmpty(entities)) { return Collections.EMPTY_LIST; } - List permissions = new ArrayList<>(); + List permissions = new ArrayList<>(); Map> resourceId2Entities = entities.stream().collect(Collectors.groupingBy(UserResourceRoleEntity::getResourceId)); for (Entry> entry : resourceId2Entities.entrySet()) { @@ -70,7 +70,7 @@ public List getResourcePermissions(List saveAll(List userResourceRoleLis public Set getResourceRoleIdentifiersByUserId(long organizationId, long userId) { List userResourceRoleEntities = userResourceRoleRepository.findByOrganizationIdAndUserId(organizationId, userId); - if (CollectionUtils.isEmpty(userResourceRoleEntities)) { - return Collections.emptySet(); - } - return userResourceRoleEntities.stream() + List globalResourceRoles = + globalResourceRoleService.findGlobalResourceRoleUsersByOrganizationIdAndUserId(organizationId, userId) + .stream().map(UserGlobalResourceRole::getResourceRole).collect(Collectors.toList()); + + Set resourceRoleIdentifiers = userResourceRoleEntities.stream() .map(i -> StringUtils.join(i.getResourceId(), ":", i.getResourceRoleId())) .collect(Collectors.toSet()); + if (CollectionUtils.isEmpty(globalResourceRoles)) { + return resourceRoleIdentifiers; + } + // Has global resource role + Map resourceRoleName2Id = resourceRoleRepository.findByResourceType(ResourceType.ODC_PROJECT) + .stream().map(resourceRoleMapper::entityToModel) + .collect(Collectors.toMap(role -> role.getRoleName().name(), ResourceRole::getId, (v1, v2) -> v2)); + projectRepository.findAllByOrganizationId(organizationId).stream() + .forEach(p -> globalResourceRoles.stream() + .map(r -> StringUtils.join(p.getId(), ":", resourceRoleName2Id.get(r.name()))) + .forEach(resourceRoleIdentifiers::add)); + return resourceRoleIdentifiers; } @SkipAuthorize public Map> getProjectId2ResourceRoleNames() { - return getProjectId2ResourceRoleNames(authenticationFacade.currentUserId()); + return getProjectId2ResourceRoleNames(authenticationFacade.currentUserId(), + authenticationFacade.currentOrganizationId()); } @SkipAuthorize - public Map> getProjectId2ResourceRoleNames(Long userId) { + public Map> getProjectId2ResourceRoleNames(Long userId, Long organizationId) { Map id2ResourceRoles = resourceRoleRepository.findByResourceType(ResourceType.ODC_PROJECT) .stream().map(resourceRoleMapper::entityToModel) .collect(Collectors.toMap(ResourceRole::getId, resourceRole -> resourceRole, (v1, v2) -> v2)); - return userResourceRoleRepository.findByUserIdAndResourceType(userId, ResourceType.ODC_PROJECT).stream() + Map> result = userResourceRoleRepository + .findByUserIdAndResourceTypeAndOrganizationId(userId, ResourceType.ODC_PROJECT, organizationId).stream() .collect(Collectors.groupingBy(UserResourceRoleEntity::getResourceId, Collectors.mapping( e -> id2ResourceRoles.get(e.getResourceRoleId()).getRoleName(), Collectors.toSet()))); + Set globalResourceRoles = + globalResourceRoleService.findGlobalResourceRoleUsersByOrganizationIdAndUserId(organizationId, userId) + .stream() + .map(UserGlobalResourceRole::getResourceRole).collect(Collectors.toSet()); + if (CollectionUtils.isEmpty(globalResourceRoles)) { + return result; + } + projectRepository.findAllByOrganizationId(organizationId).stream().forEach(p -> { + if (!result.containsKey(p.getId())) { + result.put(p.getId(), globalResourceRoles); + } else { + result.get(p.getId()).addAll(globalResourceRoles); + } + }); + return result; } @SkipAuthorize("internal usage") @@ -138,15 +180,54 @@ public Optional findResourceRoleById(@NonNull Long id) { @Transactional(rollbackFor = Exception.class) @SkipAuthorize("internal usage") - public List listByResourceTypeAndId(ResourceType resourceType, Long resourceId) { - return fromEntities(userResourceRoleRepository.listByResourceTypeAndId(resourceType, resourceId)); + public List listByResourceTypeAndResourceId(ResourceType resourceType, Long resourceId) { + List userResourceRoles = + fromEntities(userResourceRoleRepository.listByResourceTypeAndId(resourceType, resourceId)); + if (resourceType == ResourceType.ODC_DATABASE) { + return userResourceRoles; + } + List globalResourceRoles = + globalResourceRoleService + .findGlobalResourceRoleUsersByOrganizationId(authenticationFacade.currentOrganizationId()); + if (CollectionUtils.isEmpty(globalResourceRoles)) { + return userResourceRoles; + } + Map resourceRoleName2Id = getProjectResourceRoleName2Id(); + globalResourceRoles.stream().map( + i -> new UserResourceRole(i.getUserId(), resourceId, ResourceType.ODC_PROJECT, i.getResourceRole(), + resourceRoleName2Id.get(i.getResourceRole()), true)) + .forEach(userResourceRoles::add); + return userResourceRoles; } @Transactional(rollbackFor = Exception.class) @SkipAuthorize("internal usage") - public List listByResourceTypeAndIdIn(ResourceType resourceType, + public Set listUserIdsByResourceTypeAndResourceId(ResourceType resourceType, Long resourceId) { + return listByResourceTypeAndResourceId(resourceType, resourceId).stream().map(UserResourceRole::getUserId) + .collect(Collectors.toSet()); + } + + @Transactional(rollbackFor = Exception.class) + @SkipAuthorize("internal usage") + public List listByResourceTypeAndResourceIdIn(ResourceType resourceType, @NotEmpty Collection resourceIds) { - return fromEntities(userResourceRoleRepository.listByResourceTypeAndIdIn(resourceType, resourceIds)); + List userResourceRoles = + fromEntities(userResourceRoleRepository.listByResourceTypeAndIdIn(resourceType, resourceIds)); + if (resourceType == ResourceType.ODC_DATABASE) { + return userResourceRoles; + } + List globalResourceRoles = + globalResourceRoleService + .findGlobalResourceRoleUsersByOrganizationId(authenticationFacade.currentOrganizationId()); + if (CollectionUtils.isEmpty(globalResourceRoles)) { + return userResourceRoles; + } + Map resourceRoleName2Id = getProjectResourceRoleName2Id(); + globalResourceRoles.stream().flatMap(i -> resourceIds.stream().map( + resourceId -> new UserResourceRole(i.getUserId(), resourceId, ResourceType.ODC_PROJECT, + i.getResourceRole(), resourceRoleName2Id.get(i.getResourceRole()), true))) + .forEach(userResourceRoles::add); + return userResourceRoles; } @Transactional(rollbackFor = Exception.class) @@ -183,19 +264,105 @@ public List listResourceRoles(List resourceType) { @SkipAuthorize("internal authenticated") public List listByOrganizationIdAndUserId(Long organizationId, Long userId) { - return fromEntities(userResourceRoleRepository.findByOrganizationIdAndUserId(organizationId, userId)); + List userResourceRoles = + fromEntities(userResourceRoleRepository.findByOrganizationIdAndUserId(organizationId, userId)); + List globalResourceRoles = + globalResourceRoleService.findGlobalResourceRoleUsersByOrganizationIdAndUserId(organizationId, userId); + if (CollectionUtils.isEmpty(globalResourceRoles)) { + return userResourceRoles; + } + Map resourceRoleName2Id = getProjectResourceRoleName2Id(); + projectRepository.findAllByOrganizationId(organizationId).stream() + .forEach(p -> globalResourceRoles.stream() + .map(i -> new UserResourceRole(i.getUserId(), p.getId(), ResourceType.ODC_PROJECT, + i.getResourceRole(), resourceRoleName2Id.get(i.getResourceRole()), true)) + .forEach(userResourceRoles::add)); + return userResourceRoles; } @SkipAuthorize("internal usage") public List listByUserId(Long userId) { - return fromEntities(userResourceRoleRepository.findByUserId(userId)); + List userResourceRoles = fromEntities(userResourceRoleRepository.findByUserId(userId)); + List globalResourceRoles = + globalResourceRoleService.findGlobalResourceRoleUsersByOrganizationIdAndUserId( + authenticationFacade.currentOrganizationId(), + userId); + if (CollectionUtils.isEmpty(globalResourceRoles)) { + return userResourceRoles; + } + Map resourceRoleName2Id = getProjectResourceRoleName2Id(); + projectRepository.findAllByOrganizationId(authenticationFacade.currentOrganizationId()).stream() + .forEach(p -> globalResourceRoles.stream() + .map(i -> new UserResourceRole(i.getUserId(), p.getId(), ResourceType.ODC_PROJECT, + i.getResourceRole(), resourceRoleName2Id.get(i.getResourceRole()), true)) + .forEach(userResourceRoles::add)); + return userResourceRoles; } @SkipAuthorize("internal usage") public List listByResourceIdAndTypeAndName(Long resourceId, ResourceType resourceType, String roleName) { - return fromEntities( + List userResourceRoles = fromEntities( userResourceRoleRepository.findByResourceIdAndTypeAndName(resourceId, resourceType, roleName)); + if (resourceType == ResourceType.ODC_DATABASE) { + return userResourceRoles; + } + List globalResourceRoles = + globalResourceRoleService.findGlobalResourceRoleUsersByOrganizationIdAndRole( + authenticationFacade.currentOrganizationId(), resourceType, ResourceRoleName.valueOf(roleName)); + if (CollectionUtils.isEmpty(globalResourceRoles)) { + return userResourceRoles; + } + Map resourceRoleName2Id = getProjectResourceRoleName2Id(); + globalResourceRoles.stream().map(i -> new UserResourceRole(i.getUserId(), resourceId, resourceType, + i.getResourceRole(), resourceRoleName2Id.get(i.getResourceRole()), true)) + .forEach(userResourceRoles::add); + return userResourceRoles; + } + + @SkipAuthorize("internal usage") + public List listByResourceIdentifierIn(Set resourceIdentifiers) { + List userResourceRoles = + fromEntities(userResourceRoleRepository.findByResourceIdsAndResourceRoleIdsIn(resourceIdentifiers)); + List globalUserResourceRoles = globalResourceRoleService + .findGlobalResourceRoleUsersByOrganizationIdAndRoleIn(authenticationFacade.currentOrganizationId(), + filterResourceRoleNames(ResourceType.ODC_PROJECT, resourceIdentifiers)); + if (CollectionUtils.isEmpty(globalUserResourceRoles)) { + return userResourceRoles; + } + Map resourceRoleName2Id = getProjectResourceRoleName2Id(); + projectRepository.findAllByOrganizationId(authenticationFacade.currentOrganizationId()).stream() + .forEach(p -> globalUserResourceRoles.stream() + .map(i -> new UserResourceRole(i.getUserId(), p.getId(), ResourceType.ODC_PROJECT, + i.getResourceRole(), resourceRoleName2Id.get(i.getResourceRole()), true)) + .forEach(userResourceRoles::add)); + return userResourceRoles; + } + + private Map getProjectResourceRoleName2Id() { + return resourceRoleRepository.findByResourceType(ResourceType.ODC_PROJECT).stream().collect(Collectors.toMap( + ResourceRoleEntity::getRoleName, ResourceRoleEntity::getId, (v1, v2) -> v2)); + } + + private Set filterResourceRoleNames(ResourceType resourceType, Set resourceIdentifiers) { + if (CollectionUtils.isEmpty(resourceIdentifiers)) { + return Collections.emptySet(); + } + Map id2ResourceRoleName = resourceRoleRepository.findByResourceType(resourceType) + .stream() + .collect(Collectors.toMap(ResourceRoleEntity::getId, ResourceRoleEntity::getRoleName, (v1, v2) -> v2)); + Set filtered = new HashSet<>(); + resourceIdentifiers.stream().forEach(identifier -> { + String[] parts = identifier.split(":"); + if (parts.length != 2) { + throw new UnexpectedException("invalid resource identifier, identifier=" + identifier); + } + Long roleId = Long.parseLong(parts[1]); + if (id2ResourceRoleName.containsKey(roleId)) { + filtered.add(id2ResourceRoleName.get(roleId)); + } + }); + return filtered; } private List fromEntities(Collection entities) { @@ -219,7 +386,17 @@ private UserResourceRole fromEntity(UserResourceRoleEntity entity, ResourceRoleE model.setResourceType(resourceRole.getResourceType()); model.setResourceId(entity.getResourceId()); model.setUserId(entity.getUserId()); + model.setResourceRoleId(resourceRole.getId()); return model; } + @SkipAuthorize("internal usage") + public static UserResourceRoleEntity toEntity(UserResourceRole model) { + UserResourceRoleEntity entity = new UserResourceRoleEntity(); + entity.setResourceId(model.getResourceId()); + entity.setUserId(model.getUserId()); + entity.setResourceRoleId(model.getResourceRoleId()); + return entity; + } + } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/UserPermissionService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/UserPermissionService.java index f1aebab9a1..2a8756bc35 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/UserPermissionService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/UserPermissionService.java @@ -148,5 +148,4 @@ public void bindUserAndDataSourcePermission(@NonNull Long userId, @NonNull Long }); } - } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/UserService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/UserService.java index 8d83c9e701..5e8b77c7d5 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/UserService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/UserService.java @@ -25,7 +25,6 @@ import java.util.Collections; import java.util.HashMap; import java.util.HashSet; -import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -43,7 +42,6 @@ import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.io.IOUtils; -import org.apache.commons.lang.Validate; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.i18n.LocaleContextHolder; import org.springframework.data.domain.Page; @@ -65,7 +63,6 @@ import com.github.benmanes.caffeine.cache.LoadingCache; import com.google.common.base.MoreObjects; import com.oceanbase.odc.common.util.StringUtils; -import com.oceanbase.odc.core.authority.permission.Permission; import com.oceanbase.odc.core.authority.permission.ResourcePermission; import com.oceanbase.odc.core.authority.util.Authenticated; import com.oceanbase.odc.core.authority.util.PreAuthenticate; @@ -328,7 +325,6 @@ public User create(@NotNull @Valid CreateUserReq createUserReq) { log.debug("New user has been inserted, user: {}", userEntity); if (!Objects.isNull(createUserReq.getRoleIds()) && !createUserReq.getRoleIds().isEmpty()) { - inspectVerticalUnauthorized(authenticationFacade.currentUser(), createUserReq.getRoleIds()); for (Long roleId : createUserReq.getRoleIds()) { Role role = new Role(roleRepository.findById(roleId) .orElseThrow(() -> new NotFoundException(ResourceType.ODC_ROLE, "id", roleId))); @@ -466,12 +462,18 @@ public BinaryDataResult getBatchImportTemplateFile() throws IOException { } } + @SkipAuthorize("odc internal usage") public Set getCurrentUserResourceRoleIdentifiers() { long currentUserId = authenticationFacade.currentUserId(); long currentOrganizationId = authenticationFacade.currentOrganizationId(); return resourceRoleService.getResourceRoleIdentifiersByUserId(currentOrganizationId, currentUserId); } + @SkipAuthorize("odc internal usage") + public Set getCurrentUserJoinedProjectIds() { + return resourceRoleService.getProjectId2ResourceRoleNames().keySet(); + } + private void acquirePermissions(@NonNull Collection users) { List managementPermissions = new ArrayList<>(); List operationPermissions = new ArrayList<>(); @@ -603,6 +605,7 @@ public PaginatedData listUserBasicsWithoutPermissionCheck() { List users = userRepository.findByOrganizationId(authenticationFacade.currentOrganizationId()).stream() .map(User::new).collect(Collectors.toList()); + acquireRolesAndRoleIds(users); return new PaginatedData<>(users, CustomPage.empty()); } @@ -744,7 +747,6 @@ public User update(long id, UpdateUserReq updateUserReq) { attachedRoleIds = relations.stream().map(UserRoleEntity::getRoleId).collect(Collectors.toSet()); } Long creatorId = authenticationFacade.currentUserId(); - inspectVerticalUnauthorized(authenticationFacade.currentUser(), updateUserReq.getRoleIds()); userRoleRepository.deleteByOrganizationIdAndUserId(authenticationFacade.currentOrganizationId(), id); userRoleRepository.flush(); for (Long roleId : updateUserReq.getRoleIds()) { @@ -921,19 +923,6 @@ public static class UserDeleteEvent { private String accountName; } - private void inspectVerticalUnauthorized(User operator, List roleIdsToBeAttached) { - Validate.notNull(roleIdsToBeAttached, - "RoleIdsToBeAttached can not be null for UserServcei#inspectVerticalUnauthorized"); - List permissions = new LinkedList<>( - permissionMapper.getResourcePermissions(permissionRepository.findByRoleIds(roleIdsToBeAttached))); - boolean checkResult = - authorizationFacade.isImpliesPermissions(operator, permissions); - if (!checkResult) { - String errMsg = "Cannot grant permissions that the current user does not have"; - throw new BadRequestException(ErrorCodes.GrantPermissionFailed, new Object[] {errMsg}, errMsg); - } - } - private List getUserIdsByRoleIds(@NonNull List roleIds) { List userIds = new ArrayList<>(); if (CollectionUtils.isEmpty(roleIds)) { @@ -1029,7 +1018,6 @@ private void batchBindRoles(List createUserReqs, Map(roleIds)); Map roleId2Entity = roleRepository.findByIdIn(roleIds).stream() .collect(Collectors.toMap(RoleEntity::getId, entity -> entity)); Verify.equals(roleIds.size(), roleId2Entity.keySet().size(), "roleIds.size()"); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/auth/BaseAuthorizer.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/auth/BaseAuthorizer.java index e9d7fff075..df871b7d45 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/auth/BaseAuthorizer.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/auth/BaseAuthorizer.java @@ -17,6 +17,7 @@ import java.security.Principal; import java.util.Collection; +import java.util.List; import java.util.Objects; import com.oceanbase.odc.core.authority.auth.Authorizer; @@ -43,6 +44,28 @@ public void checkPermission(Principal principal, Collection permissi } } + @Override + public boolean isPermitted(Principal principal, Collection permissions, SecurityContext context) { + User odcUser = (User) principal; + if (Objects.isNull(odcUser.getId())) { + return false; + } + List permittedPermissions = listPermittedPermissions(principal); + for (Permission permission : permissions) { + boolean accessDenied = true; + for (Permission resourcePermission : permittedPermissions) { + if (resourcePermission.implies(permission)) { + accessDenied = false; + break; + } + } + if (accessDenied) { + return false; + } + } + return true; + } + /** * An Authorizer may not work for all Principal of the * Subject, so a method is needed to indicate whether a certain Authorizer @@ -55,4 +78,6 @@ public void checkPermission(Principal principal, Collection permissi public boolean supports(Class principal) { return User.class.isAssignableFrom(principal); } + + protected abstract List listPermittedPermissions(Principal principal); } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/auth/ComposedAuthorizer.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/auth/ComposedAuthorizer.java new file mode 100644 index 0000000000..74e347f8e9 --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/auth/ComposedAuthorizer.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.iam.auth; + +import java.security.Principal; +import java.util.Collections; +import java.util.List; + +import org.apache.commons.collections.ListUtils; + +import com.oceanbase.odc.core.authority.permission.ComposedPermission; +import com.oceanbase.odc.core.authority.permission.Permission; + +/** + * @Author: Lebie + * @Date: 2024/11/11 16:05 + * @Description: [] + */ +public class ComposedAuthorizer extends BaseAuthorizer { + private final DefaultAuthorizer defaultAuthorizer; + private final ResourceRoleAuthorizer resourceRoleAuthorizer; + + public ComposedAuthorizer(DefaultAuthorizer defaultAuthorizer, ResourceRoleAuthorizer resourceRoleAuthorizer) { + this.defaultAuthorizer = defaultAuthorizer; + this.resourceRoleAuthorizer = resourceRoleAuthorizer; + } + + @Override + protected List listPermittedPermissions(Principal principal) { + return Collections.singletonList( + new ComposedPermission(ListUtils.union(defaultAuthorizer.listPermittedPermissions(principal), + resourceRoleAuthorizer.listPermittedPermissions(principal)))); + } +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/auth/DefaultAuthorizationFacade.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/auth/DefaultAuthorizationFacade.java index b033bfb432..e27222b67b 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/auth/DefaultAuthorizationFacade.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/auth/DefaultAuthorizationFacade.java @@ -42,12 +42,13 @@ import org.springframework.data.jpa.domain.Specification; import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; +import com.oceanbase.odc.core.authority.SecurityManager; import com.oceanbase.odc.core.authority.model.DefaultSecurityResource; import com.oceanbase.odc.core.authority.model.SecurityResource; import com.oceanbase.odc.core.authority.permission.ConnectionPermission; import com.oceanbase.odc.core.authority.permission.DatabasePermission; import com.oceanbase.odc.core.authority.permission.Permission; -import com.oceanbase.odc.core.authority.permission.PrivateConnectionPermission; +import com.oceanbase.odc.core.authority.permission.ProjectPermission; import com.oceanbase.odc.core.authority.permission.ResourcePermission; import com.oceanbase.odc.core.authority.permission.ResourceRoleBasedPermission; import com.oceanbase.odc.core.shared.constant.PermissionType; @@ -59,9 +60,9 @@ import com.oceanbase.odc.metadb.iam.UserEntity; import com.oceanbase.odc.metadb.iam.UserRepository; import com.oceanbase.odc.metadb.iam.resourcerole.UserResourceRoleEntity; -import com.oceanbase.odc.metadb.iam.resourcerole.UserResourceRoleRepository; import com.oceanbase.odc.service.iam.ResourcePermissionExtractor; import com.oceanbase.odc.service.iam.ResourceRoleBasedPermissionExtractor; +import com.oceanbase.odc.service.iam.ResourceRoleService; import com.oceanbase.odc.service.iam.model.User; import com.oceanbase.odc.service.resourcegroup.model.ResourceIdentifier; @@ -78,7 +79,7 @@ public abstract class DefaultAuthorizationFacade implements AuthorizationFacade @Autowired private PermissionRepository repository; @Autowired - private UserResourceRoleRepository resourceRoleService; + private ResourceRoleService resourceRoleService; @Autowired private UserRepository userRepository; @Autowired @@ -86,6 +87,9 @@ public abstract class DefaultAuthorizationFacade implements AuthorizationFacade @Autowired @Qualifier("authorizationFacadeExecutor") private ThreadPoolTaskExecutor authorizationFacadeExecutor; + @Autowired + @Qualifier("servletSecurityManager") + private SecurityManager securityManager; @Override public Set getAllPermittedActions(Principal principal, ResourceType resourceType, String resourceId) { @@ -96,10 +100,7 @@ public Set getAllPermittedActions(Principal principal, ResourceType reso if (resourceType.equals(ResourceType.valueOf(((ResourcePermission) permission).getResourceType())) && ("*".equals(((ResourcePermission) permission).getResourceId()) || resourceId.equals(((ResourcePermission) permission).getResourceId()))) { - if (resourceType == ResourceType.ODC_PRIVATE_CONNECTION) { - returnVal.addAll( - PrivateConnectionPermission.getActionList(((ResourcePermission) permission).getMask())); - } else if (resourceType == ResourceType.ODC_CONNECTION) { + if (resourceType == ResourceType.ODC_CONNECTION) { returnVal.addAll( ConnectionPermission.getActionList(((ResourcePermission) permission).getMask())); } else if (resourceType == ResourceType.ODC_DATABASE) { @@ -165,10 +166,10 @@ public Map> getRelatedResourcesAndActions(Principa Set actions = returnVal.computeIfAbsent(resource, identifier -> new HashSet<>()); if (ResourceType.ODC_DATABASE.name().equals(resource.resourceType())) { actions.addAll(DatabasePermission.getActionList(((ResourcePermission) permission).getMask())); - } else if (ResourceType.ODC_PRIVATE_CONNECTION.name().equals(resource.resourceType())) { - actions.addAll(PrivateConnectionPermission.getActionList(((ResourcePermission) permission).getMask())); } else if (ResourceType.ODC_CONNECTION.name().equals(resource.resourceType())) { actions.addAll(ConnectionPermission.getActionList(((ResourcePermission) permission).getMask())); + } else if (ResourceType.ODC_PROJECT.name().equals(resource.resourceType())) { + actions.addAll(((ProjectPermission) permission).getActions()); } else { actions.addAll(ResourcePermission.getActionList(((ResourcePermission) permission).getMask())); } @@ -178,20 +179,7 @@ public Map> getRelatedResourcesAndActions(Principa @Override public boolean isImpliesPermissions(@NotNull Principal principal, @NotNull Collection permissions) { - List permittedPermissions = getAllPermissions(principal); - for (Permission permission : permissions) { - boolean implies = false; - for (Permission permittedPermission : permittedPermissions) { - if (permittedPermission.implies(permission)) { - implies = true; - break; - } - } - if (!implies) { - return false; - } - } - return true; + return this.securityManager.isPermitted(permissions); } private User entityToUser(UserEntity entity) { @@ -217,9 +205,9 @@ private List getAllPermissions(Principal principal) { .stream().filter(permission -> !Objects.isNull(permission)).collect(Collectors.toList()); List resourceRoles = resourceRoleService - .findByOrganizationIdAndUserId(authenticationFacade.currentOrganizationId(), odcUser.getId()) + .listByOrganizationIdAndUserId(authenticationFacade.currentOrganizationId(), odcUser.getId()) .stream() - .filter(Objects::nonNull).collect(Collectors.toList()); + .filter(Objects::nonNull).map(ResourceRoleService::toEntity).collect(Collectors.toList()); return ListUtils.union(permissionMapper.getResourcePermissions(permissionEntityList), resourceRoleBasedPermissionExtractor.getResourcePermissions(resourceRoles)); } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/auth/DefaultAuthorizer.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/auth/DefaultAuthorizer.java index 82f522b734..b831cd617f 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/auth/DefaultAuthorizer.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/auth/DefaultAuthorizer.java @@ -16,13 +16,12 @@ package com.oceanbase.odc.service.iam.auth; import java.security.Principal; -import java.util.Collection; +import java.util.Collections; import java.util.List; import java.util.Objects; import java.util.stream.Collectors; import com.oceanbase.odc.core.authority.auth.Authorizer; -import com.oceanbase.odc.core.authority.auth.SecurityContext; import com.oceanbase.odc.core.authority.permission.Permission; import com.oceanbase.odc.metadb.iam.PermissionEntity; import com.oceanbase.odc.metadb.iam.PermissionRepository; @@ -47,33 +46,15 @@ public DefaultAuthorizer(PermissionRepository repository, ResourcePermissionExtr this.permissionMapper = permissionMapper; } - // Todo 权限增加缓存 @Override - public boolean isPermitted(Principal principal, Collection permissions, SecurityContext context) { + protected List listPermittedPermissions(Principal principal) { User odcUser = (User) principal; if (Objects.isNull(odcUser.getId())) { - return false; + return Collections.emptyList(); } List permissionEntities = repository.findByUserIdAndUserStatusAndRoleStatusAndOrganizationId( odcUser.getId(), true, true, odcUser.getOrganizationId()).stream().filter(Objects::nonNull) .collect(Collectors.toList()); - if (permissionEntities.isEmpty()) { - return false; - } - Collection permissionCollection = - permissionMapper.getResourcePermissions(permissionEntities); - for (Permission permission : permissions) { - boolean accessDenied = true; - for (Permission resourcePermission : permissionCollection) { - if (resourcePermission.implies(permission)) { - accessDenied = false; - break; - } - } - if (accessDenied) { - return false; - } - } - return true; + return permissionMapper.getResourcePermissions(permissionEntities); } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/auth/DefaultPermissionProvider.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/auth/DefaultPermissionProvider.java index 9b3e2a46bc..261d6521b8 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/auth/DefaultPermissionProvider.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/auth/DefaultPermissionProvider.java @@ -15,14 +15,16 @@ */ package com.oceanbase.odc.service.iam.auth; +import java.util.Arrays; import java.util.Collection; import com.oceanbase.odc.core.authority.model.SecurityResource; +import com.oceanbase.odc.core.authority.permission.ComposedPermission; import com.oceanbase.odc.core.authority.permission.ConnectionPermission; import com.oceanbase.odc.core.authority.permission.DatabasePermission; import com.oceanbase.odc.core.authority.permission.Permission; import com.oceanbase.odc.core.authority.permission.PermissionProvider; -import com.oceanbase.odc.core.authority.permission.PrivateConnectionPermission; +import com.oceanbase.odc.core.authority.permission.ProjectPermission; import com.oceanbase.odc.core.authority.permission.ResourcePermission; import com.oceanbase.odc.core.authority.permission.ResourceRoleBasedPermission; import com.oceanbase.odc.core.shared.constant.ResourceType; @@ -41,10 +43,10 @@ public class DefaultPermissionProvider implements PermissionProvider { public Permission getPermissionByActions(SecurityResource resource, Collection actions) { if (ResourceType.ODC_CONNECTION.name().equals(resource.resourceType())) { return new ConnectionPermission(resource.resourceId(), String.join(",", actions)); - } else if (ResourceType.ODC_PRIVATE_CONNECTION.name().equals(resource.resourceType())) { - return new PrivateConnectionPermission(resource.resourceId(), String.join(",", actions)); } else if (ResourceType.ODC_DATABASE.name().equals(resource.resourceType())) { return new DatabasePermission(resource.resourceId(), String.join(",", actions)); + } else if (ResourceType.ODC_PROJECT.name().equals(resource.resourceType())) { + return new ProjectPermission(resource, String.join(",", actions)); } return new ResourcePermission(resource, String.join(",", actions)); } @@ -54,4 +56,11 @@ public Permission getPermissionByResourceRoles(SecurityResource resource, Collec return new ResourceRoleBasedPermission(resource, String.join(",", resourceRoles)); } + @Override + public Permission getPermissionByActionsAndResourceRoles(SecurityResource resource, Collection actions, + Collection resourceRoles) { + return new ComposedPermission(Arrays.asList(getPermissionByActions(resource, actions), + getPermissionByResourceRoles(resource, resourceRoles))); + } + } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/auth/ResourceRoleAuthorizer.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/auth/ResourceRoleAuthorizer.java index 3ba569ce31..2a3ae8b03f 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/auth/ResourceRoleAuthorizer.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/auth/ResourceRoleAuthorizer.java @@ -16,17 +16,15 @@ package com.oceanbase.odc.service.iam.auth; import java.security.Principal; -import java.util.Collection; +import java.util.Collections; import java.util.List; import java.util.Objects; import java.util.stream.Collectors; -import com.oceanbase.odc.core.authority.auth.SecurityContext; import com.oceanbase.odc.core.authority.permission.Permission; -import com.oceanbase.odc.core.authority.permission.ResourceRoleBasedPermission; import com.oceanbase.odc.metadb.iam.resourcerole.UserResourceRoleEntity; -import com.oceanbase.odc.metadb.iam.resourcerole.UserResourceRoleRepository; import com.oceanbase.odc.service.iam.ResourceRoleBasedPermissionExtractor; +import com.oceanbase.odc.service.iam.ResourceRoleService; import com.oceanbase.odc.service.iam.model.User; /** @@ -35,46 +33,30 @@ * @Description: [] */ public class ResourceRoleAuthorizer extends BaseAuthorizer { - protected final UserResourceRoleRepository repository; + protected final ResourceRoleService resourceRoleService; protected final ResourceRoleBasedPermissionExtractor permissionMapper; - public ResourceRoleAuthorizer(UserResourceRoleRepository repository, + public ResourceRoleAuthorizer(ResourceRoleService resourceRoleService, ResourceRoleBasedPermissionExtractor permissionMapper) { - this.repository = repository; + this.resourceRoleService = resourceRoleService; this.permissionMapper = permissionMapper; } - @Override - public boolean isPermitted(Principal principal, Collection permissions, SecurityContext context) { + protected List listPermittedPermissions(Principal principal) { User odcUser = (User) principal; if (Objects.isNull(odcUser.getId())) { - return false; + return Collections.emptyList(); } - /** * find all user-related resource role, and implies with permissions respectively */ List resourceRoles = - repository.findByUserId(odcUser.getId()).stream() - .filter(Objects::nonNull).collect(Collectors.toList()); + resourceRoleService.listByUserId(odcUser.getId()).stream() + .filter(Objects::nonNull).map(ResourceRoleService::toEntity).collect(Collectors.toList()); if (resourceRoles.isEmpty()) { - return false; - } - Collection permissionCollection = - permissionMapper.getResourcePermissions(resourceRoles); - for (Permission permission : permissions) { - boolean accessDenied = true; - for (ResourceRoleBasedPermission resourceRoleBasedPermission : permissionCollection) { - if (resourceRoleBasedPermission.implies(permission)) { - accessDenied = false; - break; - } - } - if (accessDenied) { - return false; - } + return Collections.emptyList(); } - return true; + return permissionMapper.getResourcePermissions(resourceRoles); } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/model/UserGlobalResourceRole.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/model/UserGlobalResourceRole.java new file mode 100644 index 0000000000..a785691968 --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/model/UserGlobalResourceRole.java @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.iam.model; + +import com.oceanbase.odc.core.shared.constant.ResourceRoleName; +import com.oceanbase.odc.service.iam.util.GlobalResourceRoleUtil; + +import lombok.Data; + +/** + * @Author: Lebie + * @Date: 2024/11/19 17:15 + * @Description: [] + */ +@Data +public class UserGlobalResourceRole { + private Long userId; + private ResourceRoleName resourceRole; + + public UserGlobalResourceRole(Long userId, String globalRoleName) { + this.userId = userId; + this.resourceRole = GlobalResourceRoleUtil.getResourceRoleName(globalRoleName); + } +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/model/UserResourceRole.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/model/UserResourceRole.java index b2896050d8..d1ee19428c 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/model/UserResourceRole.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/model/UserResourceRole.java @@ -44,6 +44,10 @@ public class UserResourceRole implements PermissionConfiguration { private ResourceRoleName resourceRole; + private Long resourceRoleId; + + private boolean derivedFromGlobalProjectRole = false; + public boolean isProjectMember() { return this.resourceType == ResourceType.ODC_PROJECT && (this.resourceRole == ResourceRoleName.OWNER || this.resourceRole == ResourceRoleName.DBA diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/util/GlobalResourceRoleUtil.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/util/GlobalResourceRoleUtil.java new file mode 100644 index 0000000000..77b82b9f52 --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/iam/util/GlobalResourceRoleUtil.java @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.iam.util; + +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import com.oceanbase.odc.core.shared.constant.ResourceRoleName; + +/** + * @Author: Lebie + * @Date: 2024/11/19 16:05 + * @Description: [] + */ +public class GlobalResourceRoleUtil { + public final static String GLOBAL_PROJECT_OWNER = "global_project_owner"; + public final static String GLOBAL_PROJECT_DBA = "global_project_dba"; + public final static String GLOBAL_PROJECT_SECURITY_ADMINISTRATOR = "global_project_security_administrator"; + private final static Map globalRoleName2ResourceRoleName = new HashMap<>(); + private final static Map resourceRoleName2GlobalRoleName = new HashMap<>(); + + static { + globalRoleName2ResourceRoleName.put(GLOBAL_PROJECT_OWNER, ResourceRoleName.OWNER); + globalRoleName2ResourceRoleName.put(GLOBAL_PROJECT_DBA, ResourceRoleName.DBA); + globalRoleName2ResourceRoleName.put(GLOBAL_PROJECT_SECURITY_ADMINISTRATOR, + ResourceRoleName.SECURITY_ADMINISTRATOR); + + resourceRoleName2GlobalRoleName.put(ResourceRoleName.OWNER, GLOBAL_PROJECT_OWNER); + resourceRoleName2GlobalRoleName.put(ResourceRoleName.DBA, GLOBAL_PROJECT_DBA); + resourceRoleName2GlobalRoleName.put(ResourceRoleName.SECURITY_ADMINISTRATOR, + GLOBAL_PROJECT_SECURITY_ADMINISTRATOR); + } + + public static ResourceRoleName getResourceRoleName(String globalRoleName) { + return globalRoleName2ResourceRoleName.get(globalRoleName); + } + + public static String getGlobalRoleName(ResourceRoleName resourceRoleName) { + return resourceRoleName2GlobalRoleName.get(resourceRoleName); + } + + public static Set getGlobalRoleName(Set resourceRoleNames) { + return resourceRoleNames.stream().map(resourceRoleName2GlobalRoleName::get).collect(Collectors.toSet()); + } + +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/IntegrationConfigurationValidator.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/IntegrationConfigurationProcessor.java similarity index 53% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/integration/IntegrationConfigurationValidator.java rename to server/odc-service/src/main/java/com/oceanbase/odc/service/integration/IntegrationConfigurationProcessor.java index 5c97f784eb..2f9efbdcd1 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/IntegrationConfigurationValidator.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/IntegrationConfigurationProcessor.java @@ -36,6 +36,9 @@ import com.oceanbase.odc.service.integration.model.IntegrationConfig; import com.oceanbase.odc.service.integration.model.SSOIntegrationConfig; import com.oceanbase.odc.service.integration.model.SqlInterceptorProperties; +import com.oceanbase.odc.service.integration.saml.SamlCredentialManager; +import com.oceanbase.odc.service.integration.saml.SamlParameter; +import com.oceanbase.odc.service.integration.saml.SamlParameter.SecretInfo; /** * @author gaoda.xy @@ -43,22 +46,58 @@ */ @Component @Validated -public class IntegrationConfigurationValidator { +public class IntegrationConfigurationProcessor { @Autowired private IntegrationRepository integrationRepository; + @Autowired + private SamlCredentialManager samlCredentialManager; + public void check(@NotNull @Valid ApprovalProperties properties) {} public void check(@NotNull @Valid SqlInterceptorProperties properties) {} - public void checkAndFillConfig(@NotNull @Valid IntegrationConfig config, Long organizationId, Boolean enabled, + public void checkAndFillConfig(@NotNull @Valid IntegrationConfig config, @Nullable IntegrationConfig savedConfig, + Long organizationId, Boolean enabled, @Nullable Long integrationId) { SSOIntegrationConfig ssoIntegrationConfig = SSOIntegrationConfig.of(config, organizationId); + fillSamlSecret(config, savedConfig, organizationId, ssoIntegrationConfig); config.setConfiguration(JsonUtils.toJson(ssoIntegrationConfig)); checkNotEnabledInDbBeforeSave(enabled, organizationId, integrationId); } + public void fillSamlSecret(IntegrationConfig config, IntegrationConfig savedConfig, Long organizationId, + SSOIntegrationConfig ssoIntegrationConfig) { + if ("SAML".equals(ssoIntegrationConfig.getType()) && config.getEncryption().getSecret() == null) { + SecretInfo secretInfo = new SecretInfo(); + SamlParameter newParameter = (SamlParameter) ssoIntegrationConfig.getSsoParameter(); + if (savedConfig != null) { + SecretInfo savedSecretInfo = + JsonUtils.fromJson(savedConfig.getEncryption().getSecret(), SecretInfo.class); + SSOIntegrationConfig savedSsoConfig = SSOIntegrationConfig.of(savedConfig, organizationId); + SamlParameter savedParameter = (SamlParameter) savedSsoConfig.getSsoParameter(); + if (Objects.equals(savedParameter.getSigning().getCertificate(), + newParameter.getSigning().getCertificate())) { + secretInfo.setSigningPrivateKey(savedSecretInfo.getSigningPrivateKey()); + } + if (Objects.equals(savedParameter.getDecryption().getCertificate(), + newParameter.getDecryption().getCertificate())) { + secretInfo.setDecryptionPrivateKey(savedSecretInfo.getDecryptionPrivateKey()); + } + } + String certificate = newParameter.getSigning().getCertificate(); + if (secretInfo.getSigningPrivateKey() == null && certificate != null) { + secretInfo.setSigningPrivateKey(samlCredentialManager.getPrivateKeyByCert(certificate)); + } + String decryptionCertificate = newParameter.getDecryption().getCertificate(); + if (secretInfo.getDecryptionPrivateKey() == null && decryptionCertificate != null) { + secretInfo.setDecryptionPrivateKey(samlCredentialManager.getPrivateKeyByCert(decryptionCertificate)); + } + config.getEncryption().setSecret(JsonUtils.toJson(secretInfo)); + } + } + public void checkNotEnabledInDbBeforeSave(Boolean enabled, Long organizationId, @Nullable Long integrationId) { if (Boolean.TRUE.equals(enabled)) { List dbSSO = integrationRepository.findByTypeAndOrganizationId(SSO, diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/IntegrationConfigurationValidatorDelegate.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/IntegrationConfigurationProcessorDelegate.java similarity index 62% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/integration/IntegrationConfigurationValidatorDelegate.java rename to server/odc-service/src/main/java/com/oceanbase/odc/service/integration/IntegrationConfigurationProcessorDelegate.java index dcad7029e3..79545da5a3 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/IntegrationConfigurationValidatorDelegate.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/IntegrationConfigurationProcessorDelegate.java @@ -15,6 +15,10 @@ */ package com.oceanbase.odc.service.integration; +import java.util.Optional; + +import javax.annotation.Nullable; + import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @@ -25,22 +29,24 @@ import com.oceanbase.odc.service.integration.model.SqlInterceptorProperties; @Component -public class IntegrationConfigurationValidatorDelegate { +public class IntegrationConfigurationProcessorDelegate { @Autowired - private IntegrationConfigurationValidator configurationValidator; + private IntegrationConfigurationProcessor configurationValidator; @Autowired private AuthenticationFacade authenticationFacade; - public void preProcessConfig(IntegrationConfig config) { - if (config.getType() == IntegrationType.APPROVAL) { - configurationValidator.check(ApprovalProperties.from(config)); - } else if (config.getType() == IntegrationType.SQL_INTERCEPTOR) { - configurationValidator.check(SqlInterceptorProperties.from(config)); - } else if (config.getType() == IntegrationType.SSO) { - configurationValidator.checkAndFillConfig(config, authenticationFacade.currentOrganizationId(), - config.getEnabled(), null); + public void preProcessConfig(IntegrationConfig newConfig, @Nullable IntegrationConfig savedConfig) { + if (newConfig.getType() == IntegrationType.APPROVAL) { + configurationValidator.check(ApprovalProperties.from(newConfig)); + } else if (newConfig.getType() == IntegrationType.SQL_INTERCEPTOR) { + configurationValidator.check(SqlInterceptorProperties.from(newConfig)); + } else if (newConfig.getType() == IntegrationType.SSO) { + configurationValidator.checkAndFillConfig(newConfig, savedConfig, + authenticationFacade.currentOrganizationId(), + newConfig.getEnabled(), + Optional.ofNullable(savedConfig).map(IntegrationConfig::getId).orElse(null)); } } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/IntegrationService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/IntegrationService.java index 8181172512..1cde2f007d 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/IntegrationService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/IntegrationService.java @@ -74,6 +74,7 @@ import com.oceanbase.odc.service.integration.model.QueryIntegrationParams; import com.oceanbase.odc.service.integration.model.SSOIntegrationConfig; import com.oceanbase.odc.service.integration.model.SqlInterceptorProperties; +import com.oceanbase.odc.service.integration.saml.SamlCredentialManager; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; @@ -91,7 +92,7 @@ public class IntegrationService { private AuthenticationFacade authenticationFacade; @Autowired - private IntegrationConfigurationValidatorDelegate integrationConfigurationValidatorDelegate; + private IntegrationConfigurationProcessorDelegate integrationConfigurationProcessorDelegate; @Autowired private HorizontalDataPermissionValidator permissionValidator; @@ -117,6 +118,9 @@ public class IntegrationService { @Autowired private CacheManager defaultCacheManager; + @Autowired + private SamlCredentialManager samlCredentialManager; + @PreAuthenticate(actions = "create", resourceType = "ODC_INTEGRATION", isForAll = true) public Boolean exists(@NotBlank String name, @NotNull IntegrationType type) { Long organizationId = authenticationFacade.currentOrganizationId(); @@ -132,7 +136,7 @@ public IntegrationConfig create(@NotNull @Valid IntegrationConfig config) { .findByNameAndTypeAndOrganizationId(config.getName(), config.getType(), organizationId); PreConditions.validNoDuplicated(ResourceType.ODC_EXTERNAL_APPROVAL, "name", config.getName(), existsEntity::isPresent); - integrationConfigurationValidatorDelegate.preProcessConfig(config); + integrationConfigurationProcessorDelegate.preProcessConfig(config, null); Encryption encryption = config.getEncryption(); encryption.check(); applicationContext.publishEvent(IntegrationEvent.createPreCreate(config)); @@ -214,12 +218,13 @@ public IntegrationConfig delete(@NotNull Long id) { @PreAuthenticate(actions = "update", resourceType = "ODC_INTEGRATION", indexOfIdParam = 0) public IntegrationConfig update(@NotNull Long id, @NotNull @Valid IntegrationConfig config) { IntegrationEntity entity = nullSafeGet(id); - permissionValidator.checkCurrentOrganization(new IntegrationConfig(entity)); + IntegrationConfig saveConfig = getDecodeConfig(entity); + permissionValidator.checkCurrentOrganization(saveConfig); if (Boolean.TRUE.equals(entity.getBuiltin())) { throw new UnsupportedException(ErrorCodes.IllegalOperation, new Object[] {"builtin integration"}, "Operation on builtin integration is not allowed"); } - integrationConfigurationValidatorDelegate.preProcessConfig(config); + integrationConfigurationProcessorDelegate.preProcessConfig(config, saveConfig); Encryption encryption = config.getEncryption(); applicationContext.publishEvent( IntegrationEvent.createPreUpdate(config, new IntegrationConfig(entity), entity.getSalt())); @@ -239,6 +244,14 @@ public IntegrationConfig update(@NotNull Long id, @NotNull @Valid IntegrationCon return new IntegrationConfig(entity); } + @SkipAuthorize("odc internal usage") + public IntegrationConfig getDecodeConfig(IntegrationEntity entity) { + IntegrationConfig integrationConfig = new IntegrationConfig(entity); + String secret = decodeSecret(entity.getSecret(), entity.getSalt(), entity.getOrganizationId()); + integrationConfig.getEncryption().setSecret(secret); + return integrationConfig; + } + @Transactional(rollbackFor = Exception.class) @PreAuthenticate(actions = "update", resourceType = "ODC_INTEGRATION", indexOfIdParam = 0) public IntegrationConfig setEnabled(@NotNull Long id, @NotNull Boolean enabled) { @@ -343,7 +356,6 @@ public SSOIntegrationConfig getSSOIntegrationConfig(IntegrationEntity integratio return ssoIntegrationConfig; } - private String encodeSecret(String plainSecret, String salt, Long organizationId) { if (plainSecret == null) { return null; @@ -352,7 +364,6 @@ private String encodeSecret(String plainSecret, String salt, Long organizationId return encryptor.encrypt(plainSecret); } - @SkipAuthorize("odc internal usage") public String decodeSecret(String encryptedSecret, String salt, Long organizationId) { if (encryptedSecret == null) { @@ -369,4 +380,9 @@ private void updateCache(Long key) { } } + @SkipAuthorize("odc internal usage") + public SSOCredential generateSSOCredential() { + return new SSOCredential(samlCredentialManager.generateCertWithCachedPrivateKey()); + } + } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/K8sJobClientSelector.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/SSOCredential.java similarity index 67% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/K8sJobClientSelector.java rename to server/odc-service/src/main/java/com/oceanbase/odc/service/integration/SSOCredential.java index 08188b7b48..ebfeb40dd5 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/K8sJobClientSelector.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/SSOCredential.java @@ -13,14 +13,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.task.caller; +package com.oceanbase.odc.service.integration; -/** - * select the matched K8sJobClient by JobContext.
- * in some deployment scenario, there may exist multiple k8s cluster for job execution. - */ -public interface K8sJobClientSelector { +import lombok.AllArgsConstructor; +import lombok.Data; + +@Data +@AllArgsConstructor +public class SSOCredential { - K8sJobClient select(JobContext jobContext); + String certificate; } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/model/Encryption.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/model/Encryption.java index ab3768627d..a61c02151f 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/model/Encryption.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/model/Encryption.java @@ -71,7 +71,8 @@ public EncryptionAlgorithm getAlgorithm() { public enum EncryptionAlgorithm { /** - * No encryption + * No encryption. Especially， for sso integration, raw means that each type sso handle the + * encryption and decryption process themselves, Encryption#secret just store it. */ RAW, diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/model/Oauth2Parameter.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/model/Oauth2Parameter.java index ee50d0a2b5..ff183ce3dc 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/model/Oauth2Parameter.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/model/Oauth2Parameter.java @@ -34,6 +34,7 @@ import com.fasterxml.jackson.annotation.JsonProperty.Access; import com.oceanbase.odc.core.shared.constant.OdcConstants; import com.oceanbase.odc.service.common.util.UrlUtils; +import com.oceanbase.odc.service.integration.oauth2.SSOStateManager; import lombok.Data; import lombok.NonNull; @@ -69,8 +70,8 @@ public class Oauth2Parameter implements SSOParameter { /** * {@link Oauth2Parameter} * - * @see com.oceanbase.odc.service.integration.oauth2.Oauth2StateManager put redirect paramters into - * Oauth2StateManager's cache, default value false to adaptive history data + * @see SSOStateManager put redirect paramters into Oauth2StateManager's cache, default value false + * to adaptive history data */ private Boolean useStateParams = true; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/model/SSOIntegrationConfig.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/model/SSOIntegrationConfig.java index 628ba474e1..a2a72afd9b 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/model/SSOIntegrationConfig.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/model/SSOIntegrationConfig.java @@ -33,6 +33,8 @@ import com.oceanbase.odc.common.json.JsonUtils; import com.oceanbase.odc.core.shared.Verify; import com.oceanbase.odc.service.integration.model.Encryption.EncryptionAlgorithm; +import com.oceanbase.odc.service.integration.saml.SamlParameter; +import com.oceanbase.odc.service.integration.saml.SamlParameter.SecretInfo; import lombok.AllArgsConstructor; import lombok.Data; @@ -61,19 +63,12 @@ public class SSOIntegrationConfig implements Serializable { @JsonSubTypes.Type(value = Oauth2Parameter.class, name = "OAUTH2"), @JsonSubTypes.Type(value = OidcParameter.class, names = "OIDC"), @JsonSubTypes.Type(value = LdapParameter.class, names = "LDAP"), + @JsonSubTypes.Type(value = SamlParameter.class, names = "SAML"), }) SSOParameter ssoParameter; MappingRule mappingRule; - public boolean isOauth2OrOidc() { - return ImmutableSet.of("OAUTH2", "OIDC").contains(type); - } - - public boolean isLdap() { - return Objects.equals(type, "LDAP"); - } - public static SSOIntegrationConfig of(IntegrationConfig integrationConfig, Long organizationId) { SSOIntegrationConfig ssoIntegrationConfig = JsonUtils.fromJson(integrationConfig.getConfiguration(), SSOIntegrationConfig.class); @@ -96,6 +91,10 @@ public static SSOIntegrationConfig of(IntegrationConfig integrationConfig, Long LdapParameter ldapParameter = (LdapParameter) ssoIntegrationConfig.getSsoParameter(); ldapParameter.setManagerPassword(integrationConfig.getEncryption().getSecret()); break; + case "SAML": + SamlParameter samlParameter = (SamlParameter) ssoIntegrationConfig.getSsoParameter(); + samlParameter.fillSecret(integrationConfig.getEncryption().getSecret()); + break; default: throw new UnsupportedOperationException("unknown type=" + ssoIntegrationConfig.getType()); } @@ -119,11 +118,25 @@ public static String parseRegistrationName(String registrationId) { return split[1]; } + public boolean isOauth2OrOidc() { + return ImmutableSet.of("OAUTH2", "OIDC").contains(type); + } + + public boolean isLdap() { + return Objects.equals(type, "LDAP"); + } + + public boolean isSaml() { + return Objects.equals(type, "SAML"); + } + public String resolveRegistrationId() { if (isOauth2OrOidc()) { return ((Oauth2Parameter) ssoParameter).getRegistrationId(); } else if (isLdap()) { return ((LdapParameter) ssoParameter).getRegistrationId(); + } else if (isSaml()) { + return ((SamlParameter) ssoParameter).getRegistrationId(); } else { throw new UnsupportedOperationException(); @@ -134,40 +147,15 @@ public Long resolveOrganizationId() { return parseOrganizationId(resolveRegistrationId()); } - @Getter - @Setter - @AllArgsConstructor - @NoArgsConstructor - public static class MappingRule { - public static final String USER_PROFILE_NESTED = "NESTED"; - - public static final String TO_BE_REPLACED = "TO_BE_REPLACED"; - - @NotBlank - private String userAccountNameField; - private Set userNickNameField; - private String userProfileViewType; - private String nestedAttributeField; - private List extraInfo; - } - - @Data - public static class CustomAttribute { - private String attributeName; - private String expression; - - public String toAutomationExpression() { - return "extra#" + attributeName; - } - } - public String resolveLoginRedirectUrl() { switch (type) { case "OAUTH2": case "OIDC": return ((Oauth2Parameter) ssoParameter).getLoginRedirectUrl(); + case "SAML": + return ((SamlParameter) ssoParameter).resolveLoginUrl(); default: - throw new UnsupportedOperationException("unknown type=" + type); + return null; } } @@ -188,6 +176,12 @@ public void fillDecryptSecret(String decryptSecret) { if (isLdap()) { ((LdapParameter) ssoParameter).setManagerPassword(decryptSecret); } + if (isSaml()) { + SecretInfo secretInfo = JsonUtils.fromJson(decryptSecret, SecretInfo.class); + SamlParameter samlParameter = (SamlParameter) ssoParameter; + samlParameter.getSigning().setPrivateKey(secretInfo.getSigningPrivateKey()); + samlParameter.getDecryption().setPrivateKey(secretInfo.getDecryptionPrivateKey()); + } } public ClientRegistration toClientRegistration() { @@ -216,4 +210,31 @@ public ClientRegistration toTestClientRegistration(String testType) { } } + @Getter + @Setter + @AllArgsConstructor + @NoArgsConstructor + public static class MappingRule { + public static final String USER_PROFILE_NESTED = "NESTED"; + + public static final String TO_BE_REPLACED = "TO_BE_REPLACED"; + + @NotBlank + private String userAccountNameField; + private Set userNickNameField; + private String userProfileViewType; + private String nestedAttributeField; + private List extraInfo; + } + + @Data + public static class CustomAttribute { + private String attributeName; + private String expression; + + public String toAutomationExpression() { + return "extra#" + attributeName; + } + } + } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/oauth2/SSOEventHandler.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/oauth2/SSOEventHandler.java index 693021c119..87b7ff26dd 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/oauth2/SSOEventHandler.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/oauth2/SSOEventHandler.java @@ -22,7 +22,7 @@ import com.oceanbase.odc.common.json.JsonUtils; import com.oceanbase.odc.core.shared.Verify; -import com.oceanbase.odc.service.integration.IntegrationConfigurationValidator; +import com.oceanbase.odc.service.integration.IntegrationConfigurationProcessor; import com.oceanbase.odc.service.integration.IntegrationEvent; import com.oceanbase.odc.service.integration.IntegrationEventHandler; import com.oceanbase.odc.service.integration.IntegrationService; @@ -41,7 +41,7 @@ public class SSOEventHandler implements IntegrationEventHandler { private LdapConfigRegistrationManager ldapConfigRegistrationManager; @Autowired - private IntegrationConfigurationValidator configurationValidator; + private IntegrationConfigurationProcessor integrationConfigurationProcessor; @Autowired private IntegrationService integrationService; @@ -71,7 +71,7 @@ public void preDelete(IntegrationEvent integrationEvent) { public void preUpdate(IntegrationEvent integrationEvent) { IntegrationConfig preConfig = integrationEvent.getPreConfig(); IntegrationConfig currentConfig = integrationEvent.getCurrentConfig(); - configurationValidator.checkNotEnabledInDbBeforeSave(currentConfig.getEnabled(), + integrationConfigurationProcessor.checkNotEnabledInDbBeforeSave(currentConfig.getEnabled(), currentConfig.getOrganizationId(), currentConfig.getId()); // current config will not have secret when it is updated, secret can't change, so use preConfig String decryptSecret = integrationService.decodeSecret(preConfig.getEncryption().getSecret(), @@ -108,7 +108,9 @@ private SSOIntegrationConfig getDecryptConfiguration(IntegrationConfig config, S Verify.verify(config.getType() == SSO, "wrong integration type"); SSOIntegrationConfig ssoIntegrationConfig = JsonUtils.fromJson(config.getConfiguration(), SSOIntegrationConfig.class); - ssoIntegrationConfig.fillDecryptSecret(decryptConfiguration); + if (!ssoIntegrationConfig.isSaml()) { + ssoIntegrationConfig.fillDecryptSecret(decryptConfiguration); + } return ssoIntegrationConfig; } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/oauth2/Oauth2StateManager.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/oauth2/SSOStateManager.java similarity index 95% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/integration/oauth2/Oauth2StateManager.java rename to server/odc-service/src/main/java/com/oceanbase/odc/service/integration/oauth2/SSOStateManager.java index 514375bb57..fe7f665cb5 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/oauth2/Oauth2StateManager.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/oauth2/SSOStateManager.java @@ -23,7 +23,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpMethod; -import org.springframework.security.oauth2.core.endpoint.OAuth2ParameterNames; import org.springframework.stereotype.Component; import com.fasterxml.jackson.core.type.TypeReference; @@ -41,7 +40,7 @@ import lombok.SneakyThrows; @Component -public class Oauth2StateManager { +public class SSOStateManager { private final Cache> STATE_PARAM_CACHE = CacheBuilder.newBuilder().maximumSize(1000) .expireAfterWrite(10, TimeUnit.MINUTES) @@ -86,7 +85,7 @@ public Map getStateParameters(String state) { }; @SneakyThrows - public void addStateToCurrentRequestParam() { + public void addStateToCurrentRequestParam(String stateKey) { HttpServletRequest request = WebRequestUtils.getCurrentRequest(); Verify.notNull(request, "request"); // state cached in mem, in the case of multiple nodes，need to rely on the StatefulRoute capability @@ -94,7 +93,7 @@ public void addStateToCurrentRequestParam() { SuccessResponse> stateResponse = requestDispatcher .forward(requestDispatcher.getHostUrl(stateHostGenerator.getHost(), properties.getRequestPort()), HttpMethod.GET, - "/api/v2/sso/state?state=" + request.getParameter(OAuth2ParameterNames.STATE), + "/api/v2/sso/state?state=" + request.getParameter(stateKey), requestDispatcher.getRequestHeaders(request), null) .getContentByType( new TypeReference>>() {}); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/oauth2/TestLoginManager.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/oauth2/TestLoginManager.java index c232d96fb0..0a7fb47d0d 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/oauth2/TestLoginManager.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/oauth2/TestLoginManager.java @@ -15,6 +15,7 @@ */ package com.oceanbase.odc.service.integration.oauth2; +import static com.oceanbase.odc.core.shared.constant.OdcConstants.ODC_BACK_URL_PARAM; import static com.oceanbase.odc.core.shared.constant.OdcConstants.TEST_LOGIN_ID_PARAM; import static com.oceanbase.odc.service.integration.model.SSOIntegrationConfig.parseRegistrationName; @@ -26,6 +27,8 @@ import javax.validation.constraints.NotBlank; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.security.saml2.Saml2RelyingPartyProperties.Registration.Acs; +import org.springframework.security.saml2.core.Saml2ParameterNames; import org.springframework.security.web.util.matcher.AntPathRequestMatcher; import org.springframework.stereotype.Service; @@ -42,6 +45,7 @@ import com.oceanbase.odc.service.common.util.WebRequestUtils; import com.oceanbase.odc.service.iam.auth.AuthenticationFacade; import com.oceanbase.odc.service.iam.auth.TestLoginTerminateException; +import com.oceanbase.odc.service.integration.IntegrationConfigurationProcessor; import com.oceanbase.odc.service.integration.IntegrationService; import com.oceanbase.odc.service.integration.ldap.LdapConfigRegistrationManager; import com.oceanbase.odc.service.integration.model.IntegrationConfig; @@ -49,6 +53,7 @@ import com.oceanbase.odc.service.integration.model.LdapContextHolder; import com.oceanbase.odc.service.integration.model.LdapContextHolder.LdapContext; import com.oceanbase.odc.service.integration.model.SSOIntegrationConfig; +import com.oceanbase.odc.service.integration.saml.AddableRelyingPartyRegistrationRepository; import com.oceanbase.odc.service.state.StatefulUuidStateIdGenerator; import lombok.extern.slf4j.Slf4j; @@ -59,8 +64,14 @@ public class TestLoginManager { public static final String REGISTRATION_ID_URI_VARIABLE_NAME = "registrationId"; - public static final AntPathRequestMatcher authorizationRequestMatcher = new AntPathRequestMatcher( + public static final AntPathRequestMatcher oAuth2AuthorizationRequestMatcher = new AntPathRequestMatcher( "/login/oauth2/code" + "/{" + REGISTRATION_ID_URI_VARIABLE_NAME + "}"); + /** + * @see Acs#getEntityId() + */ + public static final AntPathRequestMatcher samlAuthorizationRequestMatcher = + new AntPathRequestMatcher("/login/saml2/sso/{registrationId}"); + private static final AntPathRequestMatcher LDAP_REQUEST_MATCHER = new AntPathRequestMatcher("/api/v2/iam/ldap/login", "POST"); public final Cache testLoginInfoCache = @@ -80,25 +91,64 @@ public class TestLoginManager { @Autowired private StatefulUuidStateIdGenerator statefulUuidStateIdGenerator; + @Autowired + private SSOStateManager ssoStateManager; + + @Autowired(required = false) + private AddableRelyingPartyRegistrationRepository addableRelyingPartyRegistrationRepository; + + @Autowired + private IntegrationConfigurationProcessor integrationConfigurationProcessor; + @SkipAuthorize public static boolean isOAuthTestLoginRequest(HttpServletRequest request) { - String registrationId = resolveRegistrationId(request); + String registrationId = resolveOauthRegistrationId(request); if (registrationId == null) { return false; } return "test".equals(parseRegistrationName(registrationId)); } - private static String resolveRegistrationId(HttpServletRequest request) { - if (authorizationRequestMatcher.matches(request)) { - return authorizationRequestMatcher.matcher(request).getVariables() + private static String resolveOauthRegistrationId(HttpServletRequest request) { + if (oAuth2AuthorizationRequestMatcher.matches(request)) { + return oAuth2AuthorizationRequestMatcher.matcher(request).getVariables() .get(REGISTRATION_ID_URI_VARIABLE_NAME); } return null; } @SkipAuthorize - public void saveOauth2TestIdIfNeed(@NotBlank String loginInfo) { + public static boolean isSamlTestLoginRequest(HttpServletRequest request) { + String registrationId = resolveSamlRegistrationId(request); + if (registrationId == null) { + return false; + } + return "test".equals(parseRegistrationName(registrationId)); + } + + private static String resolveSamlRegistrationId(HttpServletRequest request) { + if (samlAuthorizationRequestMatcher.matches(request)) { + return samlAuthorizationRequestMatcher.matcher(request).getVariables() + .get(REGISTRATION_ID_URI_VARIABLE_NAME); + } + return null; + } + + @SkipAuthorize + public void saveSamlInfoIfNeed(String info) { + HttpServletRequest currentRequest = WebRequestUtils.getCurrentRequest(); + Verify.notNull(currentRequest, "currentRequest"); + if (!isSamlTestLoginRequest(currentRequest)) { + return; + } + String testId = WebRequestUtils.getStringValueFromParameterOrAttribute(currentRequest, + Saml2ParameterNames.RELAY_STATE); + Verify.notNull(testId, "testId"); + testLoginInfoCache.put(testId, info); + } + + @SkipAuthorize + public void saveOauth2InfoIfNeed(@NotBlank String loginInfo) { HttpServletRequest currentRequest = WebRequestUtils.getCurrentRequest(); Verify.notNull(currentRequest, "currentRequest"); if (!isOAuthTestLoginRequest(currentRequest)) { @@ -119,17 +169,10 @@ public void saveLdapTestIdIfNeed(@NotBlank String loginInfo) { } @PreAuthenticate(actions = "create", resourceType = "ODC_INTEGRATION", isForAll = true) - public SSOTestInfo getSSOTestInfo(IntegrationConfig config, String type) { + public SSOTestInfo getSSOTestInfo(IntegrationConfig config, String type, String odcBackUrl) { SSOIntegrationConfig ssoConfig = SSOIntegrationConfig.of(config, authenticationFacade.currentOrganizationId()); - Verify.verify(ssoConfig.isOauth2OrOidc() || ssoConfig.isLdap(), "not support sso type"); - if (config.getEncryption().getSecret() == null) { - Optional integration = integrationService.findByTypeAndOrganizationIdAndName( - IntegrationType.SSO, authenticationFacade.currentOrganizationId(), config.getName()); - Verify.verify(integration.isPresent(), "lack of secret"); - IntegrationEntity integrationEntity = integration.get(); - ssoConfig.fillDecryptSecret(integrationService.decodeSecret(integrationEntity.getSecret(), - integrationEntity.getSalt(), integrationEntity.getOrganizationId())); - } + Verify.verify(ssoConfig.isOauth2OrOidc() || ssoConfig.isLdap() || ssoConfig.isSaml(), "not support sso type"); + fillTestSecret(config, ssoConfig); String testId = statefulUuidStateIdGenerator.generateStateId("SSO_TEST_ID"); String redirectUrl = null; String testRegistrationId = null; @@ -146,10 +189,40 @@ public SSOTestInfo getSSOTestInfo(IntegrationConfig config, String type) { } ldapConfigRegistrationManager.addTestConfig(ssoConfig); testRegistrationId = ssoConfig.resolveRegistrationId(); + } else if (ssoConfig.isSaml()) { + if (addableRelyingPartyRegistrationRepository == null) { + throw new UnsupportedOperationException("add test sso is not support"); + } + addableRelyingPartyRegistrationRepository.addTestConfig(ssoConfig); + redirectUrl = UrlUtils.appendQueryParameter(ssoConfig.resolveLoginRedirectUrl(), + TEST_LOGIN_ID_PARAM, testId); + redirectUrl = UrlUtils.appendQueryParameter(redirectUrl, + Saml2ParameterNames.RELAY_STATE, testId); + ssoStateManager.setStateParameter(testId, ODC_BACK_URL_PARAM, odcBackUrl); } return new SSOTestInfo(redirectUrl, testId, testRegistrationId); } + private void fillTestSecret(IntegrationConfig config, SSOIntegrationConfig ssoIntegrationConfig) { + Optional integration = integrationService.findByTypeAndOrganizationIdAndName( + IntegrationType.SSO, authenticationFacade.currentOrganizationId(), config.getName()); + if (ssoIntegrationConfig.isSaml()) { + IntegrationConfig savedConfig = null; + if (integration.isPresent()) { + savedConfig = integrationService.getDecodeConfig(integration.get()); + } + integrationConfigurationProcessor.fillSamlSecret(config, savedConfig, + authenticationFacade.currentOrganizationId(), + ssoIntegrationConfig); + ssoIntegrationConfig.fillDecryptSecret(config.getEncryption().getSecret()); + } else if (config.getEncryption().getSecret() == null) { + Verify.verify(integration.isPresent(), "lack of secret"); + IntegrationEntity integrationEntity = integration.get(); + ssoIntegrationConfig.fillDecryptSecret(integrationService.decodeSecret(integrationEntity.getSecret(), + integrationEntity.getSalt(), integrationEntity.getOrganizationId())); + } + } + @Nullable @PreAuthenticate(actions = "create", resourceType = "ODC_INTEGRATION", isForAll = true) public String getTestUserInfo(String testId) { @@ -186,6 +259,18 @@ public void abortIfLdapTestLogin() { } } + @SkipAuthorize + public void abortIfSamlTestLogin() { + HttpServletRequest currentRequest = WebRequestUtils.getCurrentRequest(); + if (currentRequest == null) { + return; + } + if (isSamlTestLoginRequest(currentRequest)) { + ssoStateManager.addStateToCurrentRequestParam(Saml2ParameterNames.RELAY_STATE); + throw new TestLoginTerminateException(); + } + } + @SkipAuthorize public LdapContext loadLdapContext(HttpServletRequest request) { boolean isLdapLogin = LDAP_REQUEST_MATCHER.matches(request); @@ -204,5 +289,4 @@ public LdapContext loadLdapContext(HttpServletRequest request) { LdapContextHolder.setParameter(ldapContext); return ldapContext; } - } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/saml/AddableRelyingPartyRegistrationRepository.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/saml/AddableRelyingPartyRegistrationRepository.java new file mode 100644 index 0000000000..bce1923cc0 --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/saml/AddableRelyingPartyRegistrationRepository.java @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.integration.saml; + +import java.util.Objects; +import java.util.concurrent.TimeUnit; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.security.saml2.provider.service.registration.RelyingPartyRegistration; +import org.springframework.security.saml2.provider.service.registration.RelyingPartyRegistrationRepository; +import org.springframework.stereotype.Component; + +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; +import com.oceanbase.odc.core.shared.Verify; +import com.oceanbase.odc.service.integration.IntegrationService; +import com.oceanbase.odc.service.integration.model.SSOIntegrationConfig; + +@Component +public class AddableRelyingPartyRegistrationRepository implements RelyingPartyRegistrationRepository { + + public final Cache testConfigRegistrations = + Caffeine.newBuilder().maximumSize(100).expireAfterWrite(10, TimeUnit.MINUTES).build(); + + @Autowired + IntegrationService integrationService; + + @Override + public RelyingPartyRegistration findByRegistrationId(String registrationId) { + SSOIntegrationConfig configByRegistrationId = findConfigByRegistrationId(registrationId); + return configByRegistrationId == null ? null + : SamlRegistrationConfigHelper.asRegistration(configByRegistrationId); + } + + public SSOIntegrationConfig findConfigByRegistrationId(String registrationId) { + Verify.notBlank(registrationId, "registrationId cannot be empty"); + SSOIntegrationConfig sSoClientRegistration = integrationService.getSSoIntegrationConfig(); + if (sSoClientRegistration == null) { + return testConfigRegistrations.get(registrationId, key -> null); + } + Verify.notNull(sSoClientRegistration, "Saml sSoClientRegistration"); + SamlParameter parameter = (SamlParameter) sSoClientRegistration.getSsoParameter(); + if (Objects.equals(registrationId, parameter.getRegistrationId())) { + return sSoClientRegistration; + } + return testConfigRegistrations.get(registrationId, key -> null); + } + + public void addTestConfig(SSOIntegrationConfig ssoConfig) { + SamlParameter parameter = (SamlParameter) ssoConfig.getSsoParameter(); + parameter.amendTest(); + testConfigRegistrations.put(ssoConfig.resolveRegistrationId(), ssoConfig); + } +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/saml/SamlCredentialManager.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/saml/SamlCredentialManager.java new file mode 100644 index 0000000000..221019e150 --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/saml/SamlCredentialManager.java @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.integration.saml; + +import static com.oceanbase.odc.service.integration.saml.SamlRegistrationConfigHelper.removeBase64CertificatePem; + +import java.security.PrivateKey; +import java.security.cert.X509Certificate; +import java.util.concurrent.TimeUnit; + +import org.springframework.stereotype.Component; + +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; +import com.oceanbase.odc.common.lang.Pair; +import com.oceanbase.odc.service.integration.util.EncryptionUtil; + +import lombok.SneakyThrows; + +@Component +public class SamlCredentialManager { + + public final Cache certPrivateKeyCache = + Caffeine.newBuilder().maximumSize(100).expireAfterWrite(5, TimeUnit.MINUTES).build(); + + @SneakyThrows + public String generateCertWithCachedPrivateKey() { + Pair pair = EncryptionUtil.generateKeyPair(); + String privateKeyPem = EncryptionUtil.convertPrivateKeyToPem(pair.left); + String certificate = EncryptionUtil.convertCertificateToPem(pair.right); + certPrivateKeyCache.put(removeBase64CertificatePem(certificate), privateKeyPem); + return certificate; + } + + public String getPrivateKeyByCert(String certificate) { + return certPrivateKeyCache.get(removeBase64CertificatePem(certificate), (key) -> { + throw new RuntimeException("Certificate expired"); + }); + } + +} + diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/saml/SamlParameter.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/saml/SamlParameter.java new file mode 100644 index 0000000000..684cccd14a --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/saml/SamlParameter.java @@ -0,0 +1,170 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.integration.saml; + +import static com.oceanbase.odc.service.integration.model.SSOIntegrationConfig.parseOrganizationId; + +import javax.annotation.Nullable; + +import org.springframework.boot.autoconfigure.security.saml2.Saml2RelyingPartyProperties.AssertingParty; +import org.springframework.boot.autoconfigure.security.saml2.Saml2RelyingPartyProperties.Registration.Acs; +import org.springframework.security.saml2.provider.service.registration.Saml2MessageBinding; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonProperty.Access; +import com.oceanbase.odc.common.json.JsonUtils; +import com.oceanbase.odc.core.shared.Verify; +import com.oceanbase.odc.service.common.util.UrlUtils; +import com.oceanbase.odc.service.integration.model.SSOParameter; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +public class SamlParameter implements SSOParameter { + + private String registrationId; + private String name; + + /** + * @see Acs#getLocation() + */ + private String acsLocation = "{baseUrl}/login/saml2/sso/{registrationId}"; + + /** + * URI to the metadata endpoint for discovery-based configuration. If specified singlesignon + * manually, null is allowed. + * + * @see AssertingParty#getMetadataUri() + */ + @Nullable + private String metadataUri; + + /** + * Ensure request from sp to ldp is not tampered with privateKey generate by server, certificate + * provided by the user only support one Credential + * + * @see org.springframework.boot.autoconfigure.security.saml2.Saml2RelyingPartyProperties.Registration.Signing + */ + + private String acsEntityId = "{baseUrl}/saml2/service-provider-metadata/{registrationId}"; + + /** + * @see Acs#getBinding() + */ + private String acsBinding = "POST"; + + + private Signing signing = new Signing(); + + private String providerEntityId; + + /** + * Ensure request from ldp to sp is not tampered with + * + * @see org.springframework.boot.autoconfigure.security.saml2.Saml2RelyingPartyProperties.AssertingParty.Verification + */ + private Verification verification = new Verification(); + + private Singlesignon singlesignon = new Singlesignon(); + + /** + * Used for decrypting the SAML authentication request. + * + * @see org.springframework.boot.autoconfigure.security.saml2.Saml2RelyingPartyProperties.Decryption + */ + private Decryption decryption = new Decryption(); + + public void fillSecret(String decryptSecret) { + SecretInfo credential = JsonUtils.fromJson(decryptSecret, SecretInfo.class); + if (credential == null) { + return; + } + if (signing != null) { + this.signing.privateKey = credential.getSigningPrivateKey(); + } + if (decryption != null) { + this.decryption.privateKey = credential.getDecryptionPrivateKey(); + } + } + + public String resolveLoginUrl() { + String acsLocationPath = "/login/saml2/sso"; + Verify.verify(acsLocation.contains(acsLocationPath), "invalid acsLocation=" + acsLocation); + String baseUrl = this.acsLocation.split(acsLocationPath)[0]; + return baseUrl + "/saml2/authenticate/" + registrationId; + } + + public void amendTest() { + registrationId = parseOrganizationId(registrationId) + "-" + "test"; + acsLocation = UrlUtils.getUrlHost(acsLocation) + "/login/saml2/sso/" + registrationId; + acsEntityId = UrlUtils.getUrlHost(acsEntityId) + "/saml2/service-provider-metadata/" + registrationId; + } + + /** + * @see AssertingParty#getSinglesignon() + */ + @Data + public static class Singlesignon { + private String url; + + /** + * Whether to redirect or post authentication requests. + * + * @see Saml2MessageBinding + */ + private String binding; + + /** + * Whether to sign authentication requests. + */ + private Boolean signRequest; + } + + @Data + public static class Signing { + /** + * generate by server + */ + @JsonProperty(access = Access.WRITE_ONLY) + private String privateKey; + /** + * provider by user + */ + private String certificate; + } + + @Data + public static class Verification { + private String certificate; + } + + @Data + public static class Decryption { + @JsonProperty(access = Access.WRITE_ONLY) + private String privateKey; + private String certificate; + } + + @Data + @AllArgsConstructor + @NoArgsConstructor + public static class SecretInfo { + private String signingPrivateKey; + private String decryptionPrivateKey; + } +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/saml/SamlRegistrationConfigHelper.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/saml/SamlRegistrationConfigHelper.java new file mode 100644 index 0000000000..2bba0244d6 --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/saml/SamlRegistrationConfigHelper.java @@ -0,0 +1,232 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.integration.saml; + +import static com.oceanbase.odc.service.integration.util.EncryptionUtil.CERTIFICATE_KEY_PREFIX; +import static com.oceanbase.odc.service.integration.util.EncryptionUtil.CERTIFICATE_KEY_SUFFIX; +import static com.oceanbase.odc.service.integration.util.EncryptionUtil.PRIVATE_KEY_PREFIX; +import static com.oceanbase.odc.service.integration.util.EncryptionUtil.PRIVATE_KEY_SUFFIX; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.HttpURLConnection; +import java.net.URL; +import java.security.KeyFactory; +import java.security.cert.CertificateFactory; +import java.security.cert.X509Certificate; +import java.security.interfaces.RSAPrivateKey; +import java.security.spec.PKCS8EncodedKeySpec; +import java.util.Collection; +import java.util.Optional; +import java.util.function.Consumer; +import java.util.function.Supplier; + +import org.springframework.boot.context.properties.PropertyMapper; +import org.springframework.core.io.DefaultResourceLoader; +import org.springframework.core.io.Resource; +import org.springframework.core.io.ResourceLoader; +import org.springframework.core.io.UrlResource; +import org.springframework.security.saml2.Saml2Exception; +import org.springframework.security.saml2.core.Saml2X509Credential; +import org.springframework.security.saml2.core.Saml2X509Credential.Saml2X509CredentialType; +import org.springframework.security.saml2.provider.service.registration.RelyingPartyRegistration; +import org.springframework.security.saml2.provider.service.registration.RelyingPartyRegistration.AssertingPartyDetails; +import org.springframework.security.saml2.provider.service.registration.RelyingPartyRegistration.Builder; +import org.springframework.security.saml2.provider.service.registration.RelyingPartyRegistrations; +import org.springframework.security.saml2.provider.service.registration.Saml2MessageBinding; +import org.springframework.util.StringUtils; + +import com.oceanbase.odc.common.util.EncodeUtils; +import com.oceanbase.odc.core.shared.Verify; +import com.oceanbase.odc.service.integration.model.SSOIntegrationConfig; +import com.oceanbase.odc.service.integration.saml.SamlParameter.Signing; + +import lombok.Setter; + +public final class SamlRegistrationConfigHelper { + + private static final ResourceLoader resourceLoader = new DefaultResourceLoader(); + + public static RelyingPartyRegistration asRegistration(SSOIntegrationConfig ssoIntegrationConfig) { + Verify.verify("SAML".equals(ssoIntegrationConfig.getType()), "Invalid type=" + ssoIntegrationConfig.getType()); + SamlParameter parameter = (SamlParameter) ssoIntegrationConfig.getSsoParameter(); + boolean usingMetadata = StringUtils.hasText(parameter.getMetadataUri()); + Builder builder = (usingMetadata) + ? fromMetadataLocation(parameter.getMetadataUri()) + .registrationId(parameter.getRegistrationId()) + : RelyingPartyRegistration.withRegistrationId(parameter.getRegistrationId()); + builder.assertionConsumerServiceLocation(parameter.getAcsLocation()); + builder.assertionConsumerServiceBinding(Saml2MessageBinding.valueOf(parameter.getAcsBinding())); + builder.assertingPartyDetails(mapAssertingParty(parameter, usingMetadata)); + builder.signingX509Credentials( + (credentials) -> addCredentialIfNotNull(credentials, () -> asSigningCredential(parameter))); + builder.decryptionX509Credentials( + (credentials) -> addCredentialIfNotNull(credentials, () -> asDecryptionCredential(parameter))); + builder.assertingPartyDetails((details) -> details + .verificationX509Credentials((credentials) -> addCredentialIfNotNull(credentials, + () -> asVerificationCredential(parameter)))); + builder.entityId(parameter.getAcsEntityId()); + RelyingPartyRegistration registration = builder.build(); + boolean signRequest = registration.getAssertingPartyDetails().getWantAuthnRequestsSigned(); + validateSigningCredentials(parameter, signRequest); + return registration; + } + + public static RelyingPartyRegistration.Builder fromMetadataLocation(String metadataLocation) { + Resource resource = resourceLoader.getResource(metadataLocation); + if (resource instanceof UrlResource) { + UrlResource urlResource = (UrlResource) resource; + resource = new TimeoutUrlResourceAdaptor(urlResource.getURL()); + } + try (InputStream source = resource.getInputStream()) { + return RelyingPartyRegistrations.fromMetadata(source); + } catch (IOException ex) { + if (ex.getCause() instanceof Saml2Exception) { + throw (Saml2Exception) ex.getCause(); + } + throw new Saml2Exception(ex); + } + } + + private static void addCredentialIfNotNull(Collection credentials, + Supplier supplier) { + Saml2X509Credential saml2X509Credential = supplier.get(); + if (saml2X509Credential != null) { + credentials.add(saml2X509Credential); + } + } + + private static Consumer mapAssertingParty(SamlParameter parameter, + boolean usingMetadata) { + return (details) -> { + PropertyMapper map = PropertyMapper.get().alwaysApplyingWhenNonNull(); + map.from(parameter::getProviderEntityId).to(details::entityId); + map.from(() -> Optional.ofNullable(parameter.getSinglesignon()) + .map(SamlParameter.Singlesignon::getBinding) + .map(Saml2MessageBinding::valueOf) + .orElse(null)) + .to(details::singleSignOnServiceBinding); + map.from(() -> Optional.ofNullable(parameter.getSinglesignon()) + .map(SamlParameter.Singlesignon::getUrl) + .orElse(null)) + .to(details::singleSignOnServiceLocation); + map.from(() -> Optional.ofNullable(parameter.getSinglesignon()) + .map(SamlParameter.Singlesignon::getSignRequest) + .orElse(null)) + .when((ignored) -> !usingMetadata) + .to(details::wantAuthnRequestsSigned); + }; + } + + private static Saml2X509Credential asSigningCredential(SamlParameter parameter) { + Signing signing = parameter.getSigning(); + if (signing == null || signing.getCertificate() == null) { + return null; + } + return asSaml2X509Credential(signing.getPrivateKey(), signing.getCertificate()); + } + + private static Saml2X509Credential asVerificationCredential(SamlParameter parameter) { + if (parameter.getVerification() == null || parameter.getVerification().getCertificate() == null) { + return null; + } + X509Certificate x509Certificate = getCertificateFromBase64(parameter.getVerification().getCertificate()); + return new Saml2X509Credential(x509Certificate, Saml2X509Credential.Saml2X509CredentialType.ENCRYPTION, + Saml2X509Credential.Saml2X509CredentialType.VERIFICATION); + } + + private static Saml2X509Credential asDecryptionCredential(SamlParameter parameter) { + if (parameter.getDecryption() == null || parameter.getDecryption().getCertificate() == null) { + return null; + } + Verify.notNull(parameter.getDecryption().getCertificate(), "certificate"); + Verify.notNull(parameter.getDecryption().getPrivateKey(), "privateKey"); + return asSaml2X509Credential(parameter.getDecryption().getPrivateKey(), + parameter.getDecryption().getCertificate()); + } + + private static Saml2X509Credential asSaml2X509Credential(String base64PrivateKey, String base64Certificate) { + RSAPrivateKey rsaPrivateKey = base64ToRSAPrivateKey(base64PrivateKey); + X509Certificate x509Certificate = getCertificateFromBase64(base64Certificate); + return new Saml2X509Credential(rsaPrivateKey, x509Certificate, Saml2X509CredentialType.SIGNING); + } + + private static RSAPrivateKey base64ToRSAPrivateKey(String base64PrivateKey) { + try { + byte[] privateKeyBytes = EncodeUtils.base64DecodeFromString(removeBase64PrivateKeyPem(base64PrivateKey)); + Verify.notNull(privateKeyBytes, "privateKeyBytes cannot be null"); + PKCS8EncodedKeySpec pkcs8EncodedKeySpec = new PKCS8EncodedKeySpec(privateKeyBytes); + KeyFactory keyFactory = KeyFactory.getInstance("RSA"); + return (RSAPrivateKey) keyFactory.generatePrivate(pkcs8EncodedKeySpec); + } catch (Exception e) { + throw new RuntimeException("Invalid base64PrivateKey=" + base64PrivateKey, e); + } + } + + public static String removeBase64PrivateKeyPem(String base64PrivateKeyPem) { + return base64PrivateKeyPem + .replace(PRIVATE_KEY_PREFIX, "") + .replace(PRIVATE_KEY_SUFFIX, "") + .replaceAll("\\s", ""); + } + + private static X509Certificate getCertificateFromBase64(String base64Certificate) { + try { + byte[] certificateBytes = EncodeUtils.base64DecodeFromString(removeBase64CertificatePem(base64Certificate)); + CertificateFactory certificateFactory = CertificateFactory.getInstance("X.509"); + Verify.notNull(certificateFactory, "certificateFactory cannot be null"); + InputStream in = new ByteArrayInputStream(certificateBytes); + return (X509Certificate) certificateFactory.generateCertificate(in); + } catch (Exception e) { + throw new IllegalArgumentException("Invalid Base64 certificate=" + base64Certificate, e); + } + } + + public static String removeBase64CertificatePem(String base64Certificate) { + return base64Certificate + .replace(CERTIFICATE_KEY_PREFIX, "") + .replace(CERTIFICATE_KEY_SUFFIX, "") + .replace("\\n", "") + .replaceAll("\\s", ""); + } + + private static void validateSigningCredentials(SamlParameter parameter, boolean signRequest) { + if (signRequest) { + Verify.verify( + parameter.getSigning() != null && parameter.getSigning().getCertificate() != null + && parameter.getSigning().getPrivateKey() != null, + "Signing credentials must not be empty when authentication requests require signing."); + } + } + + @Setter + static class TimeoutUrlResourceAdaptor extends UrlResource { + + private int connectTimeout = 2000; + private int readTimeout = 2000; + + public TimeoutUrlResourceAdaptor(URL url) { + super(url); + } + + @Override + protected void customizeConnection(HttpURLConnection con) throws IOException { + con.setConnectTimeout(connectTimeout); + con.setReadTimeout(readTimeout); + } + } +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/util/EncryptionUtil.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/util/EncryptionUtil.java index cc98aa623c..5a9a1ceede 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/util/EncryptionUtil.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/integration/util/EncryptionUtil.java @@ -15,15 +15,38 @@ */ package com.oceanbase.odc.service.integration.util; +import java.math.BigInteger; +import java.security.KeyPair; +import java.security.KeyPairGenerator; +import java.security.PrivateKey; +import java.security.PublicKey; +import java.security.SecureRandom; +import java.security.Security; +import java.security.cert.CertificateEncodingException; +import java.security.cert.X509Certificate; +import java.util.Date; import java.util.Objects; import java.util.concurrent.TimeUnit; +import org.bouncycastle.asn1.x500.X500Name; +import org.bouncycastle.asn1.x509.Extension; +import org.bouncycastle.asn1.x509.KeyUsage; +import org.bouncycastle.cert.jcajce.JcaX509CertificateConverter; +import org.bouncycastle.cert.jcajce.JcaX509v3CertificateBuilder; +import org.bouncycastle.jce.provider.BouncyCastleProvider; +import org.bouncycastle.operator.ContentSigner; +import org.bouncycastle.operator.jcajce.JcaContentSignerBuilder; + import com.github.benmanes.caffeine.cache.Caffeine; import com.github.benmanes.caffeine.cache.LoadingCache; import com.oceanbase.odc.common.crypto.Encryptors; import com.oceanbase.odc.common.crypto.TextEncryptor; +import com.oceanbase.odc.common.lang.Pair; +import com.oceanbase.odc.common.util.EncodeUtils; import com.oceanbase.odc.service.integration.model.Encryption; +import lombok.SneakyThrows; + /** * Encryption utility for integration usage. * @@ -32,9 +55,19 @@ */ public class EncryptionUtil { + public static final String PRIVATE_KEY_PREFIX = "-----BEGIN PRIVATE KEY-----"; + public static final String PRIVATE_KEY_SUFFIX = "-----END PRIVATE KEY-----"; + public static final String CERTIFICATE_KEY_PREFIX = "-----BEGIN CERTIFICATE-----"; + public static final String CERTIFICATE_KEY_SUFFIX = "-----END CERTIFICATE-----"; private static final LoadingCache encryptorCache = Caffeine.newBuilder().maximumSize(100) .expireAfterAccess(10, TimeUnit.MINUTES).build(EncryptionUtil::getEncryptor); + static { + if (Security.getProvider(BouncyCastleProvider.PROVIDER_NAME) == null) { + Security.addProvider(new BouncyCastleProvider()); + } + } + public static String encrypt(String plainText, Encryption encryption) { return Objects.requireNonNull(encryptorCache.get(encryption)).encrypt(plainText); } @@ -57,4 +90,64 @@ private static TextEncryptor getEncryptor(Encryption encryption) { return Encryptors.empty(); } } + + @SneakyThrows + public static Pair generateKeyPair() { + KeyPairGenerator keyPairGenerator = KeyPairGenerator.getInstance("RSA"); + keyPairGenerator.initialize(2048, new SecureRandom()); + KeyPair keyPair = keyPairGenerator.generateKeyPair(); + PrivateKey privateKey = keyPair.getPrivate(); + JcaX509v3CertificateBuilder certBuilder = getJcaX509v3CertificateBuilder(keyPair); + ContentSigner contentSigner = new JcaContentSignerBuilder("SHA256WithRSA").build(privateKey); + KeyUsage keyUsage = getKeyAllUsage(); + certBuilder.addExtension(Extension.keyUsage, true, keyUsage); + + X509Certificate certificate = new JcaX509CertificateConverter() + .setProvider(BouncyCastleProvider.PROVIDER_NAME) + .getCertificate(certBuilder.build(contentSigner)); + + return new Pair<>(privateKey, certificate); + } + + private static KeyUsage getKeyAllUsage() { + int keyUsageFlags = KeyUsage.digitalSignature + | KeyUsage.nonRepudiation + | KeyUsage.keyEncipherment + | KeyUsage.dataEncipherment + | KeyUsage.keyAgreement + | KeyUsage.keyCertSign + | KeyUsage.cRLSign + | KeyUsage.encipherOnly + | KeyUsage.decipherOnly; + return new KeyUsage(keyUsageFlags); + } + + private static JcaX509v3CertificateBuilder getJcaX509v3CertificateBuilder(KeyPair keyPair) { + PublicKey publicKey = keyPair.getPublic(); + + X500Name issuerName = new X500Name("CN=Test, O=ODC, L=Hangzhou, ST=Hangzhou, C=CN"); + BigInteger serialNumber = new BigInteger(64, new SecureRandom()); + Date startDate = new Date(); + Date endDate = new Date(startDate.getTime() + (365L * 24 * 60 * 60 * 1000)); + + return new JcaX509v3CertificateBuilder( + issuerName, serialNumber, startDate, endDate, issuerName, + publicKey); + } + + public static String convertPrivateKeyToPem(PrivateKey privateKey) { + byte[] encodedPrivateKey = privateKey.getEncoded(); + String base64EncodedKey = EncodeUtils.base64EncodeToString(encodedPrivateKey); + return PRIVATE_KEY_PREFIX + "\n" + + base64EncodedKey + + "\n" + PRIVATE_KEY_SUFFIX; + } + + public static String convertCertificateToPem(X509Certificate certificate) throws CertificateEncodingException { + byte[] encodedCertificate = certificate.getEncoded(); + String base64EncodedCert = EncodeUtils.base64EncodeToString(encodedCertificate); + return CERTIFICATE_KEY_PREFIX + "\n" + + base64EncodedCert + + "\n" + CERTIFICATE_KEY_SUFFIX; + } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/notification/NotificationService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/notification/NotificationService.java index d3d630ac23..968d9c3247 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/notification/NotificationService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/notification/NotificationService.java @@ -131,7 +131,8 @@ public void init() { NotificationPolicy::getPolicyMetadataId, policy -> policy, (p1, p2) -> p1, LinkedHashMap::new)); } - @PreAuthenticate(hasAnyResourceRole = {"OWNER"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) + @PreAuthenticate(hasAnyResourceRole = {"OWNER"}, actions = {"OWNER"}, resourceType = "ODC_PROJECT", + indexOfIdParam = 0) public Page listChannels(@NotNull Long projectId, @NotNull QueryChannelParams queryParams, @NotNull Pageable pageable) { Page channels = channelRepository.find(queryParams, pageable).map(channelMapper::fromEntity); @@ -140,13 +141,15 @@ public Page listChannels(@NotNull Long projectId, @NotNull QueryChannel } @Transactional(rollbackFor = Exception.class) - @PreAuthenticate(hasAnyResourceRole = {"OWNER"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) + @PreAuthenticate(hasAnyResourceRole = {"OWNER"}, actions = {"OWNER"}, resourceType = "ODC_PROJECT", + indexOfIdParam = 0) public Channel detailChannel(@NotNull Long projectId, @NotNull Long channelId) { return channelMapper.fromEntityWithConfig(nullSafeGetChannel(channelId, projectId)); } @Transactional(rollbackFor = Exception.class) - @PreAuthenticate(hasAnyResourceRole = {"OWNER"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) + @PreAuthenticate(hasAnyResourceRole = {"OWNER"}, actions = {"OWNER"}, resourceType = "ODC_PROJECT", + indexOfIdParam = 0) public Channel createChannel(@NotNull Long projectId, @NotNull Channel channel) { PreConditions.notBlank(channel.getName(), "channel.name"); PreConditions.notNull(channel.getType(), "channel.type"); @@ -166,7 +169,8 @@ public Channel createChannel(@NotNull Long projectId, @NotNull Channel channel) } @Transactional(rollbackFor = Exception.class) - @PreAuthenticate(hasAnyResourceRole = {"OWNER"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) + @PreAuthenticate(hasAnyResourceRole = {"OWNER"}, actions = {"OWNER"}, resourceType = "ODC_PROJECT", + indexOfIdParam = 0) public Channel updateChannel(@NotNull Long projectId, @NotNull Channel channel) { PreConditions.notNull(channel.getId(), "channel.id"); validator.validate(channel.getType(), channel.getChannelConfig()); @@ -189,7 +193,8 @@ public Channel updateChannel(@NotNull Long projectId, @NotNull Channel channel) } @Transactional(rollbackFor = Exception.class) - @PreAuthenticate(hasAnyResourceRole = {"OWNER"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) + @PreAuthenticate(hasAnyResourceRole = {"OWNER"}, actions = {"OWNER"}, resourceType = "ODC_PROJECT", + indexOfIdParam = 0) public Channel deleteChannel(@NotNull Long projectId, @NotNull Long channelId) { ChannelEntity entity = nullSafeGetChannel(channelId, projectId); @@ -200,7 +205,8 @@ public Channel deleteChannel(@NotNull Long projectId, @NotNull Long channelId) { } @Transactional(rollbackFor = Exception.class) - @PreAuthenticate(hasAnyResourceRole = {"OWNER"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) + @PreAuthenticate(hasAnyResourceRole = {"OWNER"}, actions = {"OWNER"}, resourceType = "ODC_PROJECT", + indexOfIdParam = 0) public MessageSendResult testChannel(@NotNull Long projectId, @NotNull Channel channel) { PreConditions.notNull(channel.getType(), "channel.type"); PreConditions.notNull(channel.getChannelConfig(), "channel.config"); @@ -227,13 +233,15 @@ public MessageSendResult testChannel(@NotNull Long projectId, @NotNull Channel c } } - @PreAuthenticate(hasAnyResourceRole = {"OWNER"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) + @PreAuthenticate(hasAnyResourceRole = {"OWNER"}, actions = {"OWNER"}, resourceType = "ODC_PROJECT", + indexOfIdParam = 0) public Boolean existsChannel(@NotNull Long projectId, @NotBlank String channelName) { Optional optional = channelRepository.findByProjectIdAndName(projectId, channelName); return optional.isPresent(); } - @PreAuthenticate(hasAnyResourceRole = {"OWNER"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) + @PreAuthenticate(hasAnyResourceRole = {"OWNER"}, actions = {"OWNER"}, resourceType = "ODC_PROJECT", + indexOfIdParam = 0) public List listPolicies(@NotNull Long projectId) { Map policies = new LinkedHashMap<>(metaPolicies); @@ -250,7 +258,8 @@ public List listPolicies(@NotNull Long projectId) { return new ArrayList<>(policies.values()); } - @PreAuthenticate(hasAnyResourceRole = {"OWNER"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) + @PreAuthenticate(hasAnyResourceRole = {"OWNER"}, actions = {"OWNER"}, resourceType = "ODC_PROJECT", + indexOfIdParam = 0) public NotificationPolicy detailPolicy(@NotNull Long projectId, @NotNull Long policyId) { NotificationPolicyEntity entity = nullSafeGetPolicy(policyId); if (!Objects.equals(projectId, entity.getProjectId())) { @@ -263,7 +272,8 @@ public NotificationPolicy detailPolicy(@NotNull Long projectId, @NotNull Long po } @Transactional(rollbackFor = Exception.class) - @PreAuthenticate(hasAnyResourceRole = {"OWNER"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) + @PreAuthenticate(hasAnyResourceRole = {"OWNER"}, actions = {"OWNER"}, resourceType = "ODC_PROJECT", + indexOfIdParam = 0) public List batchUpdatePolicies(@NotNull Long projectId, @NotEmpty List policies) { List toBeCreated = new ArrayList<>(); @@ -282,13 +292,15 @@ public List batchUpdatePolicies(@NotNull Long projectId, return results; } - @PreAuthenticate(hasAnyResourceRole = {"OWNER"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) + @PreAuthenticate(hasAnyResourceRole = {"OWNER"}, actions = {"OWNER"}, resourceType = "ODC_PROJECT", + indexOfIdParam = 0) public Page listMessages(@NotNull Long projectId, @NotNull QueryMessageParams queryParams, @NotNull Pageable pageable) { return messageRepository.find(queryParams, pageable).map(Message::fromEntity); } - @PreAuthenticate(hasAnyResourceRole = {"OWNER"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) + @PreAuthenticate(hasAnyResourceRole = {"OWNER"}, actions = {"OWNER"}, resourceType = "ODC_PROJECT", + indexOfIdParam = 0) public Message detailMessage(@NotNull Long projectId, @NotNull Long messageId) { Message message = Message.fromEntity(nullSafeGetMessage(messageId)); if (!Objects.equals(projectId, message.getProjectId())) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/notification/constant/EventLabelKeys.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/notification/constant/EventLabelKeys.java index 2db4ac8d8c..632dbafd12 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/notification/constant/EventLabelKeys.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/notification/constant/EventLabelKeys.java @@ -50,8 +50,10 @@ public class EventLabelKeys { public static final String PROJECT_NAME = "projectName"; public static final String CREATOR_ID = "creatorId"; public static final String CREATOR_NAME = "creatorName"; + public static final String CREATOR_ACCOUNT = "creatorAccountName"; public static final String APPROVER_ID = "approverId"; public static final String APPROVER_NAME = "approverName"; + public static final String APPROVER_ACCOUNT = "approverAccountName"; /** * global info diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/notification/helper/ChannelConfigValidator.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/notification/helper/ChannelConfigValidator.java index 7d2a05b67c..6026757b6a 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/notification/helper/ChannelConfigValidator.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/notification/helper/ChannelConfigValidator.java @@ -46,10 +46,7 @@ */ @Component public class ChannelConfigValidator { - private static final Pattern ILLEGAL_CHARACTERS_PATTERN = Pattern.compile("[@#;$,\\[\\]{}\\-\\\\^\"<>]"); - private static final String DINGTALK_WEBHOOK_PREFIX = "https://oapi.dingtalk.com/robot"; - private static final String FEISHU_WEBHOOK_PREFIX = "https://open.feishu.cn/open-apis/bot"; - private static final String WECOM_WEBHOOK_PREFIX = "https://qyapi.weixin.qq.com/cgi-bin/webhook"; + private static final Pattern ILLEGAL_CHARACTERS_PATTERN = Pattern.compile("[@#;$,\\[\\]{}\\\\^\"<>]"); @Autowired private IntegrationConfigProperties integrationConfigProperties; @@ -59,13 +56,13 @@ public class ChannelConfigValidator { public void validate(@NonNull ChannelType type, BaseChannelConfig channelConfig) { switch (type) { case DingTalk: - validateDingTalkChannelConfig((DingTalkChannelConfig) channelConfig); + validateWebhook(((DingTalkChannelConfig) channelConfig).getWebhook()); return; case WeCom: - validateWeComChannelConfig((WeComChannelConfig) channelConfig); + validateWebhook(((WeComChannelConfig) channelConfig).getWebhook()); return; case Feishu: - validateFeishuChannelConfig((WebhookChannelConfig) channelConfig); + validateWebhook(((WebhookChannelConfig) channelConfig).getWebhook()); return; case Webhook: validateWebhookChannelConfig((WebhookChannelConfig) channelConfig); @@ -75,45 +72,9 @@ public void validate(@NonNull ChannelType type, BaseChannelConfig channelConfig) } } - private void validateDingTalkChannelConfig(DingTalkChannelConfig channelConfig) { - Verify.notEmpty(channelConfig.getWebhook(), "webhook"); - Verify.verify(channelConfig.getWebhook().startsWith(DINGTALK_WEBHOOK_PREFIX), - "please input an valid Dingtalk webhook"); - } - - private void validateFeishuChannelConfig(WebhookChannelConfig channelConfig) { - Verify.notEmpty(channelConfig.getWebhook(), "webhook"); - Verify.verify(channelConfig.getWebhook().startsWith(FEISHU_WEBHOOK_PREFIX), - "please input an valid Feishu webhook"); - } - - private void validateWeComChannelConfig(WeComChannelConfig channelConfig) { - Verify.notEmpty(channelConfig.getWebhook(), "webhook"); - Verify.verify(channelConfig.getWebhook().startsWith(WECOM_WEBHOOK_PREFIX), - "please input an valid WeCom webhook"); - } - private void validateWebhookChannelConfig(WebhookChannelConfig channelConfig) { - Verify.notEmpty(channelConfig.getWebhook(), "webhook"); - Verify.verify( - channelConfig.getWebhook().startsWith("http://") || channelConfig.getWebhook().startsWith("https://"), - "Webhook should start with 'http://' or 'https://'"); - if (ILLEGAL_CHARACTERS_PATTERN.matcher(channelConfig.getWebhook()).find()) { - throw new IllegalArgumentException("Webhook contains illegal characters"); - } - try { - if (CollectionUtils.isNotEmpty(integrationConfigProperties.getUrlWhiteList())) { - Verify.verify(SSRFChecker.checkUrlInWhiteList(channelConfig.getWebhook(), - integrationConfigProperties.getUrlWhiteList()), - "The webhook is forbidden due to SSRF protection"); - } else { - Verify.verify(SSRFChecker.checkHostNotInBlackList(new URL(channelConfig.getWebhook()).getHost(), - notificationProperties.getHostBlackList()), - "The webhook is forbidden due to SSRF protection"); - } - } catch (MalformedURLException e) { - throw new RuntimeException(e); - } + validateWebhook(channelConfig.getWebhook()); + String httpProxy = channelConfig.getHttpProxy(); Verify.verify(StringUtils.isEmpty(httpProxy) || httpProxy.split(":").length == 3, "Illegal http proxy, it should be like 'http(s)://host:port'"); @@ -135,4 +96,27 @@ private void validateWebhookChannelConfig(WebhookChannelConfig channelConfig) { } } + + private void validateWebhook(String webhook) { + Verify.notEmpty(webhook, "webhook"); + Verify.verify( + webhook.startsWith("http://") || webhook.startsWith("https://"), + "Webhook should start with 'http://' or 'https://'"); + if (ILLEGAL_CHARACTERS_PATTERN.matcher(webhook).find()) { + throw new IllegalArgumentException("Webhook contains illegal characters"); + } + try { + if (CollectionUtils.isNotEmpty(integrationConfigProperties.getUrlWhiteList())) { + Verify.verify(SSRFChecker.checkUrlInWhiteList(webhook, + integrationConfigProperties.getUrlWhiteList()), + "The webhook is forbidden due to SSRF protection"); + } else { + Verify.verify(SSRFChecker.checkHostNotInBlackList(new URL(webhook).getHost(), + notificationProperties.getHostBlackList()), + "The webhook is forbidden due to SSRF protection"); + } + } catch (MalformedURLException e) { + throw new RuntimeException(e); + } + } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/notification/helper/EventBuilder.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/notification/helper/EventBuilder.java index da26e3ae18..11bfe37caa 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/notification/helper/EventBuilder.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/notification/helper/EventBuilder.java @@ -15,10 +15,12 @@ */ package com.oceanbase.odc.service.notification.helper; +import static com.oceanbase.odc.service.notification.constant.EventLabelKeys.APPROVER_ACCOUNT; import static com.oceanbase.odc.service.notification.constant.EventLabelKeys.APPROVER_ID; import static com.oceanbase.odc.service.notification.constant.EventLabelKeys.APPROVER_NAME; import static com.oceanbase.odc.service.notification.constant.EventLabelKeys.CLUSTER_NAME; import static com.oceanbase.odc.service.notification.constant.EventLabelKeys.CONNECTION_ID; +import static com.oceanbase.odc.service.notification.constant.EventLabelKeys.CREATOR_ACCOUNT; import static com.oceanbase.odc.service.notification.constant.EventLabelKeys.CREATOR_ID; import static com.oceanbase.odc.service.notification.constant.EventLabelKeys.CREATOR_NAME; import static com.oceanbase.odc.service.notification.constant.EventLabelKeys.DATABASE_ID; @@ -50,7 +52,9 @@ import com.oceanbase.odc.common.json.JsonUtils; import com.oceanbase.odc.core.shared.Verify; +import com.oceanbase.odc.core.shared.constant.ResourceType; import com.oceanbase.odc.core.shared.constant.TaskType; +import com.oceanbase.odc.core.shared.exception.NotFoundException; import com.oceanbase.odc.core.shared.exception.UnexpectedException; import com.oceanbase.odc.metadb.flow.FlowInstanceEntity; import com.oceanbase.odc.metadb.flow.FlowInstanceRepository; @@ -84,6 +88,7 @@ import com.oceanbase.odc.service.permission.table.model.ApplyTableParameter; import com.oceanbase.odc.service.permission.table.model.ApplyTableParameter.ApplyTable; import com.oceanbase.odc.service.schedule.flowtask.AlterScheduleParameters; +import com.oceanbase.odc.service.schedule.model.ScheduleChangeParams; import com.oceanbase.odc.service.schedule.model.ScheduleTask; import lombok.extern.slf4j.Slf4j; @@ -155,6 +160,7 @@ public Event ofApprovedTask(TaskEntity task, Long approver) { Event event = ofTask(task, TaskEvent.APPROVED); if (approver == null) { event.getLabels().putIfNonNull(APPROVER_NAME, AUTO_APPROVAL_KEY); + event.getLabels().putIfNonNull(APPROVER_ACCOUNT, AUTO_APPROVAL_KEY); } else { event.getLabels().put(APPROVER_ID, approver + ""); } @@ -166,6 +172,7 @@ public Event ofRejectedTask(TaskEntity task, Long approver) { Event event = ofTask(task, TaskEvent.APPROVAL_REJECTION); if (approver == null) { event.getLabels().putIfNonNull(APPROVER_NAME, AUTO_APPROVAL_KEY); + event.getLabels().putIfNonNull(APPROVER_ACCOUNT, AUTO_APPROVAL_KEY); } else { event.getLabels().put(APPROVER_ID, approver + ""); } @@ -242,6 +249,19 @@ private Event ofTask(TaskEntity task, TaskEvent status) { : database.getEnvironment().getName(), database.getName())) .collect(Collectors.joining(","))); labels.putIfNonNull(PROJECT_ID, projectId); + } else if (task.getTaskType() == TaskType.ALTER_SCHEDULE) { + AlterScheduleParameters parameter = JsonUtils.fromJson(task.getParametersJson(), + AlterScheduleParameters.class); + ScheduleChangeParams scheduleChangeParams = parameter.getScheduleChangeParams(); + Verify.notNull(scheduleChangeParams, "scheduleChangeParams"); + ScheduleEntity schedule = scheduleRepository.findById(scheduleChangeParams.getScheduleId()) + .orElseThrow(() -> new NotFoundException(ResourceType.ODC_SCHEDULE, "id", + scheduleChangeParams.getScheduleId())); + projectId = schedule.getProjectId(); + labels.putIfNonNull(PROJECT_ID, projectId); + labels.putIfNonNull(DATABASE_ID, schedule.getDatabaseId()); + labels.putIfNonNull(DATABASE_NAME, schedule.getDatabaseName()); + labels.putIfNonNull(TASK_TYPE, schedule.getType().name()); } else { throw new UnexpectedException("task.databaseId should not be null"); } @@ -318,6 +338,7 @@ private void resolveLabels(EventLabels labels, T task) { try { UserEntity user = userService.nullSafeGet(labels.getLongFromString(CREATOR_ID)); labels.putIfNonNull(CREATOR_NAME, user.getName()); + labels.putIfNonNull(CREATOR_ACCOUNT, user.getAccountName()); } catch (Exception e) { log.warn("failed to query creator info.", e); } @@ -339,14 +360,19 @@ private void resolveLabels(EventLabels labels, T task) { try { if ("null".equals(labels.get(APPROVER_ID))) { labels.putIfNonNull(APPROVER_NAME, AUTO_APPROVAL_KEY); + labels.putIfNonNull(APPROVER_ACCOUNT, AUTO_APPROVAL_KEY); } else if (labels.get(APPROVER_ID).startsWith("[")) { List approverIds = JsonUtils.fromJsonList(labels.get(APPROVER_ID), Long.class); List approvers = userService.batchNullSafeGet(approverIds); labels.putIfNonNull(APPROVER_NAME, String.join(" | ", approvers.stream().map(User::getName).collect(Collectors.toSet()))); + labels.putIfNonNull(APPROVER_ACCOUNT, + String.join(" | ", + approvers.stream().map(User::getAccountName).collect(Collectors.toSet()))); } else { UserEntity user = userService.nullSafeGet(labels.getLongFromString(APPROVER_ID)); labels.putIfNonNull(APPROVER_NAME, user.getName()); + labels.putIfNonNull(APPROVER_ACCOUNT, user.getAccountName()); } } catch (Exception e) { log.warn("failed to query approver.", e); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/server/ObjectStorageHandler.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/ObjectStorageHandler.java similarity index 98% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/server/ObjectStorageHandler.java rename to server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/ObjectStorageHandler.java index 64ba7681a5..cb92d36e66 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/server/ObjectStorageHandler.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/ObjectStorageHandler.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.task.executor.server; +package com.oceanbase.odc.service.objectstorage; import java.io.File; import java.io.FileInputStream; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/client/CloudObjectStorageClient.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/client/CloudObjectStorageClient.java index 02898febbf..48724627a0 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/client/CloudObjectStorageClient.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/client/CloudObjectStorageClient.java @@ -302,13 +302,13 @@ public long calculatePartSize(long fileLength) { * 也就是杭州的client只允许操作杭州的bucket，不允许跨域操作 */ private void validateBucket() { - if (objectStorageConfiguration.getCloudProvider() != CloudProvider.ALIBABA_CLOUD) { - return; - } String bucketName = getBucketName(); boolean isExist = publicEndpointCloudObjectStorage.doesBucketExist(bucketName); Verify.verify(isExist, String.format("object storage bucket '%s' not exists", bucketName)); + if (objectStorageConfiguration.getCloudProvider() != CloudProvider.ALIBABA_CLOUD) { + return; + } String region = objectStorageConfiguration.getRegion(); if (StringUtils.isNotEmpty(region)) { String location = publicEndpointCloudObjectStorage.getBucketLocation(bucketName); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/cloud/CloudObjectStorage.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/cloud/CloudObjectStorage.java index 60d75792f4..617c40d956 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/cloud/CloudObjectStorage.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/cloud/CloudObjectStorage.java @@ -16,6 +16,7 @@ package com.oceanbase.odc.service.objectstorage.cloud; import java.io.File; +import java.io.InputStream; import java.net.URL; import java.util.Date; import java.util.List; @@ -61,6 +62,9 @@ public interface CloudObjectStorage { PutObjectResult putObject(String bucketName, String key, File file, ObjectMetadata metadata) throws CloudException; + PutObjectResult putObject(String bucketName, String key, InputStream in, ObjectMetadata metadata) + throws CloudException; + default PutObjectResult putObject(String bucketName, String key, File file) { return putObject(bucketName, key, file, null); } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/cloud/client/AlibabaCloudClient.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/cloud/client/AlibabaCloudClient.java index 059a7ba14e..acd9c43aea 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/cloud/client/AlibabaCloudClient.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/cloud/client/AlibabaCloudClient.java @@ -170,6 +170,23 @@ public PutObjectResult putObject(String bucketName, String key, File file, Objec return putObject; } + @Override + public PutObjectResult putObject(String bucketName, String key, InputStream in, ObjectMetadata metadata) + throws CloudException { + PutObjectResult putObject = callOssMethod("Put object", () -> { + com.aliyun.oss.model.ObjectMetadata objectMetadata = toOss(metadata); + com.aliyun.oss.model.PutObjectResult ossResult = oss.putObject(bucketName, key, in, objectMetadata); + PutObjectResult result = new PutObjectResult(); + result.setVersionId(ossResult.getVersionId()); + result.setETag(ossResult.getETag()); + result.setRequestId(ossResult.getRequestId()); + result.setClientCRC(ossResult.getClientCRC()); + result.setServerCRC(ossResult.getServerCRC()); + return result; + }); + return putObject; + } + @Override public CopyObjectResult copyObject(String bucketName, String from, String to) throws CloudException { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/cloud/client/AmazonCloudClient.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/cloud/client/AmazonCloudClient.java index e8e80d226b..d5f6527a0f 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/cloud/client/AmazonCloudClient.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/cloud/client/AmazonCloudClient.java @@ -183,6 +183,23 @@ public PutObjectResult putObject(String bucketName, String key, File file, Objec }); } + @Override + public PutObjectResult putObject(String bucketName, String key, InputStream in, ObjectMetadata metadata) + throws CloudException { + return callAmazonMethod("Put object", () -> { + com.amazonaws.services.s3.model.ObjectMetadata objectMetadata = toS3(metadata); + PutObjectRequest putRequest = new PutObjectRequest(bucketName, key, in, objectMetadata); + if (metadata.hasTag()) { + putRequest.withTagging(toS3(metadata.getTagging())); + } + com.amazonaws.services.s3.model.PutObjectResult s3Result = s3.putObject(putRequest); + PutObjectResult result = new PutObjectResult(); + result.setVersionId(s3Result.getVersionId()); + result.setETag(s3Result.getETag()); + return result; + }); + } + @Override public CopyObjectResult copyObject(String bucketName, String from, String to) throws CloudException { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/cloud/client/NullCloudClient.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/cloud/client/NullCloudClient.java index 7accc9b39b..91dec37407 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/cloud/client/NullCloudClient.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/cloud/client/NullCloudClient.java @@ -16,6 +16,7 @@ package com.oceanbase.odc.service.objectstorage.cloud.client; import java.io.File; +import java.io.InputStream; import java.net.URL; import java.util.Date; import java.util.List; @@ -77,6 +78,12 @@ public PutObjectResult putObject(String bucketName, String key, File file, Objec throw new UnsupportedException(); } + @Override + public PutObjectResult putObject(String bucketName, String key, InputStream in, ObjectMetadata metadata) + throws CloudException { + throw new UnsupportedException(); + } + @Override public CopyObjectResult copyObject(String bucketName, String from, String to) throws CloudException { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/util/ObjectStorageUtils.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/util/ObjectStorageUtils.java index 7b6a0c230a..e369e375d2 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/util/ObjectStorageUtils.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/objectstorage/util/ObjectStorageUtils.java @@ -24,10 +24,10 @@ import com.oceanbase.odc.common.util.StringUtils; import com.oceanbase.odc.core.shared.Verify; import com.oceanbase.odc.service.flow.task.model.SizeAwareInputStream; +import com.oceanbase.odc.service.objectstorage.ObjectStorageHandler; import com.oceanbase.odc.service.objectstorage.cloud.CloudObjectStorageService; import com.oceanbase.odc.service.objectstorage.model.ObjectMetadata; import com.oceanbase.odc.service.objectstorage.model.StorageObject; -import com.oceanbase.odc.service.task.executor.server.ObjectStorageHandler; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/ddl/OscOBMySqlAccessor.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/ddl/OscOBMySqlAccessor.java index 290b387abf..f2a3a4ee63 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/ddl/OscOBMySqlAccessor.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/ddl/OscOBMySqlAccessor.java @@ -42,7 +42,7 @@ public List listUsers(List usernames) { MySQLSqlBuilder sb = new MySQLSqlBuilder(); // mysql.users in oceanbase do not provide account locked status. // so we query locked info from ocenbase.__all_user - sb.append("SELECT distinct user_name, is_locked, host FROM oceanbase.__all_user"); + sb.append("SELECT user_name, is_locked, host FROM oceanbase.__all_user"); if (CollectionUtils.isNotEmpty(usernames)) { sb.append(" WHERE user_name IN ("); sb.values(usernames); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/oscfms/OscActionFsmBase.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/oscfms/OscActionFsmBase.java index 417e4c302d..6ba26d5121 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/oscfms/OscActionFsmBase.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/oscfms/OscActionFsmBase.java @@ -411,7 +411,7 @@ public ConnectionConfig connectionConfig() { public ConnectionSession createConnectionSession() { ConnectionConfig connectionConfig = connectionConfig(); ConnectionSession connectionSession = - new DefaultConnectSessionFactory(connectionConfig, null, null, false).generateSession(); + new DefaultConnectSessionFactory(connectionConfig, null, null, false, false).generateSession(); ConnectionSessionUtil.setCurrentSchema(connectionSession, dbName); return connectionSession; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/partitionplan/PartitionPlanService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/partitionplan/PartitionPlanService.java index ea6bc18c16..13952b9a5b 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/partitionplan/PartitionPlanService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/partitionplan/PartitionPlanService.java @@ -69,6 +69,7 @@ import com.oceanbase.odc.service.plugin.SchemaPluginUtil; import com.oceanbase.odc.service.plugin.TaskPluginUtil; import com.oceanbase.odc.service.session.ConnectSessionService; +import com.oceanbase.tools.dbbrowser.model.DBObjectType; import com.oceanbase.tools.dbbrowser.model.DBTable; import com.oceanbase.tools.dbbrowser.model.DBTableColumn; import com.oceanbase.tools.dbbrowser.model.DBTablePartition; @@ -118,7 +119,8 @@ public List getPartitionKeyDataTypes(@NonNull String sessionId, DialectType dialectType = connectionSession.getDialectType(); TableExtensionPoint tableExtensionPoint = SchemaPluginUtil.getTableExtension(dialectType); if (tableExtensionPoint == null) { - throw new UnsupportedOperationException("Unsupported dialect " + dialectType); + throw new UnsupportedOperationException("the dialect " + connectionSession.getConnectType().getDialectType() + + " doesn't support the database object type " + DBObjectType.TABLE); } DBTable dbTable = getJdbcOpt(connectionSession).execute((ConnectionCallback) con -> { try { @@ -274,7 +276,8 @@ public Map> generatePartitionDdl(@NonNull Co @NonNull PartitionPlanTableConfig tableConfig) throws Exception { TableExtensionPoint tableExtensionPoint = SchemaPluginUtil.getTableExtension(dialectType); if (tableExtensionPoint == null) { - throw new UnsupportedOperationException("Unsupported dialect " + dialectType); + throw new UnsupportedOperationException("the dialect " + dialectType + + " doesn't support the database object type " + DBObjectType.TABLE); } DBTable dbTable = tableExtensionPoint.getDetail(connection, schema, tableConfig.getTableName()); return generatePartitionDdl(connection, dialectType, dbTable, tableConfig); @@ -363,7 +366,8 @@ public String generatePartitionName(@NonNull Connection connection, @NonNull Dia @NonNull String schema, @NonNull PartitionPlanTableConfig tableConfig) throws Exception { TableExtensionPoint tableExtensionPoint = SchemaPluginUtil.getTableExtension(dialectType); if (tableExtensionPoint == null) { - throw new UnsupportedOperationException("Unsupported dialect " + dialectType); + throw new UnsupportedOperationException("the dialect " + dialectType + + " doesn't support the database object type " + DBObjectType.TABLE); } DBTable dbTable = tableExtensionPoint.getDetail(connection, schema, tableConfig.getTableName()); return generatePartitionName(connection, dialectType, dbTable, tableConfig); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/DBResourcePermissionHelper.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/DBResourcePermissionHelper.java index 7b0201b50a..be216bbfb2 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/DBResourcePermissionHelper.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/DBResourcePermissionHelper.java @@ -16,6 +16,7 @@ package com.oceanbase.odc.service.permission; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -277,7 +278,8 @@ public List filterUnauthorizedDBResources( } }); Set dbIds = resource2Types.keySet().stream().map(DBResource::getDatabaseId).collect(Collectors.toSet()); - List tbEntities = dbObjectRepository.findByDatabaseIdInAndType(dbIds, DBObjectType.TABLE); + List tbEntities = dbObjectRepository.findByDatabaseIdInAndTypeIn(dbIds, + Arrays.asList(DBObjectType.TABLE, DBObjectType.VIEW, DBObjectType.EXTERNAL_TABLE)); Map tbId2Entity = tbEntities.stream().collect(Collectors.toMap(DBObjectEntity::getId, e -> e)); Map> dbId2tbName2tbEntity = new HashMap<>(); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/database/ApplyDatabaseFlowableTask.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/database/ApplyDatabaseFlowableTask.java index 02b20fc9aa..1fe24d7a95 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/database/ApplyDatabaseFlowableTask.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/database/ApplyDatabaseFlowableTask.java @@ -44,11 +44,10 @@ import com.oceanbase.odc.metadb.iam.UserPermissionEntity; import com.oceanbase.odc.metadb.iam.UserPermissionRepository; import com.oceanbase.odc.metadb.iam.UserRepository; -import com.oceanbase.odc.metadb.iam.resourcerole.UserResourceRoleEntity; -import com.oceanbase.odc.metadb.iam.resourcerole.UserResourceRoleRepository; import com.oceanbase.odc.service.connection.database.model.DatabaseType; import com.oceanbase.odc.service.flow.task.BaseODCFlowTaskDelegate; import com.oceanbase.odc.service.flow.util.FlowTaskUtil; +import com.oceanbase.odc.service.iam.ResourceRoleService; import com.oceanbase.odc.service.permission.database.model.ApplyDatabaseParameter; import com.oceanbase.odc.service.permission.database.model.ApplyDatabaseParameter.ApplyDatabase; import com.oceanbase.odc.service.permission.database.model.ApplyDatabaseResult; @@ -77,7 +76,7 @@ public class ApplyDatabaseFlowableTask extends BaseODCFlowTaskDelegate projectMemberIds = userResourceRoleRepository.findByResourceId(projectId).stream() - .map(UserResourceRoleEntity::getUserId).collect(Collectors.toSet()); + Set projectMemberIds = + resourceRoleService.listUserIdsByResourceTypeAndResourceId(ResourceType.ODC_PROJECT, projectId); if (!projectMemberIds.contains(this.creatorId)) { log.warn("User not member of project, userId={}, projectId={}", this.creatorId, projectId); throw new IllegalStateException("User not member of project"); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/database/DatabasePermissionService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/database/DatabasePermissionService.java index 4bf5a44542..d2ab1a8695 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/database/DatabasePermissionService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/database/DatabasePermissionService.java @@ -153,7 +153,8 @@ public Page list(@NotNull Long projectId, @NotNull Query } @Transactional(rollbackFor = Exception.class) - @PreAuthenticate(hasAnyResourceRole = {"OWNER", "DBA"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) + @PreAuthenticate(hasAnyResourceRole = {"OWNER", "DBA"}, actions = {"OWNER", "DBA"}, resourceType = "ODC_PROJECT", + indexOfIdParam = 0) public List batchCreate(@NotNull Long projectId, @NotNull @Valid CreateDatabasePermissionReq req) { Set projectIds = projectService.getMemberProjectIds(req.getUserId()); @@ -220,7 +221,8 @@ public List batchCreate(@NotNull Long projectId, } @Transactional(rollbackFor = Exception.class) - @PreAuthenticate(hasAnyResourceRole = {"OWNER", "DBA"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) + @PreAuthenticate(hasAnyResourceRole = {"OWNER", "DBA"}, actions = {"OWNER", "DBA"}, resourceType = "ODC_PROJECT", + indexOfIdParam = 0) public List batchRevoke(@NotNull Long projectId, @NotEmpty List ids) { List entities = userDatabasePermissionRepository.findByProjectIdAndIdIn(projectId, ids); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/database/model/ApplyDatabaseParameter.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/database/model/ApplyDatabaseParameter.java index 6519a9e878..11f3bbb7bb 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/database/model/ApplyDatabaseParameter.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/database/model/ApplyDatabaseParameter.java @@ -47,6 +47,11 @@ public class ApplyDatabaseParameter implements Serializable, TaskParameters { * Permission types to be applied for, required */ private List types; + /** + * This field represents the duration of the permission. It is only used for the front end to + * re-initiate tasks, and the backend does not rely on it + */ + private String validDuration; /** * Expiration time, null means no expiration, optional */ diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/table/ApplyTableFlowableTask.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/table/ApplyTableFlowableTask.java index 114c2693c7..b520ef83c1 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/table/ApplyTableFlowableTask.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/table/ApplyTableFlowableTask.java @@ -45,10 +45,9 @@ import com.oceanbase.odc.metadb.iam.UserPermissionEntity; import com.oceanbase.odc.metadb.iam.UserPermissionRepository; import com.oceanbase.odc.metadb.iam.UserRepository; -import com.oceanbase.odc.metadb.iam.resourcerole.UserResourceRoleEntity; -import com.oceanbase.odc.metadb.iam.resourcerole.UserResourceRoleRepository; import com.oceanbase.odc.service.flow.task.BaseODCFlowTaskDelegate; import com.oceanbase.odc.service.flow.util.FlowTaskUtil; +import com.oceanbase.odc.service.iam.ResourceRoleService; import com.oceanbase.odc.service.permission.database.model.DatabasePermissionType; import com.oceanbase.odc.service.permission.table.model.ApplyTableParameter; import com.oceanbase.odc.service.permission.table.model.ApplyTableParameter.ApplyTable; @@ -87,7 +86,7 @@ public class ApplyTableFlowableTask extends BaseODCFlowTaskDelegate projectMemberIds = userResourceRoleRepository.findByResourceId(projectId).stream() - .map(UserResourceRoleEntity::getUserId).collect(Collectors.toSet()); + Set projectMemberIds = + resourceRoleService.listUserIdsByResourceTypeAndResourceId(ResourceType.ODC_PROJECT, projectId); if (!projectMemberIds.contains(this.creatorId)) { log.warn("User not member of project, userId={}, projectId={}", this.creatorId, projectId); throw new IllegalStateException("User not member of project"); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/table/ApplyTablePermissionPreprocessor.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/table/ApplyTablePermissionPreprocessor.java index a67b618fc3..fb1520c8f4 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/table/ApplyTablePermissionPreprocessor.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/table/ApplyTablePermissionPreprocessor.java @@ -116,7 +116,8 @@ public void process(CreateFlowInstanceReq req) { } table.setDatabaseId(database.getId()); table.setDatabaseName(database.getName()); - if (table.getType() == DBObjectType.TABLE) { + if (table.getType() == DBObjectType.TABLE || table.getType() == DBObjectType.VIEW + || table.getType() == DBObjectType.EXTERNAL_TABLE) { ConnectionConfig dataSource = id2DataSource.get(database.getDataSource().getId()); if (dataSource == null) { throw new NotFoundException(ResourceType.ODC_CONNECTION, "id", database.getDataSource().getId()); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/table/TablePermissionService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/table/TablePermissionService.java index aa79900cad..9b8c04a465 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/table/TablePermissionService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/permission/table/TablePermissionService.java @@ -167,7 +167,8 @@ public Page list(@NotNull Long projectId, @NotNull QueryTab } @Transactional(rollbackFor = Exception.class) - @PreAuthenticate(hasAnyResourceRole = {"OWNER", "DBA"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) + @PreAuthenticate(hasAnyResourceRole = {"OWNER", "DBA"}, actions = {"OWNER", "DBA"}, resourceType = "ODC_PROJECT", + indexOfIdParam = 0) public List batchCreate(@NotNull Long projectId, @NotNull @Valid CreateTablePermissionReq req) { Set projectIds = projectService.getMemberProjectIds(req.getUserId()); @@ -242,7 +243,8 @@ public List batchCreate(@NotNull Long projectId, } @Transactional(rollbackFor = Exception.class) - @PreAuthenticate(hasAnyResourceRole = {"OWNER", "DBA"}, resourceType = "ODC_PROJECT", indexOfIdParam = 0) + @PreAuthenticate(hasAnyResourceRole = {"OWNER", "DBA"}, actions = {"OWNER", "DBA"}, resourceType = "ODC_PROJECT", + indexOfIdParam = 0) public List batchRevoke(@NotNull Long projectId, @NotEmpty List ids) { List entities = userTablePermissionRepository.findByProjectIdAndIdIn(projectId, ids); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/model/PLDebugODPSpecifiedRoute.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/model/PLDebugODPSpecifiedRoute.java new file mode 100644 index 0000000000..57449ed384 --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/model/PLDebugODPSpecifiedRoute.java @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.pldebug.model; + +import lombok.Getter; +import lombok.NonNull; + +/** + * @description: + * @author: zijia.cj + * @date: 2024/12/2 15:20 + * @since: 4.3.3 + */ +@Getter +public class PLDebugODPSpecifiedRoute { + + private final String observerHost; + + private final Integer observerPort; + + public PLDebugODPSpecifiedRoute(@NonNull String observerHost, @NonNull Integer observerPort) { + this.observerHost = observerHost; + this.observerPort = observerPort; + } +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/operator/DBPLOperators.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/operator/DBPLOperators.java index c3a48d3d3e..522b21f135 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/operator/DBPLOperators.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/operator/DBPLOperators.java @@ -21,6 +21,9 @@ import com.oceanbase.odc.core.shared.constant.ConnectType; import com.oceanbase.odc.core.shared.exception.UnsupportedException; +import lombok.extern.slf4j.Slf4j; + +@Slf4j public class DBPLOperators { public static DBPLOperator create(ConnectionSession connectionSession) { @@ -33,8 +36,17 @@ public static DBPLOperator create(ConnectionSession connectionSession) { if (connectType == ConnectType.OB_ORACLE) { return new OraclePLOperator(connectionSession); + } else if (connectType == ConnectType.CLOUD_OB_ORACLE) { + String obProxyVersion = ConnectionSessionUtil.getObProxyVersion(connectionSession); + if (ConnectionSessionUtil.isSupportObProxyRoute(obProxyVersion)) { + return new OraclePLOperator(connectionSession); + } else { + throw new UnsupportedException(String.format( + "ODP version not supported, the version number must be greater than or equal to 3.1.11")); + } } else { throw new UnsupportedException(String.format("ConnectType '%s' not supported", connectType)); } } + } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/operator/GetPLErrorCallBack.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/operator/GetPLErrorCallBack.java index 7c8822f97d..0f8cf95aa0 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/operator/GetPLErrorCallBack.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/operator/GetPLErrorCallBack.java @@ -32,6 +32,7 @@ import com.oceanbase.odc.core.shared.exception.OBException; import com.oceanbase.odc.service.connection.model.ConnectionConfig; import com.oceanbase.odc.service.pldebug.model.DBPLError; +import com.oceanbase.odc.service.pldebug.model.PLDebugODPSpecifiedRoute; import com.oceanbase.odc.service.pldebug.util.PLUtils; /** @@ -42,10 +43,13 @@ public class GetPLErrorCallBack implements ConnectionCallback> { private final ConnectionConfig connectionConfig; private final DBPLError dbplError; + private final PLDebugODPSpecifiedRoute plDebugODPSpecifiedRoute; - public GetPLErrorCallBack(ConnectionConfig connectionConfig, DBPLError dbplError) { + public GetPLErrorCallBack(ConnectionConfig connectionConfig, DBPLError dbplError, + PLDebugODPSpecifiedRoute plDebugODPSpecifiedRoute) { this.connectionConfig = connectionConfig; this.dbplError = dbplError; + this.plDebugODPSpecifiedRoute = plDebugODPSpecifiedRoute; } @Override @@ -60,8 +64,10 @@ public List doInConnection(@Nullable Connection connection) throws Da tableName = "dba_errors"; } String schema = connectionConfig.getDefaultSchema(); + String specifiedRoute = PLUtils.getSpecifiedRoute(this.plDebugODPSpecifiedRoute); String sql = String.format( - "select * from %s WHERE type = '%s' and name = '%s' and owner = '%s' order by line asc;", + "%s select * from %s WHERE type = '%s' and name = '%s' and owner = '%s' order by line asc;", + specifiedRoute, tableName, dbplError.getType(), dbplError.getName(), schema); List dbplErrors = new ArrayList<>(); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/operator/OraclePLOperator.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/operator/OraclePLOperator.java index 236ea933d6..b4050c3bfb 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/operator/OraclePLOperator.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/operator/OraclePLOperator.java @@ -75,7 +75,7 @@ public Boolean isSupportPLDebug() { try { syncJdbcExecutor.execute("CALL DBMS_OUTPUT.NEW_LINE()"); } catch (DataAccessException dataAccessException) { - log.warn("CALL DBMS_DEBUG.PING() occur error {}", dataAccessException.getMessage()); + log.warn("CALL DBMS_OUTPUT.NEW_LINE() occur error {}", dataAccessException.getMessage()); String result = dataAccessException.getMessage(); if (result != null && result.contains("Unknown")) { return false; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/session/AbstractDebugSession.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/session/AbstractDebugSession.java index 124d08576c..4e707b24bf 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/session/AbstractDebugSession.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/session/AbstractDebugSession.java @@ -20,11 +20,11 @@ import java.sql.Statement; import java.util.Arrays; import java.util.List; +import java.util.Optional; import java.util.function.Supplier; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.exception.ExceptionUtils; -import org.springframework.jdbc.core.ConnectionCallback; import org.springframework.jdbc.core.JdbcOperations; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.datasource.SingleConnectionDataSource; @@ -34,15 +34,20 @@ import com.oceanbase.odc.core.session.ConnectionSession; import com.oceanbase.odc.core.session.ConnectionSessionConstants; import com.oceanbase.odc.core.session.ConnectionSessionUtil; +import com.oceanbase.odc.core.shared.Verify; +import com.oceanbase.odc.core.shared.constant.ConnectType; import com.oceanbase.odc.core.shared.constant.DialectType; import com.oceanbase.odc.core.shared.constant.OdcConstants; import com.oceanbase.odc.core.shared.exception.OBException; import com.oceanbase.odc.core.shared.exception.UnexpectedException; import com.oceanbase.odc.core.shared.model.OdcDBSession; import com.oceanbase.odc.core.sql.util.OdcDBSessionRowMapper; +import com.oceanbase.odc.plugin.connect.api.HostAddress; import com.oceanbase.odc.service.connection.model.ConnectionConfig; +import com.oceanbase.odc.service.pldebug.model.PLDebugODPSpecifiedRoute; import com.oceanbase.odc.service.pldebug.util.CallProcedureCallBack; import com.oceanbase.odc.service.pldebug.util.OBOracleCallFunctionCallBack; +import com.oceanbase.odc.service.pldebug.util.PLUtils; import com.oceanbase.odc.service.session.initializer.BackupInstanceInitializer; import com.oceanbase.odc.service.session.initializer.DataSourceInitScriptInitializer; import com.oceanbase.tools.dbbrowser.model.DBFunction; @@ -71,6 +76,7 @@ public abstract class AbstractDebugSession implements AutoCloseable { protected DebugDataSource newDataSource; protected JdbcOperations jdbcOperations; protected DialectType dialectType; + protected PLDebugODPSpecifiedRoute plDebugODPSpecifiedRoute; private static final String OB_JDBC_PROTOCOL = "oceanbase"; public abstract boolean detectSessionAlive(); @@ -79,8 +85,15 @@ public List executeProcedure(DBProcedure procedure) { try { // -1 means statement queryTimeout will be default 0, // By default there is no limit on the amount of time allowed for a running statement to complete - return getJdbcOperations().execute( - new CallProcedureCallBack(procedure, -1, getSqlBuilder())); + CallProcedureCallBack callProcedureCallBack; + if (this.plDebugODPSpecifiedRoute == null) { + callProcedureCallBack = + new CallProcedureCallBack(procedure, -1, getSqlBuilder()); + } else { + callProcedureCallBack = + new CallProcedureCallBack(procedure, -1, getSqlBuilder(), this.plDebugODPSpecifiedRoute); + } + return getJdbcOperations().execute(callProcedureCallBack); } catch (Exception e) { throw OBException.executePlFailed(String.format("Error occurs when calling procedure={%s}, message=%s", procedure.getProName(), e.getMessage())); @@ -90,10 +103,16 @@ public List executeProcedure(DBProcedure procedure) { public DBFunction executeFunction(DBFunction dbFunction) { // -1 means statement queryTimeout will be default 0, // By default there is no limit on the amount of time allowed for a running statement to complete - ConnectionCallback functionConnectionCallback = - new OBOracleCallFunctionCallBack(dbFunction, -1); + OBOracleCallFunctionCallBack obOracleCallFunctionCallBack; + if (this.plDebugODPSpecifiedRoute == null) { + obOracleCallFunctionCallBack = + new OBOracleCallFunctionCallBack(dbFunction, -1); + } else { + obOracleCallFunctionCallBack = + new OBOracleCallFunctionCallBack(dbFunction, -1, this.plDebugODPSpecifiedRoute); + } try { - return getJdbcOperations().execute(functionConnectionCallback); + return getJdbcOperations().execute(obOracleCallFunctionCallBack); } catch (Exception e) { throw OBException.executePlFailed(String.format("Error occurs when calling dbFunction={%s}, message=%s", dbFunction.getFunName(), e.getMessage())); @@ -111,21 +130,57 @@ protected void acquireNewConnection(ConnectionSession connectionSession, this.connection = newDataSource.getConnection(); } - protected DebugDataSource acquireDataSource(ConnectionSession connectionSession, List initSqls) { + protected DebugDataSource acquireDataSource(@NonNull ConnectionSession connectionSession, + @NonNull List initSqls) { ConnectionConfig config = (ConnectionConfig) ConnectionSessionUtil.getConnectionConfig(connectionSession); - DebugDataSource dataSource = new DebugDataSource(config, initSqls); String schema = ConnectionSessionUtil.getCurrentSchema(connectionSession); - String host; - Integer port; - if (StringUtils.isBlank(config.getClusterName())) { - host = config.getHost(); - port = config.getPort(); + Verify.equals(DialectType.OB_ORACLE, config.getDialectType(), "Only support OB_ORACLE"); + if (connectionSession.getConnectType() == ConnectType.OB_ORACLE + && StringUtils.isBlank(config.getClusterName())) { + // current connection is a direct observer + return buildDataSource(config, initSqls, null, + buildJdbcUrl(new HostAddress(config.getHost(), config.getPort()), schema)); } else { - String directServerIp = getDirectServerIp(connectionSession); - host = directServerIp.split(":")[0]; - port = Integer.parseInt(directServerIp.split(":")[1]); + Optional odpSpecifiedRouteDataSource = + tryGetODPSpecifiedRouteDataSource(connectionSession, initSqls, config, + schema); + if (odpSpecifiedRouteDataSource.isPresent()) { + return odpSpecifiedRouteDataSource.get(); + } + // cloud ob oracle not support direct connection to observer + if (connectionSession.getConnectType() == ConnectType.CLOUD_OB_ORACLE) { + throw new IllegalStateException(String.format( + "ODP specified route is not supported for cloud ob oracle connection, ODP version: %s", + ConnectionSessionUtil.getObProxyVersion(connectionSession))); + } + // use direct connection observer + return buildDataSource(config, initSqls, null, + buildJdbcUrl(getOBServerHostAddress(connectionSession), schema)); + } + } + + private Optional tryGetODPSpecifiedRouteDataSource(ConnectionSession connectionSession, + List initSqls, + ConnectionConfig config, String schema) { + String obProxyVersion = ConnectionSessionUtil.getObProxyVersion(connectionSession); + if (ConnectionSessionUtil.isSupportObProxyRoute(obProxyVersion)) { + HostAddress directServerIp = getOBServerHostAddress(connectionSession); + this.plDebugODPSpecifiedRoute = + new PLDebugODPSpecifiedRoute(directServerIp.getHost(), directServerIp.getPort()); + return Optional.of(buildDataSource(config, initSqls, this.plDebugODPSpecifiedRoute, + buildJdbcUrl(new HostAddress(config.getHost(), config.getPort()), schema))); } - String url = String.format("jdbc:%s://%s:%d/\"%s\"", OB_JDBC_PROTOCOL, host, port, schema); + return Optional.empty(); + } + + private String buildJdbcUrl(HostAddress hostAddress, String schema) { + return String.format("jdbc:%s://%s:%d/\"%s\"", OB_JDBC_PROTOCOL, hostAddress.getHost(), hostAddress.getPort(), + schema); + } + + private DebugDataSource buildDataSource(ConnectionConfig config, List initSqls, + PLDebugODPSpecifiedRoute route, String url) { + DebugDataSource dataSource = new DebugDataSource(config, initSqls, route); dataSource.setUrl(url); dataSource.setUsername(buildUserName(config)); dataSource.setPassword(config.getPassword()); @@ -133,7 +188,7 @@ protected DebugDataSource acquireDataSource(ConnectionSession connectionSession, return dataSource; } - private String getDirectServerIp(ConnectionSession connectionSession) { + private HostAddress getOBServerHostAddress(ConnectionSession connectionSession) { List sessions = connectionSession.getSyncJdbcExecutor(ConnectionSessionConstants.CONSOLE_DS_KEY) .query("show full processlist", new OdcDBSessionRowMapper()); @@ -150,7 +205,8 @@ private String getDirectServerIp(ConnectionSession connectionSession) { if (StringUtils.isEmpty(directServerIp)) { throw new UnexpectedException("Empty direct server ip and port from 'show full processlist'"); } - return directServerIp; + String[] ipParts = directServerIp.split(":"); + return new HostAddress(ipParts[0], Integer.parseInt(ipParts[1])); } private String buildUserName(ConnectionConfig connectionConfig) { @@ -182,7 +238,8 @@ public void close() { protected void enableDbmsOutput(Statement statement) { try { - statement.execute(String.format("call dbms_output.enable(%s);", PL_LOG_CACHE_SIZE)); + statement.execute(String.format("%s call dbms_output.enable(%s);", + PLUtils.getSpecifiedRoute(this.plDebugODPSpecifiedRoute), PL_LOG_CACHE_SIZE)); } catch (Exception e) { log.warn("enable dbms output failed, dbms_output may not exists, sid={}, reason={}", connectionSession.getId(), @@ -194,11 +251,14 @@ static class DebugDataSource extends SingleConnectionDataSource { private final List initSqls; private final List initializers; + private final PLDebugODPSpecifiedRoute plDebugODPSpecifiedRoute; - public DebugDataSource(@NonNull ConnectionConfig connectionConfig, List initSqls) { + public DebugDataSource(@NonNull ConnectionConfig connectionConfig, List initSqls, + PLDebugODPSpecifiedRoute plDebugODPSpecifiedRoute) { this.initSqls = initSqls; this.initializers = Arrays.asList(new BackupInstanceInitializer(connectionConfig), new DataSourceInitScriptInitializer(connectionConfig, true)); + this.plDebugODPSpecifiedRoute = plDebugODPSpecifiedRoute; } @Override @@ -207,7 +267,7 @@ protected void prepareConnection(Connection con) throws SQLException { if (CollectionUtils.isNotEmpty(this.initSqls)) { try (Statement statement = con.createStatement()) { for (String stmt : this.initSqls) { - statement.execute(stmt); + statement.execute(PLUtils.getSpecifiedRoute(plDebugODPSpecifiedRoute) + stmt); } } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/session/DebuggeeSession.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/session/DebuggeeSession.java index 7405771b94..f633892042 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/session/DebuggeeSession.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/session/DebuggeeSession.java @@ -36,6 +36,7 @@ import com.oceanbase.odc.service.pldebug.model.PLDebugResult; import com.oceanbase.odc.service.pldebug.model.StartPLDebugReq; import com.oceanbase.odc.service.pldebug.util.PLDebugTask; +import com.oceanbase.odc.service.pldebug.util.PLUtils; import com.oceanbase.tools.dbbrowser.model.DBPLParam; import com.oceanbase.tools.dbbrowser.model.DBPLParamMode; import com.oceanbase.tools.dbbrowser.model.DBProcedure; @@ -77,7 +78,8 @@ public DebuggeeSession(ConnectionSession connectionSession, ThreadPoolExecutor d DBProcedure.of("DBMS_DEBUG", "SET_TIMEOUT_BEHAVIOUR", Collections.singletonList(param)); executeProcedure(dbProcedure); // 初始化debug_id - stmt.executeQuery("select dbms_debug.initialize() from dual;"); + stmt.executeQuery(PLUtils.getSpecifiedRoute(this.plDebugODPSpecifiedRoute) + + "select dbms_debug.initialize() from dual;"); try (ResultSet resultSet = stmt.getResultSet()) { if (resultSet.next()) { debugId = resultSet.getString(1); @@ -86,7 +88,7 @@ public DebuggeeSession(ConnectionSession connectionSession, ThreadPoolExecutor d // 打开pl的日志输出 enableDbmsOutput(stmt); // 打开调试开关 - stmt.execute("call dbms_debug.debug_on();"); + stmt.execute(PLUtils.getSpecifiedRoute(this.plDebugODPSpecifiedRoute) + "call dbms_debug.debug_on();"); } catch (SQLSyntaxErrorException e) { if (Objects.equals(e.getErrorCode(), ERROR_CODE) && StringUtils.contains(e.getMessage(), PL_DEBUGGING_ERROR_CODE)) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/session/DebuggerSession.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/session/DebuggerSession.java index 2fe1c90edf..925482bc22 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/session/DebuggerSession.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/session/DebuggerSession.java @@ -56,6 +56,7 @@ import com.oceanbase.odc.service.pldebug.model.StartPLDebugReq; import com.oceanbase.odc.service.pldebug.operator.DBPLOperators; import com.oceanbase.odc.service.pldebug.operator.GetPLErrorCallBack; +import com.oceanbase.odc.service.pldebug.util.PLUtils; import com.oceanbase.tools.dbbrowser.model.DBBasicPLObject; import com.oceanbase.tools.dbbrowser.model.DBFunction; import com.oceanbase.tools.dbbrowser.model.DBObjectType; @@ -107,16 +108,19 @@ public DebuggerSession(DebuggeeSession debuggeeSession, StartPLDebugReq req, boo debugType = req.getDebugType(); ddl = req.getAnonymousBlock(); this.syncEnabled = syncEnabled; - + if (debuggeeSession.getPlDebugODPSpecifiedRoute() != null) { + this.plDebugODPSpecifiedRoute = debuggeeSession.getPlDebugODPSpecifiedRoute(); + } // Debugger must connect to database host the same as debuggee - // 设置超时时间, 单位：us + // Set the timeout period, which is measured in microseconds (µs) List initSqls = Collections.singletonList( String.format("set session ob_query_timeout = %s;", DEBUG_TIMEOUT_MS * 1000)); acquireNewConnection(debuggeeSession.getConnectionSession(), () -> cloneDataSource(debuggeeSession.getNewDataSource(), initSqls)); try (Statement stmt = connection.createStatement()) { - // 绑定调试目标id - stmt.execute(String.format("call dbms_debug.attach_session(%s);", debuggeeSession.getDebugId())); + // Mount this new session of debugger to the previously initialized debuggee + stmt.execute(String.format("%s call dbms_debug.attach_session(%s);", PLUtils.getSpecifiedRoute( + this.plDebugODPSpecifiedRoute), debuggeeSession.getDebugId())); } catch (Exception e) { log.warn("Call dbms_debug.attach_session() failed", e); throw OBException @@ -182,7 +186,7 @@ public DebuggerSession(DebuggeeSession debuggeeSession, StartPLDebugReq req, boo private DebugDataSource cloneDataSource(DebugDataSource originDataSource, List initSqls) { ConnectionConfig config = (ConnectionConfig) ConnectionSessionUtil.getConnectionConfig(connectionSession); - DebugDataSource debuggerDataSource = new DebugDataSource(config, initSqls); + DebugDataSource debuggerDataSource = new DebugDataSource(config, initSqls, this.plDebugODPSpecifiedRoute); debuggerDataSource.setUrl(originDataSource.getUrl()); debuggerDataSource.setUsername(originDataSource.getUsername()); debuggerDataSource.setPassword(originDataSource.getPassword()); @@ -247,7 +251,8 @@ public boolean detectSessionAlive() { boolean alive = false; try (Statement stmt = connection.createStatement()) { // 探测PL对象执行线程是否存活 - stmt.execute("select dbms_debug.target_program_running() from dual"); + stmt.execute(PLUtils.getSpecifiedRoute(this.plDebugODPSpecifiedRoute) + + "select dbms_debug.target_program_running() from dual"); try (ResultSet resultSet = stmt.getResultSet()) { if (resultSet.next()) { alive = resultSet.getBoolean(1); @@ -369,7 +374,8 @@ public List getErrors() { dbplError.setType(debugType.name()); ConnectionConfig connectionConfig = (ConnectionConfig) ConnectionSessionUtil.getConnectionConfig(connectionSession); - return getJdbcOperations().execute(new GetPLErrorCallBack(connectionConfig, dbplError)); + return getJdbcOperations() + .execute(new GetPLErrorCallBack(connectionConfig, dbplError, this.plDebugODPSpecifiedRoute)); } public List setBreakpoints(List breakpoints) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/session/PLDebugSession.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/session/PLDebugSession.java index 6f2eb7a732..4e15e42484 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/session/PLDebugSession.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/session/PLDebugSession.java @@ -38,6 +38,7 @@ import com.oceanbase.odc.service.pldebug.model.PLDebugResult; import com.oceanbase.odc.service.pldebug.model.PLDebugVariable; import com.oceanbase.odc.service.pldebug.model.StartPLDebugReq; +import com.oceanbase.odc.service.pldebug.util.PLUtils; import lombok.Data; import lombok.Setter; @@ -80,7 +81,8 @@ public PLDebugSession(long userId, IdGenerator idGenerator) { public void run() { if (Objects.nonNull(debuggerSession) && debuggerSession.detectSessionAlive()) { try (Statement stmt = debuggerSession.getConnection().createStatement()) { - stmt.execute("CALL DBMS_DEBUG.PING()"); + stmt.execute(PLUtils.getSpecifiedRoute(debuggerSession.getPlDebugODPSpecifiedRoute()) + + "CALL DBMS_DEBUG.PING()"); } catch (Exception e) { log.debug("Failed to call DBMS_DEBUG.PING()", e); } @@ -242,7 +244,8 @@ private void closeDebuggee() throws Exception { private void detach() { try (Statement stmt = debuggerSession.getConnection().createStatement()) { - stmt.execute("call dbms_debug.detach_session();"); + stmt.execute(PLUtils.getSpecifiedRoute(debuggerSession.getPlDebugODPSpecifiedRoute()) + + "call dbms_debug.detach_session();"); } catch (Exception e) { log.warn("fail to detach session on debugger", e); } @@ -250,7 +253,8 @@ private void detach() { private void debugOff() { try (Statement stmt = debuggeeSession.getConnection().createStatement()) { - stmt.execute("call dbms_debug.debug_off();"); + stmt.execute(PLUtils.getSpecifiedRoute(debuggeeSession.getPlDebugODPSpecifiedRoute()) + + "call dbms_debug.debug_off();"); } catch (Exception e) { log.warn("fail to debug off on debuggee", e); } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/util/CallProcedureCallBack.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/util/CallProcedureCallBack.java index 2cd5cba4fb..a0319a5ec0 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/util/CallProcedureCallBack.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/util/CallProcedureCallBack.java @@ -31,6 +31,7 @@ import com.oceanbase.odc.core.shared.exception.BadArgumentException; import com.oceanbase.odc.core.sql.util.DBPLObjectUtil; import com.oceanbase.odc.core.sql.util.JdbcDataTypeUtil; +import com.oceanbase.odc.service.pldebug.model.PLDebugODPSpecifiedRoute; import com.oceanbase.tools.dbbrowser.model.DBPLParam; import com.oceanbase.tools.dbbrowser.model.DBPLParamMode; import com.oceanbase.tools.dbbrowser.model.DBProcedure; @@ -50,6 +51,7 @@ public class CallProcedureCallBack implements ConnectionCallback private final DBProcedure procedure; private final SqlBuilder sqlBuilder; private final int timeoutSeconds; + private final PLDebugODPSpecifiedRoute plDebugODPSpecifiedRoute; public CallProcedureCallBack(@NonNull DBProcedure procedure, int timeoutSeconds, @NonNull SqlBuilder sqlBuilder) { @@ -58,10 +60,23 @@ public CallProcedureCallBack(@NonNull DBProcedure procedure, this.procedure = procedure; this.sqlBuilder = sqlBuilder; this.timeoutSeconds = timeoutSeconds; + this.plDebugODPSpecifiedRoute = null; + } + + public CallProcedureCallBack(@NonNull DBProcedure procedure, + int timeoutSeconds, @NonNull SqlBuilder sqlBuilder, + @NonNull PLDebugODPSpecifiedRoute plDebugODPSpecifiedRoute) { + Validate.notBlank(procedure.getProName(), "Procedure name can not be blank"); + DBPLObjectUtil.checkParams(procedure); + this.procedure = procedure; + this.sqlBuilder = sqlBuilder; + this.timeoutSeconds = timeoutSeconds; + this.plDebugODPSpecifiedRoute = plDebugODPSpecifiedRoute; } @Override public List doInConnection(Connection con) throws SQLException, DataAccessException { + sqlBuilder.append(PLUtils.getSpecifiedRoute(this.plDebugODPSpecifiedRoute)); sqlBuilder.append("CALL "); if (StringUtils.isNotBlank(procedure.getPackageName())) { sqlBuilder.identifier(procedure.getPackageName()).append("."); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/util/OBOracleCallFunctionCallBack.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/util/OBOracleCallFunctionCallBack.java index 76fe37b804..f055690212 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/util/OBOracleCallFunctionCallBack.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/util/OBOracleCallFunctionCallBack.java @@ -31,6 +31,7 @@ import com.oceanbase.odc.common.util.StringUtils; import com.oceanbase.odc.core.sql.util.DBPLObjectUtil; import com.oceanbase.odc.core.sql.util.JdbcDataTypeUtil; +import com.oceanbase.odc.service.pldebug.model.PLDebugODPSpecifiedRoute; import com.oceanbase.tools.dbbrowser.model.DBFunction; import com.oceanbase.tools.dbbrowser.model.DBPLParam; import com.oceanbase.tools.dbbrowser.model.DBPLParamMode; @@ -53,16 +54,29 @@ public class OBOracleCallFunctionCallBack implements ConnectionCallback params = new ArrayList<>(); if (function.getParams() != null) { @@ -101,8 +115,15 @@ public DBFunction doInConnection(Connection con) throws SQLException, DataAccess p.setDataType(function.getReturnType()); params.add(p); proc.setParams(params); - CallProcedureCallBack callBack = - new CallProcedureCallBack(proc, timeoutSeconds, new OracleSqlBuilder()); + CallProcedureCallBack callBack; + if (this.plDebugODPSpecifiedRoute == null) { + callBack = + new CallProcedureCallBack(proc, timeoutSeconds, new OracleSqlBuilder()); + } else { + callBack = + new CallProcedureCallBack(proc, timeoutSeconds, new OracleSqlBuilder(), + this.plDebugODPSpecifiedRoute); + } List callResult = callBack.doInConnection(con); if (CollectionUtils.isEmpty(callResult)) { return function; @@ -114,7 +135,7 @@ public DBFunction doInConnection(Connection con) throws SQLException, DataAccess return function; } finally { try (Statement stmt = con.createStatement()) { - stmt.execute("DROP PROCEDURE " + plName); + stmt.execute(PLUtils.getSpecifiedRoute(plDebugODPSpecifiedRoute) + "DROP PROCEDURE " + plName); } } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/util/PLDebugTask.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/util/PLDebugTask.java index 46ede5b407..9b30a15255 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/util/PLDebugTask.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/util/PLDebugTask.java @@ -62,7 +62,8 @@ public PLDebugResult call() { PLDebugResult result = null; JdbcOperations jdbcOperations = debuggeeSession.getJdbcOperations(); if (anonymousBlock != null) { - jdbcOperations.execute(anonymousBlock); + jdbcOperations + .execute(PLUtils.getSpecifiedRoute(debuggeeSession.getPlDebugODPSpecifiedRoute()) + anonymousBlock); result = new PLDebugResult(); } log.info("Ending debug running, debugId={}", debugId); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/util/PLUtils.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/util/PLUtils.java index 0c8c4d80b1..e0e32cf2a6 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/util/PLUtils.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/pldebug/util/PLUtils.java @@ -18,6 +18,9 @@ import com.oceanbase.odc.core.session.ConnectionSession; import com.oceanbase.odc.core.session.ConnectionSessionUtil; import com.oceanbase.odc.service.connection.model.ConnectionConfig; +import com.oceanbase.odc.service.pldebug.model.PLDebugODPSpecifiedRoute; +import com.oceanbase.tools.dbbrowser.util.OracleSqlBuilder; +import com.oceanbase.tools.dbbrowser.util.SqlBuilder; /** * @author yaobin @@ -35,4 +38,17 @@ public static boolean isSys(ConnectionConfig connectionConfig) { return "SYS".equalsIgnoreCase(connectionConfig.getUsername()); } + + public static String getSpecifiedRoute(PLDebugODPSpecifiedRoute pLDebugODPSpecifiedRoute) { + SqlBuilder sqlBuilder = new OracleSqlBuilder(); + if (pLDebugODPSpecifiedRoute != null && pLDebugODPSpecifiedRoute.getObserverHost() != null + && pLDebugODPSpecifiedRoute.getObserverPort() != null) { + sqlBuilder.append("/* TARGET_DB_SERVER = '"); + sqlBuilder.append(pLDebugODPSpecifiedRoute.getObserverHost()); + sqlBuilder.append(":"); + sqlBuilder.append(pLDebugODPSpecifiedRoute.getObserverPort()); + sqlBuilder.append("' */"); + } + return sqlBuilder.toString(); + } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/quartz/executor/AbstractQuartzJob.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/quartz/executor/AbstractQuartzJob.java index 661574d65c..d37fb4ecef 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/quartz/executor/AbstractQuartzJob.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/quartz/executor/AbstractQuartzJob.java @@ -27,7 +27,10 @@ import org.quartz.JobKey; import org.quartz.UnableToInterruptJobException; +import com.oceanbase.odc.core.shared.constant.TaskStatus; import com.oceanbase.odc.core.shared.exception.UnsupportedException; +import com.oceanbase.odc.metadb.schedule.ScheduleTaskEntity; +import com.oceanbase.odc.metadb.schedule.ScheduleTaskRepository; import com.oceanbase.odc.service.common.util.SpringContextUtil; import com.oceanbase.odc.service.monitor.DefaultMeterName; import com.oceanbase.odc.service.monitor.MeterKey; @@ -75,6 +78,16 @@ public void execute(JobExecutionContext context) throws JobExecutionException { sendEndMetric(); } catch (Exception e) { sendFailedMetric(); + try { + log.info("Start to update schedule task status to failed,jobKey={}", jobKey); + ScheduleTaskRepository taskRepository = SpringContextUtil.getBean(ScheduleTaskRepository.class); + ScheduleTaskEntity taskEntity = (ScheduleTaskEntity) context.getResult(); + if (taskEntity != null && taskEntity.getId() != null) { + taskRepository.updateStatusById(taskEntity.getId(), TaskStatus.FAILED); + } + } catch (Exception innerException) { + log.warn("Update schedule task status failed.", innerException); + } log.warn("Job execute failed,job key={},fire time={}.", context.getJobDetail().getKey(), context.getFireTime(), e); } finally { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/resource/ResourceEntityConverter.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/resource/ResourceEntityConverter.java index 19f9dc50fd..cf90419a6f 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/resource/ResourceEntityConverter.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/resource/ResourceEntityConverter.java @@ -26,7 +26,7 @@ public interface ResourceEntityConverter { /** * convert resource to ResourceEntity - * + * * @param resource * @return */ @@ -34,7 +34,7 @@ public interface ResourceEntityConverter { /** * convert resourceEntity to resource - * + * * @param resourceEntity * @return */ diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/resource/ResourceID.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/resource/ResourceID.java index 22f376831a..96d55b3983 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/resource/ResourceID.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/resource/ResourceID.java @@ -26,7 +26,7 @@ /** * resource id all resource entity id should contains this class resource id should be impl as * global unique for possible - * + * * @author longpeng.zlp * @date 2024/9/2 16:47 */ diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/resource/ResourceLocation.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/resource/ResourceLocation.java index ca23a5a55e..9026823be0 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/resource/ResourceLocation.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/resource/ResourceLocation.java @@ -25,7 +25,7 @@ /** * location of the resource - * + * * @author longpeng.zlp * @date 2024/9/4 10:42 */ diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/resource/ResourceManager.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/resource/ResourceManager.java index f080c5cb5f..3f5f3c59cd 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/resource/ResourceManager.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/resource/ResourceManager.java @@ -37,7 +37,6 @@ import com.oceanbase.odc.metadb.resource.ResourceEntity; import com.oceanbase.odc.metadb.resource.ResourceRepository; import com.oceanbase.odc.metadb.resource.ResourceSpecs; -import com.oceanbase.odc.service.resource.k8s.DefaultResourceOperatorBuilder; import com.oceanbase.odc.service.resource.k8s.model.QueryResourceParams; import lombok.NonNull; @@ -45,7 +44,7 @@ /** * resource manager to holds resource allocate and free - * + * * @author longpeng.zlp * @date 2024/8/26 20:17 */ @@ -143,7 +142,7 @@ public Page> list( /** * query resource state with resource id - * + * * @param resourceID * @return * @throws Exception @@ -208,24 +207,24 @@ public void release(@NonNull ResourceID resourceID) { if (!savedResource.isPresent()) { // create resource_resource with DESTROYING state ResourceEntity resourceEntity = new ResourceEntity(); - resourceEntity.setResourceType(DefaultResourceOperatorBuilder.CLOUD_K8S_POD_TYPE); + resourceEntity.setResourceType(resourceID.getType()); resourceEntity.setEndpoint("unknown"); resourceEntity.setCreateTime(new Date(System.currentTimeMillis())); resourceEntity.setRegion(resourceID.getResourceLocation().getRegion()); resourceEntity.setGroupName(resourceID.getResourceLocation().getGroup()); resourceEntity.setNamespace(resourceID.getNamespace()); resourceEntity.setResourceName(resourceID.getIdentifier()); - resourceEntity.setStatus(ResourceState.DESTROYING); + resourceEntity.setStatus(ResourceState.ABANDONED); resourceRepository.save(resourceEntity); } else { // update resource state to destroying - resourceRepository.updateResourceStatus(resourceID, ResourceState.DESTROYING); + resourceRepository.updateResourceStatus(resourceID, ResourceState.ABANDONED); } } /** * real destroy by resource id - * + * * @param resourceID * @return * @throws Exception @@ -234,8 +233,11 @@ public void release(@NonNull ResourceID resourceID) { @SkipAuthorize("odc internal usage") public String destroy(@NonNull ResourceID resourceID) throws Exception { Optional optional = this.resourceRepository.findByResourceID(resourceID); - if (!optional.isPresent()) { + if (!optional.isPresent()) { // may old version job log.warn("Resource is not found, resourceID={}", resourceID); + } else if (optional.get().getStatus() == ResourceState.DESTROYING) { + log.warn("Resource is already in destroying state, resourceID={}", resourceID); + return null; } return doDestroy(resourceID); } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/resource/ResourceOperator.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/resource/ResourceOperator.java index 634fe3f489..c6e7aae393 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/resource/ResourceOperator.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/resource/ResourceOperator.java @@ -22,7 +22,7 @@ /** * operator resource create / recycle / destroy - * + * * @param config of resource creating * @param resource describe * @author longpeng.zlp @@ -32,7 +32,7 @@ public interface ResourceOperator { /** * create a resource by resource context, create may not real create - * + * * @param resourceContext resource config * @return resource, may not available, maybe creating */ @@ -40,7 +40,7 @@ public interface ResourceOperator command; - - private Map environments = new HashMap<>(); - - private Map labels = new HashMap<>(); - - private String imagePullPolicy = JobConstants.IMAGE_PULL_POLICY_ALWAYS; - - private Long podPendingTimeoutSeconds; - - private Double requestCpu; - - private Long requestMem; - - private Double limitCpu; - - private Long limitMem; - - private Boolean enableMount; - - private String mountPath; - - private Long mountDiskSize; - - private Long maxNodeCount; - - private Double nodeCpu; - - private Long nodeMemInMB; -} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/resource/k8s/builder/BaseNativeK8sResourceOperatorBuilder.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/resource/k8s/builder/BaseNativeK8sResourceOperatorBuilder.java index ca07ef5c92..0cc9329520 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/resource/k8s/builder/BaseNativeK8sResourceOperatorBuilder.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/resource/k8s/builder/BaseNativeK8sResourceOperatorBuilder.java @@ -21,10 +21,10 @@ import com.oceanbase.odc.metadb.resource.ResourceEntity; import com.oceanbase.odc.service.resource.ResourceOperatorBuilder; -import com.oceanbase.odc.service.resource.k8s.client.NativeK8sJobClient; import com.oceanbase.odc.service.resource.k8s.model.K8sResource; import com.oceanbase.odc.service.task.config.K8sProperties; import com.oceanbase.odc.service.task.config.TaskFrameworkProperties; +import com.oceanbase.odc.service.task.resource.client.NativeK8sJobClient; import io.kubernetes.client.openapi.ApiClient; import io.kubernetes.client.openapi.Configuration; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleChangeLogService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleChangeLogService.java index 29b8e9c61a..b147751cb5 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleChangeLogService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleChangeLogService.java @@ -15,12 +15,17 @@ */ package com.oceanbase.odc.service.schedule; +import java.util.Iterator; import java.util.List; import java.util.stream.Collectors; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONObject; +import com.alibaba.fastjson.parser.Feature; +import com.oceanbase.odc.common.util.StringUtils; import com.oceanbase.odc.core.authority.util.SkipAuthorize; import com.oceanbase.odc.core.shared.constant.ResourceType; import com.oceanbase.odc.core.shared.exception.NotFoundException; @@ -50,9 +55,20 @@ public ScheduleChangeLog createChangeLog(ScheduleChangeLog changeLog) { } public ScheduleChangeLog getByIdAndScheduleId(Long id, Long scheduleId) { - return scheduleChangeLogRepository.findByIdAndScheduleId(id, scheduleId).map(mapper::entityToModel) + ScheduleChangeLog changeLog = scheduleChangeLogRepository.findByIdAndScheduleId(id, scheduleId).map( + mapper::entityToModel) .orElseThrow(() -> new NotFoundException( ResourceType.ODC_SCHEDULE_CHANGELOG, "id", id)); + if (StringUtils.isNotEmpty(changeLog.getNewParameter()) + && StringUtils.isNotEmpty(changeLog.getPreviousParameters())) { + JSONObject pre = JSONObject.parseObject(changeLog.getPreviousParameters()); + JSONObject curr = JSONObject.parseObject(changeLog.getNewParameter()); + removeCommonKeys(pre, curr); + changeLog.setPreviousParameters(pre.toJSONString()); + changeLog.setNewParameter(curr.toJSONString()); + return changeLog; + } + return changeLog; } public ScheduleChangeLog getByFlowInstanceId(Long flowInstanceId) { @@ -78,4 +94,38 @@ public boolean hasApprovingChangeLog(Long scheduleId) { return listByScheduleId(scheduleId).stream().map(ScheduleChangeLog::getStatus) .anyMatch(ScheduleChangeStatus.APPROVING::equals); } + + + public static void removeCommonKeys(JSONObject json1, JSONObject json2) { + Iterator keys = json1.keySet().iterator(); + while (keys.hasNext()) { + String key = keys.next(); + + if (json2.containsKey(key)) { + Object value1 = json1.get(key); + Object value2 = json2.get(key); + + if (isJsonString(value1.toString()) && isJsonString(value2.toString())) { + JSONObject nestedJson1 = JSON.parseObject(value1.toString()); + JSONObject nestedJson2 = JSON.parseObject(value2.toString()); + removeCommonKeys(nestedJson1, nestedJson2); + + json1.put(key, nestedJson1); + json2.put(key, nestedJson2); + } else if (value1.equals(value2)) { + keys.remove(); + json2.remove(key); + } + } + } + } + + public static boolean isJsonString(String value) { + try { + JSON.parseObject(value, Feature.IgnoreNotMatch); + return true; + } catch (Exception e) { + return false; + } + } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java index 7b45edeb7a..cab45c4c98 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleService.java @@ -42,7 +42,9 @@ import org.quartz.Trigger; import org.springframework.beans.BeanUtils; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.cglib.beans.BeanMap; +import org.springframework.context.annotation.Lazy; import org.springframework.core.io.InputStreamResource; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; @@ -51,6 +53,7 @@ import org.springframework.transaction.annotation.Transactional; import org.springframework.util.CollectionUtils; +import com.alibaba.fastjson.JSONObject; import com.oceanbase.odc.common.json.JsonUtils; import com.oceanbase.odc.common.util.StringUtils; import com.oceanbase.odc.core.alarm.AlarmUtils; @@ -149,6 +152,9 @@ @Service @SkipAuthorize public class ScheduleService { + + @Value("${odc.task.trigger.minimum-interval:600}") + private int minInterval; @Autowired private ScheduleRepository scheduleRepository; @@ -172,6 +178,7 @@ public class ScheduleService { private ScheduleResponseMapperFactory scheduleResponseMapperFactory; @Autowired + @Lazy private ProjectService projectService; @Autowired @@ -221,6 +228,7 @@ public class ScheduleService { private final ScheduleMapper scheduleMapper = ScheduleMapper.INSTANCE; + @Transactional(rollbackFor = Exception.class) public List dispatchCreateSchedule(CreateFlowInstanceReq createReq) { AlterScheduleParameters parameters = (AlterScheduleParameters) createReq.getParameters(); // adapt history parameters @@ -233,6 +241,7 @@ public List dispatchCreateSchedule(CreateFlowInstanceReq ScheduleChangeParams scheduleChangeParams; switch (parameters.getOperationType()) { case CREATE: { + validateTriggerConfig(parameters.getTriggerConfig()); CreateScheduleReq createScheduleReq = new CreateScheduleReq(); createScheduleReq.setParameters(parameters.getScheduleTaskParameters()); createScheduleReq.setTriggerConfig(parameters.getTriggerConfig()); @@ -270,7 +279,6 @@ public ChangeScheduleResp changeSchedule(ScheduleChangeParams req) { // create or load target schedule if (req.getOperationType() == OperationType.CREATE) { PreConditions.notNull(req.getCreateScheduleReq(), "req.createScheduleReq"); - validateTriggerConfig(req.getCreateScheduleReq().getTriggerConfig()); ScheduleEntity entity = new ScheduleEntity(); entity.setName(req.getCreateScheduleReq().getName()); @@ -314,19 +322,18 @@ public ChangeScheduleResp changeSchedule(ScheduleChangeParams req) { targetSchedule = nullSafeGetByIdWithCheckPermission(req.getScheduleId(), true); if (req.getOperationType() == OperationType.UPDATE) { validateTriggerConfig(req.getUpdateScheduleReq().getTriggerConfig()); + PreConditions.validRequestState(targetSchedule.getStatus() == ScheduleStatus.PAUSE, + ErrorCodes.UpdateNotAllowed, null, "Update schedule is not allowed."); } - if (req.getOperationType() == OperationType.UPDATE - && (targetSchedule.getStatus() != ScheduleStatus.PAUSE - || hasExecutingTask(targetSchedule.getId()))) { - log.warn("Update schedule is not allowed,status={}", targetSchedule.getStatus()); - throw new IllegalStateException("Update schedule is not allowed."); + if (req.getOperationType() == OperationType.PAUSE) { + PreConditions.validRequestState(!hasExecutingTask(targetSchedule.getId()), ErrorCodes.PauseNotAllowed, + null, "Pause schedule is not allowed."); } - if (req.getOperationType() == OperationType.DELETE - && targetSchedule.getStatus() != ScheduleStatus.TERMINATED - && targetSchedule.getStatus() != ScheduleStatus.COMPLETED) { - log.warn("Delete schedule is not allowed,status={}", targetSchedule.getStatus()); - throw new IllegalStateException( - "Delete schedule is not allowed, only can delete terminated schedule or finished schedule."); + if (req.getOperationType() == OperationType.DELETE) { + PreConditions.validRequestState((targetSchedule.getStatus() == ScheduleStatus.TERMINATED + || targetSchedule.getStatus() == ScheduleStatus.COMPLETED) + && !hasExecutingTask(targetSchedule.getId()), ErrorCodes.DeleteNotAllowed, null, + "Delete schedule is not allowed."); } } @@ -363,16 +370,36 @@ private ScheduleChangeLog createScheduleChangelog(ScheduleChangeParams req, Sche "Concurrent change schedule request is not allowed"); } + String pre = null; + String curr = null; + if (req.getOperationType() == OperationType.UPDATE) { + JSONObject preJsonObject = new JSONObject(); + preJsonObject.put("triggerConfig", targetSchedule.getTriggerConfig()); + preJsonObject.put("parameters", targetSchedule.getParameters()); + pre = preJsonObject.toJSONString(); + JSONObject currJsonOBject = new JSONObject(); + currJsonOBject.put("triggerConfig", req.getUpdateScheduleReq().getTriggerConfig()); + currJsonOBject.put("parameters", req.getUpdateScheduleReq().getParameters()); + curr = currJsonOBject.toJSONString(); + } else if (req.getOperationType() == OperationType.CREATE) { + JSONObject currJsonOBject = new JSONObject(); + currJsonOBject.put("triggerConfig", req.getCreateScheduleReq().getTriggerConfig()); + currJsonOBject.put("parameters", req.getCreateScheduleReq().getParameters()); + curr = currJsonOBject.toJSONString(); + } + ScheduleChangeLog changeLog = scheduleChangeLogService.createChangeLog( - ScheduleChangeLog.build(targetSchedule.getId(), req.getOperationType(), - JsonUtils.toJson(targetSchedule.getParameters()), - req.getOperationType() == OperationType.UPDATE - ? JsonUtils.toJson(req.getUpdateScheduleReq().getParameters()) - : null, + ScheduleChangeLog.build(targetSchedule.getId(), req.getOperationType(), pre, curr, ScheduleChangeStatus.APPROVING)); log.info("Create change log success,changLog={}", changeLog); req.setScheduleChangeLogId(changeLog.getId()); - Long approvalFlowInstanceId = approvalFlowService.create(req); + Long approvalFlowInstanceId; + if (organizationService.get(targetSchedule.getId()).isPresent() + && organizationService.get(targetSchedule.getId()).get().getType() == OrganizationType.INDIVIDUAL) { + approvalFlowInstanceId = null; + } else { + approvalFlowInstanceId = approvalFlowService.create(req); + } if (approvalFlowInstanceId != null) { changeLog.setFlowInstanceId(approvalFlowInstanceId); scheduleChangeLogService.updateFlowInstanceIdById(changeLog.getId(), approvalFlowInstanceId); @@ -397,10 +424,8 @@ private void validateTriggerConfig(TriggerConfig triggerConfig) { throw new IllegalArgumentException("Invalid cron expression"); } long intervalMills = nextFiveFireTimes.get(1).getTime() - nextFiveFireTimes.get(0).getTime(); - if (intervalMills / 1000 < 10 * 60) { - throw new IllegalArgumentException( - "The interval between weeks is too short. The minimum interval is 10 minutes."); - } + PreConditions.validArgumentState(intervalMills / 1000 > minInterval, ErrorCodes.ScheduleIntervalTooShort, + new Object[] {minInterval}, null); } } @@ -695,9 +720,8 @@ public void refreshScheduleStatus(Long scheduleId) { if (status == ScheduleStatus.PAUSE) { return; } - int runningTask = scheduleTaskService.listTaskByJobNameAndStatus(scheduleId.toString(), - TaskStatus.getProcessingStatus()).size(); - if (runningTask > 0) { + Optional latestTask = getLatestTask(scheduleId); + if (latestTask.isPresent() && latestTask.get().getStatus().isProcessing()) { status = ScheduleStatus.ENABLED; } else { try { @@ -784,13 +808,15 @@ public Page list(@NotNull Pageable pageable, @NotNull Quer params.getCreator()).stream().map(User::getId).collect(Collectors.toSet())); } if (authenticationFacade.currentOrganization().getType() == OrganizationType.TEAM) { - Set projectIds = params.getProjectId() == null - ? projectService.getMemberProjectIds(authenticationFacade.currentUserId()) - : Collections.singleton(params.getProjectId()); - if (projectIds.isEmpty()) { + Set joinedProjectIds = projectService.getMemberProjectIds(authenticationFacade.currentUserId()); + if (CollectionUtils.isEmpty(joinedProjectIds)) { return Page.empty(); } - params.setProjectIds(projectIds); + if (CollectionUtils.isEmpty(params.getProjectIds())) { + params.setProjectIds(joinedProjectIds); + } else { + params.getProjectIds().retainAll(joinedProjectIds); + } } params.setOrganizationId(authenticationFacade.currentOrganizationId()); Page returnValue = scheduleRepository.find(pageable, params); @@ -800,6 +826,16 @@ public Page list(@NotNull Pageable pageable, @NotNull Quer : returnValue.map(o -> scheduleId2Overview.get(o.getId())); } + public Page listUnfinishedSchedulesByProjectId(@NonNull Pageable pageable, + @NonNull Long projectId) { + return list(pageable, QueryScheduleParams.builder().projectIds(Collections.singleton(projectId)) + .statuses(ScheduleStatus.listUnfinishedStatus()).build()); + } + + public int getEnabledScheduleCountByProjectId(@NonNull Long projectId) { + return scheduleRepository.getEnabledScheduleCountByProjectId(projectId); + } + public Page listScheduleOverview(@NotNull Pageable pageable, @NotNull QueryScheduleParams params) { log.info("List schedule overview req:{}", params); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleTaskService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleTaskService.java index b6273383a1..2f2f1dd7a5 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleTaskService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/ScheduleTaskService.java @@ -69,6 +69,7 @@ import com.oceanbase.odc.service.task.exception.JobException; import com.oceanbase.odc.service.task.model.ExecutorInfo; import com.oceanbase.odc.service.task.schedule.JobScheduler; +import com.oceanbase.odc.service.task.util.JobUtils; import lombok.extern.slf4j.Slf4j; @@ -139,7 +140,7 @@ public ScheduleTaskDetailResp getScheduleTaskDetailResp(Long id, Long scheduleId // sql plan task detail should display sql content res.setParameters(JsonUtils.toJson(scheduleTask.getParameters())); jobRepository.findByIdNative(scheduleTask.getJobId()) - .ifPresent(jobEntity -> res.setExecutionDetails(jobEntity.getResultJson())); + .ifPresent(jobEntity -> res.setExecutionDetails(JobUtils.retrieveJobResultStr(jobEntity))); break; case LOGICAL_DATABASE_CHANGE: res.setExecutionDetails( @@ -283,10 +284,6 @@ public List listByJobNames(Set jobNames) { .collect(Collectors.toList()); } - public List listTaskByJobNameAndStatus(String jobName, List statuses) { - return scheduleTaskRepository.findByJobNameAndStatusIn(jobName, statuses); - } - public ScheduleTask nullSafeGetByJobId(Long jobId) { return findByJobId(jobId) .orElseThrow(() -> new NotFoundException(ResourceType.ODC_SCHEDULE_TASK, "jobId", jobId)); @@ -322,24 +319,6 @@ public ScheduleTask nullSafeGetByIdAndScheduleId(Long id, Long scheduleId) { .orElseThrow(() -> new NotFoundException(ResourceType.ODC_SCHEDULE_TASK, "id", id))); } - - public void correctScheduleTaskStatus(Long scheduleId) { - List toBeCorrectedList = listTaskByJobNameAndStatus( - scheduleId.toString(), TaskStatus.getProcessingStatus()); - // For the scenario where the task framework is switched from closed to open, it is necessary to - // correct - // the status of tasks that were not completed while in the closed state. - if (taskFrameworkEnabledProperties.isEnabled()) { - toBeCorrectedList = - toBeCorrectedList.stream().filter(o -> o.getJobId() == null).collect(Collectors.toList()); - } - toBeCorrectedList.forEach(task -> { - updateStatusById(task.getId(), TaskStatus.CANCELED); - log.info("Task status correction successful,scheduleTaskId={}", task.getId()); - }); - } - - @SkipAuthorize("odc internal usage") public void triggerDataArchiveDelete(Long scheduleTaskId) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/factory/ScheduleResponseMapperFactory.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/factory/ScheduleResponseMapperFactory.java index 03d6ee398a..d61c3a42cf 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/factory/ScheduleResponseMapperFactory.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/factory/ScheduleResponseMapperFactory.java @@ -48,6 +48,8 @@ import com.oceanbase.odc.metadb.schedule.ScheduleTaskRepository; import com.oceanbase.odc.metadb.task.JobEntity; import com.oceanbase.odc.metadb.task.JobRepository; +import com.oceanbase.odc.service.collaboration.project.ProjectService; +import com.oceanbase.odc.service.collaboration.project.model.Project; import com.oceanbase.odc.service.common.model.InnerUser; import com.oceanbase.odc.service.connection.ConnectionService; import com.oceanbase.odc.service.connection.database.DatabaseService; @@ -113,6 +115,10 @@ public class ScheduleResponseMapperFactory { private ScheduleRepository scheduleRepository; @Autowired private JobRepository jobRepository; + @Autowired + private ProjectService projectService; + + public ScheduleDetailResp generateScheduleDetailResp(@NonNull Schedule schedule) { ScheduleDetailResp scheduleDetailResp = new ScheduleDetailResp(); @@ -128,6 +134,7 @@ public ScheduleDetailResp generateScheduleDetailResp(@NonNull Schedule schedule) scheduleDetailResp.setCreateTime(schedule.getCreateTime()); scheduleDetailResp.setUpdateTime(schedule.getUpdateTime()); scheduleDetailResp.setProjectId(schedule.getProjectId()); + scheduleDetailResp.setProject(projectService.detail(schedule.getProjectId())); scheduleDetailResp.setDescription(schedule.getDescription()); scheduleDetailResp.setNextFireTimes( @@ -256,6 +263,7 @@ public ScheduleDetailRespHist generateHistoryScheduleDetail(Schedule schedule) { resp.setStatus(schedule.getStatus()); resp.setProjectId(schedule.getProjectId()); + resp.setProject(projectService.detail(schedule.getProjectId())); resp.setJobParameters(schedule.getParameters()); resp.setTriggerConfig(schedule.getTriggerConfig()); resp.setNextFireTimes(QuartzCronExpressionUtils.getNextFiveFireTimes(schedule.getTriggerConfig())); @@ -332,6 +340,10 @@ public Map generateHistoryScheduleList(@NonNull Coll .filter(entry -> flowInstanceId2Candidates.get(entry.getValue()) != null).collect( Collectors.toMap(Entry::getKey, entry -> flowInstanceId2Candidates.get(entry.getValue()))); + Map id2Project = projectService + .listByIds(schedules.stream().map(ScheduleEntity::getProjectId).collect(Collectors.toSet())).stream() + .collect(Collectors.toMap(Project::getId, o -> o, (o1, o2) -> o2)); + return schedules.stream().map(schedule -> { ScheduleOverviewHist resp = new ScheduleOverviewHist(); resp.setId(schedule.getId()); @@ -352,6 +364,7 @@ public Map generateHistoryScheduleList(@NonNull Coll if (CollectionUtils.isNotEmpty(candidates)) { resp.setCandidateApprovers(candidates.stream().map(InnerUser::new).collect(Collectors.toSet())); } + resp.setProject(id2Project.get(schedule.getProjectId())); return resp; }).collect(Collectors.toMap(ScheduleOverviewHist::getId, o -> o)); } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/AbstractDlmJob.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/AbstractDlmJob.java index 92db90608c..98c202020e 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/AbstractDlmJob.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/AbstractDlmJob.java @@ -17,6 +17,7 @@ import java.util.HashMap; import java.util.Map; +import java.util.stream.Collectors; import org.quartz.JobExecutionContext; @@ -29,13 +30,15 @@ import com.oceanbase.odc.service.connection.database.DatabaseService; import com.oceanbase.odc.service.connection.database.model.Database; import com.oceanbase.odc.service.connection.model.ConnectionConfig; +import com.oceanbase.odc.service.dlm.DLMConfiguration; +import com.oceanbase.odc.service.dlm.DLMService; import com.oceanbase.odc.service.dlm.DataSourceInfoMapper; import com.oceanbase.odc.service.dlm.DlmLimiterService; +import com.oceanbase.odc.service.dlm.model.DlmTableUnit; import com.oceanbase.odc.service.quartz.util.ScheduleTaskUtils; import com.oceanbase.odc.service.schedule.ScheduleService; -import com.oceanbase.odc.service.task.config.TaskFrameworkEnabledProperties; +import com.oceanbase.odc.service.task.base.dataarchive.DataArchiveTask; import com.oceanbase.odc.service.task.constants.JobParametersKeyConstants; -import com.oceanbase.odc.service.task.executor.task.DataArchiveTask; import com.oceanbase.odc.service.task.executor.task.TaskDescription; import com.oceanbase.odc.service.task.schedule.DefaultJobDefinition; import com.oceanbase.odc.service.task.schedule.JobScheduler; @@ -58,24 +61,23 @@ public abstract class AbstractDlmJob implements OdcJob { public final DatabaseService databaseService; public final ScheduleService scheduleService; public final DlmLimiterService limiterService; + public final DLMService dlmService; public JobScheduler jobScheduler = null; - - public final TaskFrameworkEnabledProperties taskFrameworkProperties; - public final TaskFrameworkService taskFrameworkService; + public final DLMConfiguration dlmConfiguration; + public AbstractDlmJob() { scheduleTaskRepository = SpringContextUtil.getBean(ScheduleTaskRepository.class); databaseService = SpringContextUtil.getBean(DatabaseService.class); scheduleService = SpringContextUtil.getBean(ScheduleService.class); limiterService = SpringContextUtil.getBean(DlmLimiterService.class); - taskFrameworkProperties = SpringContextUtil.getBean(TaskFrameworkEnabledProperties.class); taskFrameworkService = SpringContextUtil.getBean(TaskFrameworkService.class); - if (taskFrameworkProperties.isEnabled()) { - jobScheduler = SpringContextUtil.getBean(JobScheduler.class); - } + jobScheduler = SpringContextUtil.getBean(JobScheduler.class); + dlmService = SpringContextUtil.getBean(DLMService.class); + dlmConfiguration = SpringContextUtil.getBean(DLMConfiguration.class); } public DataSourceInfo getDataSourceInfo(Long databaseId) { @@ -83,6 +85,8 @@ public DataSourceInfo getDataSourceInfo(Long databaseId) { ConnectionConfig config = databaseService.findDataSourceForTaskById(databaseId); DataSourceInfo dataSourceInfo = DataSourceInfoMapper.toDataSourceInfo(config, db.getName()); dataSourceInfo.setDatabaseName(db.getName()); + dataSourceInfo.setSessionLimitRatio(dlmConfiguration.getSessionLimitingRatio()); + dataSourceInfo.setEnabledLimit(dlmConfiguration.isSessionLimitingEnabled()); return dataSourceInfo; } @@ -129,10 +133,25 @@ public Long publishJob(DLMJobReq parameters, Long timeoutMillis, CloudProvider p } public DLMJobReq getDLMJobReq(Long jobId) { - return JsonUtils.fromJson(JsonUtils.fromJson( + DLMJobReq dlmJobReq = JsonUtils.fromJson(JsonUtils.fromJson( taskFrameworkService.find(jobId).getJobParametersJson(), new TypeReference>() {}).get(JobParametersKeyConstants.META_TASK_PARAMETER_JSON), DLMJobReq.class); + Map tableName2Unit = + dlmService.findByScheduleTaskId(dlmJobReq.getScheduleTaskId()).stream() + .collect( + Collectors.toMap(DlmTableUnit::getTableName, o -> o)); + dlmJobReq.getTables().forEach(o -> { + if (tableName2Unit.containsKey(o.getTableName()) + && tableName2Unit.get(o.getTableName()).getStatistic() != null) { + o.setPartName2MinKey(tableName2Unit.get(o.getTableName()).getStatistic().getPartName2MinKey()); + o.setPartName2MaxKey(tableName2Unit.get(o.getTableName()).getStatistic().getPartName2MaxKey()); + o.setMinKey(tableName2Unit.get(o.getTableName()).getStatistic().getGlobalMinKey()); + o.setMaxKey(tableName2Unit.get(o.getTableName()).getStatistic().getGlobalMaxKey()); + } + }); + + return dlmJobReq; } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/DataArchiveDeleteJob.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/DataArchiveDeleteJob.java index be4bd69f4f..c0be02423a 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/DataArchiveDeleteJob.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/DataArchiveDeleteJob.java @@ -65,6 +65,7 @@ public void executeJob(JobExecutionContext context) { DLMJobReq parameters = getDLMJobReq(dataArchiveTask.getJobId()); parameters.setJobType(JobType.DELETE); + parameters.setFireTime(context.getFireTime()); parameters.setScheduleTaskId(taskEntity.getId()); parameters .setRateLimit(limiterService.getByOrderIdOrElseDefaultConfig(Long.parseLong(taskEntity.getJobName()))); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/DataArchiveJob.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/DataArchiveJob.java index 029c1ddcd8..6534f436aa 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/DataArchiveJob.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/DataArchiveJob.java @@ -48,12 +48,16 @@ private void executeInTaskFramework(JobExecutionContext context) { parameters.setScheduleTaskId(taskEntity.getId()); parameters.setJobType(JobType.MIGRATE); parameters.setTables(dataArchiveParameters.getTables()); + parameters.setFireTime(context.getFireTime()); for (DataArchiveTableConfig tableConfig : parameters.getTables()) { tableConfig.setConditionExpression(StringUtils.isNotEmpty(tableConfig.getConditionExpression()) ? DataArchiveConditionUtil.parseCondition(tableConfig.getConditionExpression(), dataArchiveParameters.getVariables(), context.getFireTime()) : ""); + tableConfig.setTargetTableName(DataArchiveConditionUtil.parseCondition(tableConfig.getTargetTableName(), + dataArchiveParameters.getVariables(), + context.getFireTime())); } parameters.setDeleteAfterMigration(dataArchiveParameters.isDeleteAfterMigration()); parameters.setMigrationInsertAction(dataArchiveParameters.getMigrationInsertAction()); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/DataArchiveRollbackJob.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/DataArchiveRollbackJob.java index 67ef30c5d8..dc139884da 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/DataArchiveRollbackJob.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/DataArchiveRollbackJob.java @@ -63,6 +63,7 @@ public void executeJob(JobExecutionContext context) { DataSourceInfo tempDataSource = parameters.getSourceDs(); parameters.setSourceDs(parameters.getTargetDs()); parameters.setTargetDs(tempDataSource); + parameters.setFireTime(context.getFireTime()); parameters .setRateLimit(limiterService.getByOrderIdOrElseDefaultConfig(Long.parseLong(taskEntity.getJobName()))); parameters.getTables().forEach(o -> { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/DataDeleteJob.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/DataDeleteJob.java index 392abd3f72..00e2a60fc6 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/DataDeleteJob.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/DataDeleteJob.java @@ -72,6 +72,7 @@ private void executeInTaskFramework(JobExecutionContext context) { parameters.getSourceDs().setQueryTimeout(dataDeleteParameters.getQueryTimeout()); parameters.getTargetDs().setQueryTimeout(dataDeleteParameters.getQueryTimeout()); parameters.setShardingStrategy(dataDeleteParameters.getShardingStrategy()); + parameters.setFireTime(context.getFireTime()); Long jobId = publishJob(parameters, dataDeleteParameters.getTimeoutMillis(), dataDeleteParameters.getDatabaseId()); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/LogicalDatabaseChangeJob.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/LogicalDatabaseChangeJob.java index c5c653829d..401baf9946 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/LogicalDatabaseChangeJob.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/LogicalDatabaseChangeJob.java @@ -35,9 +35,9 @@ import com.oceanbase.odc.service.schedule.ScheduleService; import com.oceanbase.odc.service.schedule.model.LogicalDatabaseChangeParameters; import com.oceanbase.odc.service.schedule.model.PublishLogicalDatabaseChangeReq; +import com.oceanbase.odc.service.task.base.logicdatabasechange.LogicalDatabaseChangeTask; import com.oceanbase.odc.service.task.config.TaskFrameworkEnabledProperties; import com.oceanbase.odc.service.task.constants.JobParametersKeyConstants; -import com.oceanbase.odc.service.task.executor.task.LogicalDatabaseChangeTask; import com.oceanbase.odc.service.task.executor.task.TaskDescription; import com.oceanbase.odc.service.task.schedule.DefaultJobDefinition; import com.oceanbase.odc.service.task.schedule.JobScheduler; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/SqlPlanJob.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/SqlPlanJob.java index c4f7a1c13c..f061859e5c 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/SqlPlanJob.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/job/SqlPlanJob.java @@ -45,9 +45,9 @@ import com.oceanbase.odc.service.schedule.ScheduleService; import com.oceanbase.odc.service.schedule.model.ScheduleType; import com.oceanbase.odc.service.sqlplan.model.SqlPlanParameters; +import com.oceanbase.odc.service.task.base.sqlplan.SqlPlanTask; import com.oceanbase.odc.service.task.config.TaskFrameworkEnabledProperties; import com.oceanbase.odc.service.task.constants.JobParametersKeyConstants; -import com.oceanbase.odc.service.task.executor.task.SqlPlanTask; import com.oceanbase.odc.service.task.schedule.DefaultJobDefinition; import com.oceanbase.odc.service.task.schedule.JobScheduler; import com.oceanbase.odc.service.task.schedule.SingleJobProperties; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/model/DlmTableUnitStatistic.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/model/DlmTableUnitStatistic.java index 063accdf22..f6aa86192e 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/model/DlmTableUnitStatistic.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/model/DlmTableUnitStatistic.java @@ -15,6 +15,9 @@ */ package com.oceanbase.odc.service.schedule.model; +import java.util.HashMap; +import java.util.Map; + import lombok.Data; /** @@ -33,4 +36,12 @@ public class DlmTableUnitStatistic { private Long readRowsPerSecond = 0L; + private String globalMinKey; + + private String globalMaxKey; + + private Map partName2MinKey = new HashMap<>(); + + private Map partName2MaxKey = new HashMap<>(); + } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/model/ScheduleDetailResp.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/model/ScheduleDetailResp.java index 4f2fdb67ee..ee557bec21 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/model/ScheduleDetailResp.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/model/ScheduleDetailResp.java @@ -19,6 +19,7 @@ import java.util.List; import com.oceanbase.odc.common.i18n.Internationalizable; +import com.oceanbase.odc.service.collaboration.project.model.Project; import com.oceanbase.odc.service.common.model.InnerUser; import com.oceanbase.odc.service.quartz.model.MisfireStrategy; @@ -62,4 +63,6 @@ public class ScheduleDetailResp { private TriggerConfig triggerConfig; + private Project project; + } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/model/ScheduleDetailRespHist.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/model/ScheduleDetailRespHist.java index 35106ff4e4..b968ca9e63 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/model/ScheduleDetailRespHist.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/model/ScheduleDetailRespHist.java @@ -23,6 +23,7 @@ import com.oceanbase.odc.core.shared.OrganizationIsolated; import com.oceanbase.odc.core.shared.constant.ResourceType; import com.oceanbase.odc.metadb.flow.FlowInstanceEntity; +import com.oceanbase.odc.service.collaboration.project.model.Project; import com.oceanbase.odc.service.common.model.InnerUser; import com.oceanbase.odc.service.connection.database.model.Database; import com.oceanbase.odc.service.quartz.model.MisfireStrategy; @@ -73,6 +74,7 @@ public class ScheduleDetailRespHist implements OrganizationIsolated { private Set candidateApprovers; private List jobs; private ScheduleTaskParameters jobParameters; + private Project project; @Override public String resourceType() { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/model/ScheduleOverviewHist.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/model/ScheduleOverviewHist.java index 6951c4f09a..aa6d373412 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/model/ScheduleOverviewHist.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/model/ScheduleOverviewHist.java @@ -19,6 +19,7 @@ import java.util.Set; import com.oceanbase.odc.common.i18n.Internationalizable; +import com.oceanbase.odc.service.collaboration.project.model.Project; import com.oceanbase.odc.service.common.model.InnerUser; import lombok.Data; @@ -51,4 +52,6 @@ public class ScheduleOverviewHist { @Internationalizable private String description; + private Project project; + } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/model/ScheduleStatus.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/model/ScheduleStatus.java index 5f84136abf..4f27d246f8 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/model/ScheduleStatus.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/model/ScheduleStatus.java @@ -15,6 +15,10 @@ */ package com.oceanbase.odc.service.schedule.model; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + /** * @Author：tinker * @Date: 2022/11/16 15:36 @@ -36,6 +40,9 @@ public enum ScheduleStatus { COMPLETED, EXECUTION_FAILED, - DELETED + DELETED; + public static List listUnfinishedStatus() { + return Collections.unmodifiableList(Arrays.asList(CREATING, APPROVING, ENABLED)); + } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/processor/AbstractDlmPreprocessor.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/processor/AbstractDlmPreprocessor.java index 46f191e9eb..352f267210 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/processor/AbstractDlmPreprocessor.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/processor/AbstractDlmPreprocessor.java @@ -39,6 +39,7 @@ import com.oceanbase.odc.service.plugin.SchemaPluginUtil; import com.oceanbase.odc.service.schedule.model.ScheduleChangeParams; import com.oceanbase.tools.dbbrowser.model.DBObjectIdentity; +import com.oceanbase.tools.dbbrowser.model.DBObjectType; import com.oceanbase.tools.dbbrowser.util.MySQLSqlBuilder; import com.oceanbase.tools.dbbrowser.util.OracleSqlBuilder; import com.oceanbase.tools.dbbrowser.util.SqlBuilder; @@ -61,7 +62,7 @@ public List getAllTables(ConnectionSession sourceSession return Objects.requireNonNull(sourceSession.getSyncJdbcExecutor(ConnectionSessionConstants.BACKEND_DS_KEY) .execute((ConnectionCallback>) con -> SchemaPluginUtil.getTableExtension( sourceSession.getDialectType()) - .list(con, schemaName))) + .list(con, schemaName, DBObjectType.TABLE))) .stream().map(o -> { DataArchiveTableConfig config = new DataArchiveTableConfig(); config.setTableName(o.getName()); @@ -180,7 +181,7 @@ public void supportDataArchivingLink(ConnectionConfig sourceDs, ConnectionConfig targetDs.getRegion())); } if (sourceDs.getDialectType().isMysql()) { - if (!targetDs.getDialectType().isMysql()) { + if (!targetDs.getDialectType().isMysql() && targetDs.getDialectType() != DialectType.FILE_SYSTEM) { throw new UnsupportedException( String.format("Unsupported data link from %s to %s.", sourceDs.getDialectType(), targetDs.getDialectType())); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/processor/DataArchivePreprocessor.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/processor/DataArchivePreprocessor.java index 63c3b30f67..dad948983c 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/processor/DataArchivePreprocessor.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/processor/DataArchivePreprocessor.java @@ -18,18 +18,14 @@ import org.springframework.beans.factory.annotation.Autowired; import com.oceanbase.odc.core.session.ConnectionSession; -import com.oceanbase.odc.core.session.ConnectionSessionConstants; import com.oceanbase.odc.core.session.ConnectionSessionFactory; -import com.oceanbase.odc.core.shared.constant.DialectType; -import com.oceanbase.odc.core.shared.exception.UnsupportedException; -import com.oceanbase.odc.plugin.connect.api.InformationExtensionPoint; +import com.oceanbase.odc.core.shared.PreConditions; +import com.oceanbase.odc.core.shared.constant.ErrorCodes; import com.oceanbase.odc.service.connection.database.DatabaseService; import com.oceanbase.odc.service.connection.database.model.Database; import com.oceanbase.odc.service.connection.model.ConnectionConfig; import com.oceanbase.odc.service.dlm.DLMConfiguration; -import com.oceanbase.odc.service.dlm.DLMTableStructureSynchronizer; import com.oceanbase.odc.service.dlm.model.DataArchiveParameters; -import com.oceanbase.odc.service.plugin.ConnectionPluginUtil; import com.oceanbase.odc.service.schedule.model.OperationType; import com.oceanbase.odc.service.schedule.model.ScheduleChangeParams; import com.oceanbase.odc.service.schedule.model.ScheduleType; @@ -65,12 +61,15 @@ public void process(ScheduleChangeParams req) { Database sourceDb = databaseService.detail(parameters.getSourceDatabaseId()); Database targetDb = databaseService.detail(parameters.getTargetDataBaseId()); supportDataArchivingLink(sourceDb.getDataSource(), targetDb.getDataSource()); + if (!parameters.getSyncTableStructure().isEmpty()) { + PreConditions.validArgumentState(sourceDb.getDialectType() != targetDb.getDialectType(), + ErrorCodes.UnsupportedSyncTableStructure, + new Object[] {sourceDb.getDialectType(), targetDb.getDialectType()}, null); + } ConnectionConfig sourceDs = sourceDb.getDataSource(); sourceDs.setDefaultSchema(sourceDb.getName()); ConnectionSessionFactory sourceSessionFactory = new DefaultConnectSessionFactory(sourceDs); - ConnectionSessionFactory targetSessionFactory = new DefaultConnectSessionFactory(targetDb.getDataSource()); ConnectionSession sourceSession = sourceSessionFactory.generateSession(); - ConnectionSession targetSession = targetSessionFactory.generateSession(); try { if (parameters.isFullDatabase()) { parameters.setTables(getAllTables(sourceSession, sourceDb.getName())); @@ -78,36 +77,9 @@ public void process(ScheduleChangeParams req) { if (parameters.getTables().isEmpty()) { throw new IllegalArgumentException("The table list is empty."); } - DialectType sourceDbType = sourceSession.getDialectType(); - DialectType targetDbType = targetSession.getDialectType(); - InformationExtensionPoint sourceInformation = - ConnectionPluginUtil.getInformationExtension(sourceDbType); - InformationExtensionPoint targetInformation = - ConnectionPluginUtil.getInformationExtension(targetDbType); - String sourceDbVersion = - sourceSession.getSyncJdbcExecutor(ConnectionSessionConstants.BACKEND_DS_KEY).execute( - sourceInformation::getDBVersion); - String targetDbVersion = - targetSession.getSyncJdbcExecutor(ConnectionSessionConstants.BACKEND_DS_KEY).execute( - targetInformation::getDBVersion); - if (!parameters.getSyncTableStructure().isEmpty()) { - boolean supportedSyncTableStructure = DLMTableStructureSynchronizer.isSupportedSyncTableStructure( - sourceDbType, sourceDbVersion, targetDbType, targetDbVersion); - if (!supportedSyncTableStructure) { - log.warn( - "Synchronization of table structure is unsupported,sourceDbType={},sourceDbVersion={},targetDbType={},targetDbVersion={}", - sourceDbType, - sourceDbVersion, targetDbType, targetDbVersion); - throw new UnsupportedException(String.format( - "Synchronization of table structure is unsupported,sourceDbType=%s,sourceDbVersion=%s,targetDbType=%s,targetDbVersion=%s", - sourceDbType, - sourceDbVersion, targetDbType, targetDbVersion)); - } - } checkTableAndCondition(sourceSession, sourceDb, parameters.getTables(), parameters.getVariables()); } finally { sourceSession.expire(); - targetSession.expire(); } log.info("Data archive preprocessing has been completed."); } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/processor/ScheduleChangePreprocessor.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/processor/ScheduleChangePreprocessor.java index 43359d91cc..729895ebe8 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/processor/ScheduleChangePreprocessor.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/schedule/processor/ScheduleChangePreprocessor.java @@ -25,6 +25,7 @@ import org.springframework.stereotype.Component; import com.oceanbase.odc.core.shared.exception.UnsupportedException; +import com.oceanbase.odc.service.connection.ConnectionService; import com.oceanbase.odc.service.connection.database.DatabaseService; import com.oceanbase.odc.service.connection.database.model.Database; import com.oceanbase.odc.service.dlm.model.DataArchiveParameters; @@ -53,6 +54,8 @@ public class ScheduleChangePreprocessor implements InitializingBean { @Autowired private DatabaseService databaseService; @Autowired + private ConnectionService connectionService; + @Autowired private ScheduleService scheduleService; @Autowired private List preprocessors; @@ -94,7 +97,7 @@ public void afterPropertiesSet() throws Exception { } private void adaptScheduleChangeParams(ScheduleChangeParams req) { - Database srcDb = databaseService.detail(getTargetDatabaseId(req)); + Database srcDb = databaseService.detailSkipPermissionCheckForRead(getTargetDatabaseId(req)); req.setProjectId(srcDb.getProject().getId()); req.setProjectName(srcDb.getProject().getName()); req.setConnectionId(srcDb.getDataSource().getId()); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/ConnectConsoleService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/ConnectConsoleService.java index fe320b3451..f0b0526e6b 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/ConnectConsoleService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/ConnectConsoleService.java @@ -83,8 +83,8 @@ import com.oceanbase.odc.service.connection.model.ConnectionConfig; import com.oceanbase.odc.service.db.browser.DBSchemaAccessors; import com.oceanbase.odc.service.db.session.DefaultDBSessionManage; +import com.oceanbase.odc.service.db.session.KillResult; import com.oceanbase.odc.service.db.session.KillSessionOrQueryReq; -import com.oceanbase.odc.service.db.session.KillSessionResult; import com.oceanbase.odc.service.dml.ValueEncodeType; import com.oceanbase.odc.service.feature.AllFeatures; import com.oceanbase.odc.service.iam.auth.AuthenticationFacade; @@ -246,10 +246,7 @@ public SqlAsyncExecuteResp streamExecute(@NotNull String sessionId, PreConditions.lessThanOrEqualTo("sqlLength", LimitMetric.SQL_LENGTH, StringUtils.length(request.getSql()), maxSqlLength); } - SqlAsyncExecuteResp result = filterKillSession(connectionSession, request); - if (result != null) { - return result; - } + List sqls = request.ifSplitSqls() ? SqlUtils.splitWithOffset(connectionSession, request.getSql(), sessionProperties.isOracleRemoveCommentPrefix()) @@ -287,6 +284,7 @@ public SqlAsyncExecuteResp streamExecute(@NotNull String sessionId, .collect(Collectors.toList()); try { if (!sqlInterceptService.preHandle(request, response, connectionSession, executeContext)) { + response.setApprovalRequired(true); return response; } } finally { @@ -504,7 +502,7 @@ public boolean killCurrentQuery(@NotNull String sessionId) { } @SkipAuthorize - public List killSessionOrQuery(KillSessionOrQueryReq request) { + public List killSessionOrQuery(KillSessionOrQueryReq request) { if (!connectionService.checkPermission( Long.valueOf(request.getDatasourceId()), Collections.singletonList("update"))) { throw new AccessDeniedException(); @@ -543,34 +541,6 @@ private boolean validateSqlSemantics(String sql, ConnectionSession session) { return false; } - /** - * for some special sql execution(eg. kill session). This will be required to connect to specific - * observer - * - * @param connectionSession connection engine - * @param request odc sql object - * @return result of sql execution - */ - private SqlAsyncExecuteResp filterKillSession(ConnectionSession connectionSession, SqlAsyncExecuteReq request) - throws Exception { - String sqlScript = request.getSql().trim().toLowerCase(); - if (!sqlScript.startsWith("kill ") || !sqlScript.contains("/*")) { - return null; - } - List sqlTuples = SqlTuple.newTuples( - Arrays.stream(sqlScript.split(";")).filter(StringUtils::isNotBlank).collect(Collectors.toList())); - List results = - defaultDbSessionManage.executeKillSession(connectionSession, sqlTuples, sqlScript); - - AsyncExecuteContext executeContext = - new AsyncExecuteContext(sqlTuples, new HashMap<>()); - Future> successFuture = FutureResult.successResult(results); - executeContext.setFuture(successFuture); - executeContext.addSqlExecutionResults(successFuture.get()); - String id = ConnectionSessionUtil.setExecuteContext(connectionSession, executeContext); - return SqlAsyncExecuteResp.newSqlAsyncExecuteResp(id, sqlTuples); - } - private SqlExecuteResult generateResult(@NonNull ConnectionSession connectionSession, @NonNull JdbcGeneralResult generalResult, @NonNull Map cxt) { SqlExecuteResult result = new SqlExecuteResult(generalResult); @@ -583,7 +553,7 @@ private SqlExecuteResult generateResult(@NonNull ConnectionSession connectionSes log.warn("Failed to init sql type", e); } try (TraceStage s = watch.start(SqlExecuteStages.INIT_EDITABLE_INFO)) { - resultTable = result.initEditableInfo(); + resultTable = result.initEditableInfo(connectionSession, cxt); } catch (Exception e) { log.warn("Failed to init editable info", e); } @@ -620,5 +590,4 @@ private Integer checkQueryLimit(Integer queryLimit) { } return queryLimit; } - } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/DBSessionManageFacade.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/DBSessionManageFacade.java index 8720cc17de..adbd554d8f 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/DBSessionManageFacade.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/DBSessionManageFacade.java @@ -20,12 +20,12 @@ import com.oceanbase.odc.core.session.ConnectionSession; import com.oceanbase.odc.core.shared.model.OdcDBSession; +import com.oceanbase.odc.service.db.session.KillResult; import com.oceanbase.odc.service.db.session.KillSessionOrQueryReq; -import com.oceanbase.odc.service.db.session.KillSessionResult; public interface DBSessionManageFacade { - List killSessionOrQuery(KillSessionOrQueryReq request); + List killSessionOrQuery(KillSessionOrQueryReq request); boolean supportKillConsoleQuery(ConnectionSession session); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/factory/DefaultConnectSessionFactory.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/factory/DefaultConnectSessionFactory.java index 3d14b881c0..7363549976 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/factory/DefaultConnectSessionFactory.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/factory/DefaultConnectSessionFactory.java @@ -71,13 +71,15 @@ public class DefaultConnectSessionFactory implements ConnectionSessionFactory { private final Boolean autoCommit; private final EventPublisher eventPublisher; private final boolean autoReconnect; + private final boolean keepAlive; @Setter private long sessionTimeoutMillis; @Setter private ConnectionSessionIdGenerator idGenerator; public DefaultConnectSessionFactory(@NonNull ConnectionConfig connectionConfig, - Boolean autoCommit, TaskManagerFactory taskManagerFactory, boolean autoReconnect) { + Boolean autoCommit, TaskManagerFactory taskManagerFactory, boolean autoReconnect, + boolean keepAlive) { this.sessionTimeoutMillis = TimeUnit.MILLISECONDS.convert( ConnectionSessionConstants.SESSION_EXPIRATION_TIME_SECONDS, TimeUnit.SECONDS); this.connectionConfig = connectionConfig; @@ -86,15 +88,16 @@ public DefaultConnectSessionFactory(@NonNull ConnectionConfig connectionConfig, this.eventPublisher = new LocalEventPublisher(); this.idGenerator = new DefaultConnectSessionIdGenerator(); this.autoReconnect = autoReconnect; + this.keepAlive = keepAlive; } public DefaultConnectSessionFactory(@NonNull ConnectionConfig connectionConfig, Boolean autoCommit, TaskManagerFactory taskManagerFactory) { - this(connectionConfig, autoCommit, taskManagerFactory, true); + this(connectionConfig, autoCommit, taskManagerFactory, true, true); } public DefaultConnectSessionFactory(@NonNull ConnectionConfig connectionConfig) { - this(connectionConfig, null, null, true); + this(connectionConfig, null, null, true, false); } @Override @@ -109,7 +112,7 @@ public ConnectionSession generateSession() { private void registerConsoleDataSource(ConnectionSession session) { OBConsoleDataSourceFactory dataSourceFactory = - new OBConsoleDataSourceFactory(connectionConfig, autoCommit, true, autoReconnect); + new OBConsoleDataSourceFactory(connectionConfig, autoCommit, true, autoReconnect, keepAlive); try { JdbcUrlParser urlParser = ConnectionPluginUtil .getConnectionExtension(connectionConfig.getDialectType()) @@ -162,6 +165,7 @@ private void initSession(ConnectionSession session) { ConnectionInfoUtil.initSessionVersion(session); ConnectionSessionUtil.setConsoleSessionResetFlag(session, false); ConnectionInfoUtil.initConsoleConnectionId(session); + ConnectionInfoUtil.initOdpVersionIfExists(session); ConnectionSessionUtil.setConnectionConfig(session, connectionConfig); ConnectionSessionUtil.setColumnAccessor(session, new DatasourceColumnAccessor(session)); if (StringUtils.isNotBlank(connectionConfig.getTenantName())) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/factory/OBConsoleDataSourceFactory.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/factory/OBConsoleDataSourceFactory.java index 9f68f7ae2c..7fa3656a8d 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/factory/OBConsoleDataSourceFactory.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/factory/OBConsoleDataSourceFactory.java @@ -78,6 +78,7 @@ public class OBConsoleDataSourceFactory implements CloneableDataSourceFactory { protected UserRole userRole; private String catalogName; private boolean autoReConnect; + private boolean keepAlive; private Map parameters; protected final ConnectionConfig connectionConfig; private final Boolean autoCommit; @@ -87,16 +88,16 @@ public class OBConsoleDataSourceFactory implements CloneableDataSourceFactory { protected final ConnectionExtensionPoint connectionExtensionPoint; public OBConsoleDataSourceFactory(@NonNull ConnectionConfig connectionConfig, Boolean autoCommit) { - this(connectionConfig, autoCommit, true); + this(connectionConfig, autoCommit, true, true, false); } public OBConsoleDataSourceFactory(@NonNull ConnectionConfig connectionConfig, - Boolean autoCommit, boolean initConnection) { - this(connectionConfig, autoCommit, initConnection, true); + Boolean autoCommit, boolean initConnection, boolean keepAlive) { + this(connectionConfig, autoCommit, initConnection, true, keepAlive); } public OBConsoleDataSourceFactory(@NonNull ConnectionConfig connectionConfig, - Boolean autoCommit, boolean initConnection, boolean autoReConnect) { + Boolean autoCommit, boolean initConnection, boolean autoReConnect, boolean keepAlive) { this.autoCommit = autoCommit; this.connectionConfig = connectionConfig; this.initConnection = initConnection; @@ -111,6 +112,7 @@ public OBConsoleDataSourceFactory(@NonNull ConnectionConfig connectionConfig, this.catalogName = connectionConfig.getCatalogName(); this.parameters = getJdbcParams(connectionConfig); this.autoReConnect = autoReConnect; + this.keepAlive = keepAlive; this.connectionExtensionPoint = ConnectionPluginUtil.getConnectionExtension(connectionConfig.getDialectType()); } @@ -202,7 +204,7 @@ public static Map getJdbcParams(@NonNull ConnectionConfig connec @Override public DataSource getDataSource() { String jdbcUrl = getJdbcUrl(); - SingleConnectionDataSource dataSource = new SingleConnectionDataSource(autoReConnect); + SingleConnectionDataSource dataSource = new SingleConnectionDataSource(autoReConnect, keepAlive); dataSource.setEventPublisher(eventPublisher); dataSource.setUrl(jdbcUrl); dataSource.setUsername(username); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/interceptor/SqlConsoleInterceptor.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/interceptor/SqlConsoleInterceptor.java index 35de0eff18..ec0bbc5ebe 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/interceptor/SqlConsoleInterceptor.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/interceptor/SqlConsoleInterceptor.java @@ -170,7 +170,7 @@ private boolean handle(@NonNull SqlAsyncExecuteReq request, @NonNull SqlAsyncExe if (Objects.nonNull(parseResult.getSyntaxError()) && parseResult.getSyntaxError()) { continue; } - if (!allowSqlTypesOpt.get().contains(parseResult.getSqlType().name())) { + if (isViolatedRule(allowSqlTypesOpt.get(), parseResult)) { ruleService.getByRulesetIdAndName(ruleSetId, SqlConsoleRules.ALLOW_SQL_TYPES.getRuleName()) .ifPresent(rule -> { Rule violationRule = new Rule(); @@ -247,6 +247,15 @@ private BasicResult determineSqlType(@NonNull SqlTuple sqlTuple, @NonNull Dialec return basicResult; } + private boolean isViolatedRule(@NonNull List allowSqlTypesOpt, @NonNull BasicResult parseResult) { + for (SqlType sqlType = parseResult.getSqlType(); sqlType != null; sqlType = sqlType.getParentType()) { + if (allowSqlTypesOpt.contains(sqlType.name())) { + return false; + } + } + return true; + } + @Override public int getOrder() { return 2; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/model/SqlAsyncExecuteResp.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/model/SqlAsyncExecuteResp.java index 8086eb6646..79dc05178d 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/model/SqlAsyncExecuteResp.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/model/SqlAsyncExecuteResp.java @@ -39,6 +39,7 @@ public class SqlAsyncExecuteResp { private List sqls; private List unauthorizedDBResources; private boolean logicalSql; + private boolean approvalRequired; public SqlAsyncExecuteResp(boolean logicalSql) { this.logicalSql = logicalSql; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/model/SqlExecuteResult.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/model/SqlExecuteResult.java index f201357557..d0a979a73a 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/model/SqlExecuteResult.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/model/SqlExecuteResult.java @@ -30,6 +30,7 @@ import java.util.Set; import java.util.TreeSet; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang.StringUtils; @@ -38,6 +39,7 @@ import com.oceanbase.odc.common.util.ExceptionUtils; import com.oceanbase.odc.common.util.TraceStage; import com.oceanbase.odc.common.util.TraceWatch; +import com.oceanbase.odc.common.util.VersionUtils; import com.oceanbase.odc.core.session.ConnectionSession; import com.oceanbase.odc.core.session.ConnectionSessionUtil; import com.oceanbase.odc.core.shared.constant.DialectType; @@ -52,6 +54,7 @@ import com.oceanbase.odc.core.sql.parser.AbstractSyntaxTree; import com.oceanbase.odc.core.sql.parser.AbstractSyntaxTreeFactories; import com.oceanbase.odc.service.common.util.PLObjectErrMsgUtils; +import com.oceanbase.odc.service.db.browser.DBSchemaAccessors; import com.oceanbase.odc.service.feature.AllFeatures; import com.oceanbase.odc.service.session.model.OdcResultSetMetaData.OdcTable; import com.oceanbase.odc.service.sqlcheck.model.CheckViolation; @@ -63,6 +66,7 @@ import com.oceanbase.tools.dbbrowser.parser.result.BasicResult; import com.oceanbase.tools.dbbrowser.schema.DBSchemaAccessor; +import cn.hutool.core.collection.CollectionUtil; import lombok.Data; import lombok.Getter; import lombok.NoArgsConstructor; @@ -82,6 +86,8 @@ @NoArgsConstructor @Slf4j public class SqlExecuteResult { + public static final String SHOW_EXTERNAL_TABLES_IN_SCHEMA = "SHOW_EXTERNAL_TABLES_IN_SCHEMA"; + public static final String MIN_OB_VERSION_FOR_EXTERNAL_TABLE = "4.3.2"; private List columnLabels; private List columns; private SqlExecuteStatus status = SqlExecuteStatus.CREATED; @@ -135,8 +141,9 @@ public void initWarningMessage(ConnectionSession connectionSession) { } } - public OdcTable initEditableInfo() { + public OdcTable initEditableInfo(@NonNull ConnectionSession connectionSession, @NonNull Map cxt) { boolean editable = true; + editable = !checkContainsExternalTable(connectionSession, cxt); OdcTable resultTable = null; Set relatedTablesOrViews = new HashSet<>(); if (Objects.isNull(this.resultSetMetaData)) { @@ -204,6 +211,39 @@ public OdcTable initEditableInfo() { return resultTable; } + private boolean checkContainsExternalTable(@NonNull ConnectionSession connectionSession, + @NonNull Map cxt) { + DialectType dialectType = connectionSession.getDialectType(); + if (dialectType == DialectType.OB_MYSQL || dialectType == DialectType.OB_ORACLE) { + String obVersion = ConnectionSessionUtil.getVersion(connectionSession); + if (VersionUtils.isGreaterThanOrEqualsTo(obVersion, MIN_OB_VERSION_FOR_EXTERNAL_TABLE) + && resultSetMetaData != null) { + List columnList = resultSetMetaData.getFieldMetaDataList(); + Map schemaAndTable2Column = columnList.stream() + .collect(Collectors.groupingBy(jcmd -> jcmd.getCatalogName() + "." + jcmd.getTableName(), + Collectors.collectingAndThen(Collectors.toList(), + lst -> lst.get(0)))); + Set columnSet = new HashSet<>(schemaAndTable2Column.values()); + for (JdbcColumnMetaData columnMetaData : columnSet) { + String catalogName = columnMetaData.getCatalogName(); + String tableName = columnMetaData.getTableName(); + Map> schema2ExternalTables = + (Map>) cxt.computeIfAbsent(SHOW_EXTERNAL_TABLES_IN_SCHEMA, + k -> new HashMap<>()); + List externalTables = schema2ExternalTables.computeIfAbsent(catalogName, k -> { + DBSchemaAccessor schemaAccessor = DBSchemaAccessors.create(connectionSession); + return schemaAccessor.showExternalTables(catalogName); + }); + if (CollectionUtil.contains(externalTables, tableName)) { + return true; + } + } + } + } + return false; + } + + public void initColumnInfo(@NonNull ConnectionSession connectionSession, OdcTable resultTable, @NonNull DBSchemaAccessor schemaAccessor) { if (this.resultSetMetaData == null) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/util/DBSchemaExtractor.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/util/DBSchemaExtractor.java index 03186e1f7a..c3a9b69b60 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/session/util/DBSchemaExtractor.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/session/util/DBSchemaExtractor.java @@ -149,7 +149,7 @@ private static Set listDBSchemas(AbstractSyntaxTree ast, Diale visitor.visit(ast.getRoot()); identities = visitor.getIdentities(); } else { - OBMySQLRelationFactorVisitor visitor = new OBMySQLRelationFactorVisitor(); + OBMySQLRelationFactorVisitor visitor = new OBMySQLRelationFactorVisitor(defaultSchema); visitor.visit(ast.getRoot()); identities = visitor.getIdentities(); } @@ -167,7 +167,7 @@ private static Set listDBSchemas(AbstractSyntaxTree ast, Diale visitor.visit(ast.getRoot()); identities = visitor.getIdentities(); } else { - OBOracleRelationFactorVisitor visitor = new OBOracleRelationFactorVisitor(); + OBOracleRelationFactorVisitor visitor = new OBOracleRelationFactorVisitor(defaultSchema); visitor.visit(ast.getRoot()); identities = visitor.getIdentities(); } @@ -200,6 +200,36 @@ private static class OBMySQLRelationFactorVisitor extends OBParserBaseVisitor identities = new HashSet<>(); + private final String defaultSchema; + + private OBMySQLRelationFactorVisitor(String defaultSchema) { + this.defaultSchema = defaultSchema; + } + + @Override + public RelationFactor visitDrop_function_stmt( + com.oceanbase.tools.sqlparser.obmysql.OBParser.Drop_function_stmtContext ctx) { + RelationFactor relationFactor = MySQLFromReferenceFactory.getRelationFactor( + ctx.relation_factor().normal_relation_factor()); + return extractIdentityExceptTableAndView(relationFactor); + } + + @Override + public RelationFactor visitDrop_procedure_stmt( + com.oceanbase.tools.sqlparser.obmysql.OBParser.Drop_procedure_stmtContext ctx) { + RelationFactor relationFactor = MySQLFromReferenceFactory.getRelationFactor( + ctx.relation_factor().normal_relation_factor()); + return extractIdentityExceptTableAndView(relationFactor); + } + + @Override + public RelationFactor visitDrop_trigger_stmt( + com.oceanbase.tools.sqlparser.obmysql.OBParser.Drop_trigger_stmtContext ctx) { + RelationFactor relationFactor = MySQLFromReferenceFactory.getRelationFactor( + ctx.relation_factor().normal_relation_factor()); + return extractIdentityExceptTableAndView(relationFactor); + } + @Override public RelationFactor visitPartition_option(Partition_optionContext ctx) { return null; @@ -289,6 +319,15 @@ private void addRelationFactor(RelationFactor rf) { identities.add(new DBSchemaIdentity(rf.getSchema(), rf.getRelation())); } } + + private RelationFactor extractIdentityExceptTableAndView(RelationFactor relationFactor) { + if (StringUtils.isNotBlank(relationFactor.getSchema())) { + identities.add(new DBSchemaIdentity(relationFactor.getSchema(), null)); + } else if (StringUtils.isNotBlank(defaultSchema)) { + identities.add(new DBSchemaIdentity(defaultSchema, null)); + } + return null; + } } @@ -353,6 +392,62 @@ private static class OBOracleRelationFactorVisitor private final Set identities = new HashSet<>(); + private final String defaultSchema; + + private OBOracleRelationFactorVisitor() { + this.defaultSchema = null; + } + + private OBOracleRelationFactorVisitor(String defaultSchema) { + this.defaultSchema = defaultSchema; + } + + @Override + public RelationFactor visitDrop_package_stmt(OBParser.Drop_package_stmtContext ctx) { + RelationFactor relationFactor = OracleFromReferenceFactory.getRelationFactor(ctx.relation_factor()); + return extractIdentityExceptTableAndView(relationFactor); + } + + @Override + public RelationFactor visitDrop_procedure_stmt(OBParser.Drop_procedure_stmtContext ctx) { + RelationFactor relationFactor = OracleFromReferenceFactory.getRelationFactor(ctx.relation_factor()); + return extractIdentityExceptTableAndView(relationFactor); + } + + @Override + public RelationFactor visitDrop_function_stmt(OBParser.Drop_function_stmtContext ctx) { + RelationFactor relationFactor = OracleFromReferenceFactory.getRelationFactor(ctx.relation_factor()); + return extractIdentityExceptTableAndView(relationFactor); + } + + @Override + public RelationFactor visitDrop_trigger_stmt(OBParser.Drop_trigger_stmtContext ctx) { + RelationFactor relationFactor = OracleFromReferenceFactory.getRelationFactor(ctx.relation_factor()); + return extractIdentityExceptTableAndView(relationFactor); + } + + @Override + public RelationFactor visitDrop_type_stmt(OBParser.Drop_type_stmtContext ctx) { + RelationFactor relationFactor = OracleFromReferenceFactory.getRelationFactor(ctx.relation_factor()); + return extractIdentityExceptTableAndView(relationFactor); + } + + @Override + public RelationFactor visitDrop_sequence_stmt(OBParser.Drop_sequence_stmtContext ctx) { + RelationFactor relationFactor = OracleFromReferenceFactory.getRelationFactor(ctx.relation_factor()); + return extractIdentityExceptTableAndView(relationFactor); + } + + @Override + public RelationFactor visitDrop_synonym_stmt(OBParser.Drop_synonym_stmtContext ctx) { + if (Objects.nonNull(ctx.database_factor()) && StringUtils.isNotBlank(ctx.database_factor().getText())) { + identities.add(new DBSchemaIdentity(ctx.database_factor().getText(), null)); + } else if (StringUtils.isNotBlank(defaultSchema)) { + identities.add(new DBSchemaIdentity(defaultSchema, null)); + } + return null; + } + @Override public RelationFactor visitPartition_option(OBParser.Partition_optionContext ctx) { return null; @@ -463,6 +558,14 @@ private void addRelationFactor(RelationFactor rf) { } } + private RelationFactor extractIdentityExceptTableAndView(RelationFactor relationFactor) { + if (StringUtils.isNotBlank(relationFactor.getSchema())) { + identities.add(new DBSchemaIdentity(relationFactor.getSchema(), null)); + } else if (StringUtils.isNotBlank(defaultSchema)) { + identities.add(new DBSchemaIdentity(defaultSchema, null)); + } + return null; + } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/shadowtable/ShadowTableComparingService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/shadowtable/ShadowTableComparingService.java index 97d261ba67..3e90026066 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/shadowtable/ShadowTableComparingService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/shadowtable/ShadowTableComparingService.java @@ -222,6 +222,7 @@ private void checkPermission(ShadowTableComparingTaskEntity taskEntity) { } return; } - flowInstanceService.mapFlowInstance(taskEntity.getFlowInstanceId(), flowInstance -> flowInstance, false); + flowInstanceService.mapFlowInstanceWithReadPermission(taskEntity.getFlowInstanceId(), + flowInstance -> flowInstance); } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/BaseSqlChecker.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/BaseSqlChecker.java index 7729f23a5a..c511590f44 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/BaseSqlChecker.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/BaseSqlChecker.java @@ -59,7 +59,7 @@ public List check(@NonNull String sqlScript) { List sqls = null; if (dialectType.isMysql() || dialectType.isDoris()) { sqls = splitByCommentProcessor(sqlScript); - } else if (dialectType == DialectType.OB_ORACLE) { + } else if (dialectType.isOracle()) { if (DEFAULT_DELIMITER.equals(this.delimiter)) { // 如果用户没有改 delimiter 就用现成的分句逻辑 SqlSplitter sqlSplitter = new SqlSplitter(PlSqlLexer.class, this.delimiter); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/SqlCheckService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/SqlCheckService.java index 7b6359ba7f..bf7be02bde 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/SqlCheckService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/SqlCheckService.java @@ -141,7 +141,7 @@ public List check(@NotNull Long environmentId, @NonNull String d } Environment env = this.environmentService.detail(environmentId); List rules = this.ruleService.list(env.getRulesetId(), QueryRuleMetadataParams.builder().build()); - OBConsoleDataSourceFactory factory = new OBConsoleDataSourceFactory(config, true, false); + OBConsoleDataSourceFactory factory = new OBConsoleDataSourceFactory(config, true, false, false); factory.resetSchema(origin -> OBConsoleDataSourceFactory.getSchema(databaseName, config.getDialectType())); SqlCheckContext checkContext = new SqlCheckContext((long) sqls.size()); try (SingleConnectionDataSource dataSource = (SingleConnectionDataSource) factory.getDataSource()) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/factory/CreateTableAsExistsFactory.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/factory/CreateTableAsExistsFactory.java new file mode 100644 index 0000000000..1f58504347 --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/factory/CreateTableAsExistsFactory.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.sqlcheck.factory; + +import java.util.Map; + +import com.oceanbase.odc.core.shared.constant.DialectType; +import com.oceanbase.odc.service.sqlcheck.SqlCheckRule; +import com.oceanbase.odc.service.sqlcheck.SqlCheckRuleFactory; +import com.oceanbase.odc.service.sqlcheck.model.SqlCheckRuleType; +import com.oceanbase.odc.service.sqlcheck.rule.CreateTableAsExists; + +import lombok.NonNull; + +/** + * @description: + * @author: zijia.cj + * @date: 2025/1/8 20:33 + * @since: 4.3.3 + */ +public class CreateTableAsExistsFactory implements SqlCheckRuleFactory { + @Override + public SqlCheckRuleType getSupportsType() { + return SqlCheckRuleType.CREATE_TABLE_AS_EXISTS; + } + + @Override + public SqlCheckRule generate(@NonNull DialectType dialectType, Map parameters) { + return new CreateTableAsExists(); + } +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/factory/CreateTableLikeExistsFactory.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/factory/CreateTableLikeExistsFactory.java new file mode 100644 index 0000000000..2a783209a6 --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/factory/CreateTableLikeExistsFactory.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.sqlcheck.factory; + +import java.util.Map; + +import com.oceanbase.odc.core.shared.constant.DialectType; +import com.oceanbase.odc.service.sqlcheck.SqlCheckRule; +import com.oceanbase.odc.service.sqlcheck.SqlCheckRuleFactory; +import com.oceanbase.odc.service.sqlcheck.model.SqlCheckRuleType; +import com.oceanbase.odc.service.sqlcheck.rule.CreateTableLikeExists; + +import lombok.NonNull; + +/** + * @description: + * @author: zijia.cj + * @date: 2025/1/8 18:41 + * @since: 4.3.3 + */ +public class CreateTableLikeExistsFactory implements SqlCheckRuleFactory { + @Override + public SqlCheckRuleType getSupportsType() { + return SqlCheckRuleType.CREATE_TABLE_LIKE_EXISTS; + } + + @Override + public SqlCheckRule generate(@NonNull DialectType dialectType, Map parameters) { + return new CreateTableLikeExists(); + } +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/factory/RestrictColumnNameCaseFactory.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/factory/RestrictColumnNameCaseFactory.java index a3c66b85fd..ffcec07344 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/factory/RestrictColumnNameCaseFactory.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/factory/RestrictColumnNameCaseFactory.java @@ -46,7 +46,7 @@ public SqlCheckRule generate(@NonNull DialectType dialectType, Map parameters) { String key = getParameterNameKey("allowed-max-sql-affected-count"); - if (parameters == null || parameters.isEmpty() || parameters.get(key) == null) { - return new MySQLAffectedRowsExceedLimit(1000L, dialectType, jdbc); + long maxSqlAffectedRows = DEFAULT_MAX_SQL_AFFECTED_ROWS; + if (parameters != null && !parameters.isEmpty() && parameters.get(key) != null) { + maxSqlAffectedRows = Long.valueOf(parameters.get(key).toString()); + } + switch (dialectType) { + case ORACLE: + case OB_ORACLE: + return new OracleAffectedRowsExceedLimit(maxSqlAffectedRows, dialectType, + jdbc); + case MYSQL: + case OB_MYSQL: + return new MySQLAffectedRowsExceedLimit(maxSqlAffectedRows, dialectType, + jdbc); + default: + throw new IllegalArgumentException("Unsupported dialect type: " + dialectType); } - return new MySQLAffectedRowsExceedLimit(Long.valueOf(parameters.get(key).toString()), dialectType, jdbc); } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/factory/TruncateTableExistsFactory.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/factory/TruncateTableExistsFactory.java index 9c410f575a..1caadad998 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/factory/TruncateTableExistsFactory.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/factory/TruncateTableExistsFactory.java @@ -29,7 +29,7 @@ public class TruncateTableExistsFactory implements SqlCheckRuleFactory { @Override public SqlCheckRuleType getSupportsType() { - return SqlCheckRuleType.TRUNCATE_TBLE_EXISTS; + return SqlCheckRuleType.TRUNCATE_TABLE_EXISTS; } @Override diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/factory/Unable2JudgeAffectedRowsFactory.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/factory/Unable2JudgeAffectedRowsFactory.java index 224931cbe2..434cea6246 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/factory/Unable2JudgeAffectedRowsFactory.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/factory/Unable2JudgeAffectedRowsFactory.java @@ -23,7 +23,7 @@ import com.oceanbase.odc.service.sqlcheck.SqlCheckRule; import com.oceanbase.odc.service.sqlcheck.SqlCheckRuleFactory; import com.oceanbase.odc.service.sqlcheck.model.SqlCheckRuleType; -import com.oceanbase.odc.service.sqlcheck.rule.MySQLAffectedRowsExceedLimit; +import com.oceanbase.odc.service.sqlcheck.rule.BaseAffectedRowsExceedLimit; import com.oceanbase.odc.service.sqlcheck.rule.Unable2JudgeAffectedRows; import lombok.NonNull; @@ -44,7 +44,7 @@ public SqlCheckRuleType getSupportsType() { @Override public SqlCheckRule generate(@NonNull DialectType dialectType, Map parameters) { SqlAffectedRowsFactory sqlAffectedRowsFactory = new SqlAffectedRowsFactory(this.jdbc); - MySQLAffectedRowsExceedLimit targetRule = (MySQLAffectedRowsExceedLimit) sqlAffectedRowsFactory + BaseAffectedRowsExceedLimit targetRule = (BaseAffectedRowsExceedLimit) sqlAffectedRowsFactory .generate(dialectType, parameters); return new Unable2JudgeAffectedRows(targetRule); } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/model/SqlCheckRuleType.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/model/SqlCheckRuleType.java index 9001b43a42..53850924bf 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/model/SqlCheckRuleType.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/model/SqlCheckRuleType.java @@ -243,7 +243,7 @@ public enum SqlCheckRuleType implements Translatable { /** * Truncate table 语句存在 */ - TRUNCATE_TBLE_EXISTS("truncate-table-exists"), + TRUNCATE_TABLE_EXISTS("truncate-table-exists"), /** * Restrict the number of lines affected by SQL */ @@ -251,7 +251,16 @@ public enum SqlCheckRuleType implements Translatable { /** * Unable to judge restrict the number of lines affected by SQL */ - ESTIMATE_SQL_AFFECTED_ROWS_FAILED("estimate-sql-affected-rows-failed"); + ESTIMATE_SQL_AFFECTED_ROWS_FAILED("estimate-sql-affected-rows-failed"), + /** + * Create like statement exists + */ + CREATE_TABLE_LIKE_EXISTS("create-table-like-exists"), + /** + * Create as statement exists + */ + CREATE_TABLE_AS_EXISTS("create-table-as-exists"); + private final String name; private static final String NAME_CODE = "name"; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/BaseAffectedRowsExceedLimit.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/BaseAffectedRowsExceedLimit.java index 5e7e204f68..5dcb722c4b 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/BaseAffectedRowsExceedLimit.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/BaseAffectedRowsExceedLimit.java @@ -18,6 +18,8 @@ import java.util.Collections; import java.util.List; +import org.springframework.jdbc.core.JdbcOperations; + import com.oceanbase.odc.service.sqlcheck.SqlCheckContext; import com.oceanbase.odc.service.sqlcheck.SqlCheckRule; import com.oceanbase.odc.service.sqlcheck.SqlCheckUtil; @@ -63,6 +65,78 @@ public List check(@NonNull Statement statement, @NonNull SqlChec return Collections.emptyList(); } + /** + * OB execute 'explain' statement + * + * @param originalSql target sql + * @param jdbcOperations jdbc Object + * @return affected rows + */ + public long getOBAffectedRows(String originalSql, JdbcOperations jdbcOperations) { + /** + * 
+         *     obclient> explain delete from T1 where 1=1;
+         * +-------------------------------------------------------+
+         * | Query Plan                                            |
+         * +-------------------------------------------------------+
+         * | =================================================     |
+         * | |ID|OPERATOR         |NAME|EST.ROWS|EST.TIME(us)|     |
+         * | -------------------------------------------------     |
+         * | |0 |DELETE           |    |2       |21          |     |
+         * | |1 |└─TABLE FULL SCAN|t1  |2       |5           |     |
+         * | =================================================     |
+         * | Outputs & filters:                                    |
+         * | -------------------------------------                 |
+         * |   0 - output(nil), filter(nil)                        |
+         * |       table_columns([{t1: ({t1: (t1.id)})}])          |
+         * |   1 - output([t1.id]), filter(nil), rowset=16         |
+         * |       access([t1.id]), partitions(p0)                 |
+         * |       is_index_back=false, is_global_index=false,     |
+         * |       range_key([t1.id]), range(MIN ; MAX)always true |
+         * +-------------------------------------------------------+
+         * 14 rows in set (0.00 sec)
+         *
+ */ + String explainSql = "EXPLAIN " + originalSql; + List queryResults = jdbcOperations.query(explainSql, (rs, rowNum) -> rs.getString("Query Plan")); + return getOBAndOracleAffectRowsFromResult(queryResults); + } + + public long getOBAndOracleAffectRowsFromResult(List queryResults) { + long estRowsValue = 0; + for (int rowNum = 0; rowNum < queryResults.size(); rowNum++) { + String resultRow = queryResults.get(rowNum); + estRowsValue = getEstRowsValue(resultRow); + if (estRowsValue != 0) { + break; + } + } + return estRowsValue; + } + + private long getEstRowsValue(String singleRow) { + String[] parts = singleRow.split("\\|"); + if (parts.length > 5) { + String value = parts[4].trim(); + return parseLong(value); + } + return 0; + } + + /** + * Safely parse a long value. + * + * @param value string to parse + * @return parsed long or 0 if parsing fails + */ + private long parseLong(String value) { + try { + return Long.parseLong(value); + } catch (NumberFormatException e) { + return 0; + } + } + public abstract long getStatementAffectedRows(Statement statement) throws Exception; } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/BaseMissingRequiredColumns.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/BaseMissingRequiredColumns.java index d415c58f86..3a25d2515e 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/BaseMissingRequiredColumns.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/BaseMissingRequiredColumns.java @@ -18,6 +18,7 @@ import java.util.Collections; import java.util.HashSet; import java.util.List; +import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; @@ -56,7 +57,11 @@ public List check(@NonNull Statement statement, @NonNull SqlChec if (!(statement instanceof CreateTable)) { return Collections.emptyList(); } - List columns = ((CreateTable) statement).getColumnDefinitions().stream() + CreateTable createTable = (CreateTable) statement; + if (Objects.nonNull(createTable.getLikeTable()) || Objects.nonNull(createTable.getAs())) { + return Collections.emptyList(); + } + List columns = createTable.getColumnDefinitions().stream() .map(d -> unquoteIdentifier(d.getColumnReference().getColumn())).collect(Collectors.toList()); Set tmp = new HashSet<>(requiredColumns); tmp.removeIf(s -> columns.contains(unquoteIdentifier(s))); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/ColumnCharsetExists.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/ColumnCharsetExists.java index b8cc793e36..5a633d62f4 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/ColumnCharsetExists.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/ColumnCharsetExists.java @@ -63,7 +63,7 @@ public List check(@NonNull Statement statement, @NonNull SqlChec @Override public List getSupportsDialectTypes() { - return Arrays.asList(DialectType.OB_ORACLE, DialectType.MYSQL, DialectType.OB_MYSQL); + return Arrays.asList(DialectType.OB_ORACLE, DialectType.MYSQL, DialectType.OB_MYSQL, DialectType.ORACLE); } private List builds(String sql, Stream stream) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/ColumnCollationExists.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/ColumnCollationExists.java index e8654ff338..87fde2bab1 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/ColumnCollationExists.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/ColumnCollationExists.java @@ -64,7 +64,7 @@ public List check(@NonNull Statement statement, @NonNull SqlChec @Override public List getSupportsDialectTypes() { return Arrays.asList(DialectType.OB_ORACLE, DialectType.MYSQL, DialectType.OB_MYSQL, - DialectType.ODP_SHARDING_OB_MYSQL); + DialectType.ODP_SHARDING_OB_MYSQL, DialectType.ORACLE); } private List builds(String sql, Stream stream) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/ColumnNameInBlackList.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/ColumnNameInBlackList.java index 674726d3b1..6060baf29d 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/ColumnNameInBlackList.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/ColumnNameInBlackList.java @@ -79,7 +79,7 @@ public List check(@NonNull Statement statement, @NonNull SqlChec @Override public List getSupportsDialectTypes() { return Arrays.asList(DialectType.OB_MYSQL, DialectType.MYSQL, DialectType.OB_ORACLE, - DialectType.ODP_SHARDING_OB_MYSQL); + DialectType.ODP_SHARDING_OB_MYSQL, DialectType.ORACLE); } private String unquoteIdentifier(String name) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/CreateTableAsExists.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/CreateTableAsExists.java new file mode 100644 index 0000000000..815a1006de --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/CreateTableAsExists.java @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.sqlcheck.rule; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +import com.oceanbase.odc.core.shared.constant.DialectType; +import com.oceanbase.odc.service.sqlcheck.SqlCheckContext; +import com.oceanbase.odc.service.sqlcheck.SqlCheckRule; +import com.oceanbase.odc.service.sqlcheck.SqlCheckUtil; +import com.oceanbase.odc.service.sqlcheck.model.CheckViolation; +import com.oceanbase.odc.service.sqlcheck.model.SqlCheckRuleType; +import com.oceanbase.tools.sqlparser.statement.Statement; +import com.oceanbase.tools.sqlparser.statement.createtable.CreateTable; + +import lombok.NonNull; + +/** + * @description: + * @author: zijia.cj + * @date: 2025/1/8 17:34 + * @since: 4.3.3 + */ +public class CreateTableAsExists implements SqlCheckRule { + @Override + public SqlCheckRuleType getType() { + return SqlCheckRuleType.CREATE_TABLE_AS_EXISTS; + } + + @Override + public List check(@NonNull Statement statement, @NonNull SqlCheckContext context) { + if (!(statement instanceof CreateTable)) { + return Collections.emptyList(); + } + CreateTable createTable = (CreateTable) statement; + if ((Objects.nonNull(createTable.getAs()))) { + return Collections.singletonList(SqlCheckUtil.buildViolation( + statement.getText(), statement, getType(), new Object[] {})); + } + return Collections.emptyList(); + } + + @Override + public List getSupportsDialectTypes() { + return Arrays.asList(DialectType.OB_MYSQL, DialectType.MYSQL, DialectType.OB_ORACLE, + DialectType.ODP_SHARDING_OB_MYSQL, DialectType.ORACLE); + } +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/CreateTableLikeExists.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/CreateTableLikeExists.java new file mode 100644 index 0000000000..709b5455b9 --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/CreateTableLikeExists.java @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.sqlcheck.rule; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +import com.oceanbase.odc.core.shared.constant.DialectType; +import com.oceanbase.odc.service.sqlcheck.SqlCheckContext; +import com.oceanbase.odc.service.sqlcheck.SqlCheckRule; +import com.oceanbase.odc.service.sqlcheck.SqlCheckUtil; +import com.oceanbase.odc.service.sqlcheck.model.CheckViolation; +import com.oceanbase.odc.service.sqlcheck.model.SqlCheckRuleType; +import com.oceanbase.tools.sqlparser.statement.Statement; +import com.oceanbase.tools.sqlparser.statement.createtable.CreateTable; + +import lombok.NonNull; + +/** + * @description: + * @author: zijia.cj + * @date: 2025/1/8 17:31 + * @since: 4.3.3 + */ +public class CreateTableLikeExists implements SqlCheckRule { + @Override + public SqlCheckRuleType getType() { + return SqlCheckRuleType.CREATE_TABLE_LIKE_EXISTS; + } + + @Override + public List check(@NonNull Statement statement, @NonNull SqlCheckContext context) { + if (!(statement instanceof CreateTable)) { + return Collections.emptyList(); + } + CreateTable createTable = (CreateTable) statement; + if (Objects.nonNull(createTable.getLikeTable())) { + return Collections.singletonList(SqlCheckUtil.buildViolation( + statement.getText(), statement, getType(), new Object[] {})); + } + return Collections.emptyList(); + } + + @Override + public List getSupportsDialectTypes() { + return Arrays.asList(DialectType.MYSQL, DialectType.OB_MYSQL, DialectType.ODP_SHARDING_OB_MYSQL); + } +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/ForeignConstraintExists.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/ForeignConstraintExists.java index de1768a090..089f265fb1 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/ForeignConstraintExists.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/ForeignConstraintExists.java @@ -77,7 +77,7 @@ public List check(@NonNull Statement statement, @NonNull SqlChec @Override public List getSupportsDialectTypes() { return Arrays.asList(DialectType.OB_ORACLE, DialectType.OB_MYSQL, DialectType.MYSQL, - DialectType.ODP_SHARDING_OB_MYSQL); + DialectType.ODP_SHARDING_OB_MYSQL, DialectType.ORACLE); } private List getForeignKeys(Stream stream) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/MySQLAffectedRowsExceedLimit.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/MySQLAffectedRowsExceedLimit.java index 4b8b869f6c..2b5b4254a9 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/MySQLAffectedRowsExceedLimit.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/MySQLAffectedRowsExceedLimit.java @@ -16,11 +16,9 @@ package com.oceanbase.odc.service.sqlcheck.rule; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Objects; -import java.util.concurrent.atomic.AtomicBoolean; import org.apache.commons.collections4.CollectionUtils; import org.springframework.jdbc.core.JdbcOperations; @@ -74,28 +72,20 @@ public long getStatementAffectedRows(Statement statement) { long affectedRows = 0; if (statement instanceof Update || statement instanceof Delete || statement instanceof Insert) { String explainSql = "EXPLAIN " + statement.getText(); - try { - if (this.jdbcOperations == null) { - log.warn("JdbcOperations is null, please check your connection"); - return -1; - } else { - switch (this.dialectType) { - case MYSQL: - affectedRows = (statement instanceof Insert) - ? getMySqlAffectedRowsByCount((Insert) statement) - : getMySqlAffectedRowsByExplain(explainSql, this.jdbcOperations); - break; - case OB_MYSQL: - affectedRows = getOBMySqlAffectedRows(explainSql, this.jdbcOperations); - break; - default: - log.warn("Unsupported dialect type: {}", this.dialectType); - break; - } - } - } catch (Exception e) { - log.warn("Error in calling getAffectedRows method", e); - affectedRows = -1; + if (this.jdbcOperations == null) { + throw new IllegalStateException("JdbcOperations is null, please check your connection"); + } + switch (this.dialectType) { + case MYSQL: + affectedRows = (statement instanceof Insert) + ? getMySqlAffectedRowsByCount((Insert) statement) + : getMySqlAffectedRowsByExplain(explainSql, this.jdbcOperations); + break; + case OB_MYSQL: + affectedRows = getOBAffectedRows(statement.getText(), this.jdbcOperations); + break; + default: + throw new UnsupportedOperationException("Unsupported dialect type: " + this.dialectType); } } return affectedRows; @@ -163,71 +153,4 @@ private long getMySqlAffectedRowsByExplain(String explainSql, JdbcOperations jdb } } - /** - * OBMySQL execute 'explain' statement - * - * @param explainSql target sql - * @param jdbc jdbc Object - * @return affected rows - */ - private long getOBMySqlAffectedRows(String explainSql, JdbcOperations jdbc) { - /** - * 
-         *
-         *     explain result (json):
-         *
-         *     ==================================================    --rowNum = 1
-         *     |ID|OPERATOR          |NAME|EST.ROWS|EST.TIME(us)|    --rowNum = 2
-         *     0 |DISTRIBUTED UPDATE|    |1       |37          |     --rowNum = 3
-         *     1 |└─TABLE GET       |user|1       |5           |     --rowNum = 4
-         *     ==================================================    --rowNum = 5
-         *     ...
-         *
-         *
- */ - try { - AtomicBoolean ifFindAffectedRow = new AtomicBoolean(false); - List queryResults = jdbc.query(explainSql, (rs, rowNum) -> rs.getString("Query Plan")); - List resultSet = new ArrayList<>(); - for (int rowNum = 0; rowNum < queryResults.size(); rowNum++) { - String resultRow = queryResults.get(rowNum); - if (!ifFindAffectedRow.get() && rowNum > 2) { - // Find the first non-null value in the column 'EST.ROWS' - long estRowsValue = getEstRowsValue(resultRow); - if (estRowsValue != 0) { - ifFindAffectedRow.set(true); - resultSet.add(estRowsValue); - } - } - resultSet.add(null); - } - - Long firstNonNullResult = resultSet.stream() - .filter(Objects::nonNull) - .findFirst() - .orElse(null); - - return firstNonNullResult != null ? firstNonNullResult : 0; - - } catch (Exception e) { - log.warn("OBMySQL mode: Error in execute " + explainSql + " failed. ", e); - return -1; - } - } - - /** - * parse explain result set - * - * @param singleRow row - * @return affected rows - */ - private long getEstRowsValue(String singleRow) { - String[] parts = singleRow.split("\\|"); - if (parts.length > 4) { - String value = parts[4].trim(); - return Long.parseLong(value); - } - return 0; - } - } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/MySQLNoTableCommentExists.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/MySQLNoTableCommentExists.java index 69ee5269fc..3a3ca6e943 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/MySQLNoTableCommentExists.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/MySQLNoTableCommentExists.java @@ -18,6 +18,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Objects; import com.oceanbase.odc.core.shared.constant.DialectType; import com.oceanbase.odc.service.sqlcheck.SqlCheckContext; @@ -51,6 +52,9 @@ public List check(@NonNull Statement statement, @NonNull SqlChec return Collections.emptyList(); } CreateTable createTable = (CreateTable) statement; + if (Objects.nonNull(createTable.getLikeTable()) || Objects.nonNull(createTable.getAs())) { + return Collections.emptyList(); + } TableOptions tableOptions = createTable.getTableOptions(); if (tableOptions == null || tableOptions.getComment() == null) { return Collections.singletonList(SqlCheckUtil.buildViolation( diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/MySQLRestrictTableAutoIncrement.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/MySQLRestrictTableAutoIncrement.java index 82b04eb86e..fe49dfaa34 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/MySQLRestrictTableAutoIncrement.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/MySQLRestrictTableAutoIncrement.java @@ -19,6 +19,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Objects; import com.oceanbase.odc.core.shared.constant.DialectType; import com.oceanbase.odc.service.sqlcheck.SqlCheckContext; @@ -58,6 +59,9 @@ public List check(@NonNull Statement statement, @NonNull SqlChec return Collections.emptyList(); } CreateTable createTable = (CreateTable) statement; + if (Objects.nonNull(createTable.getLikeTable()) || Objects.nonNull(createTable.getAs())) { + return Collections.emptyList(); + } TableOptions options = createTable.getTableOptions(); if (options == null || options.getAutoIncrement() == null) { return Collections.singletonList(SqlCheckUtil.buildViolation( diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/NoDefaultValueExists.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/NoDefaultValueExists.java index bb31fef89f..8b095da289 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/NoDefaultValueExists.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/NoDefaultValueExists.java @@ -72,7 +72,7 @@ public List check(@NonNull Statement statement, @NonNull SqlChec @Override public List getSupportsDialectTypes() { return Arrays.asList(DialectType.OB_MYSQL, DialectType.MYSQL, DialectType.OB_ORACLE, - DialectType.ODP_SHARDING_OB_MYSQL); + DialectType.ODP_SHARDING_OB_MYSQL, DialectType.ORACLE); } private boolean containsIgnoreCase(String name) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/NoIndexNameExists.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/NoIndexNameExists.java index e4bb8bb5ca..41ab53ce80 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/NoIndexNameExists.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/NoIndexNameExists.java @@ -91,7 +91,7 @@ public List check(@NonNull Statement statement, @NonNull SqlChec @Override public List getSupportsDialectTypes() { return Arrays.asList(DialectType.MYSQL, DialectType.OB_MYSQL, DialectType.OB_ORACLE, - DialectType.ODP_SHARDING_OB_MYSQL); + DialectType.ODP_SHARDING_OB_MYSQL, DialectType.ORACLE); } private List builds(Stream stream) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/NoPrimaryKeyExists.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/NoPrimaryKeyExists.java index e673fba776..db1f3127e9 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/NoPrimaryKeyExists.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/NoPrimaryKeyExists.java @@ -18,6 +18,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Objects; import org.apache.commons.collections4.CollectionUtils; @@ -54,6 +55,9 @@ public List check(@NonNull Statement statement, @NonNull SqlChec return Collections.emptyList(); } CreateTable createTable = (CreateTable) statement; + if (Objects.nonNull(createTable.getLikeTable()) || Objects.nonNull(createTable.getAs())) { + return Collections.emptyList(); + } boolean containsPk = createTable.getColumnDefinitions().stream() .filter(c -> c.getColumnAttributes() != null && CollectionUtils.isNotEmpty(c.getColumnAttributes().getConstraints())) @@ -70,7 +74,7 @@ public List check(@NonNull Statement statement, @NonNull SqlChec @Override public List getSupportsDialectTypes() { return Arrays.asList(DialectType.MYSQL, DialectType.OB_MYSQL, DialectType.OB_ORACLE, - DialectType.ODP_SHARDING_OB_MYSQL); + DialectType.ODP_SHARDING_OB_MYSQL, DialectType.ORACLE); } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/NoPrimaryKeyNameExists.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/NoPrimaryKeyNameExists.java index 5e1ab564b5..1b86ce3c93 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/NoPrimaryKeyNameExists.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/NoPrimaryKeyNameExists.java @@ -78,7 +78,7 @@ public List check(@NonNull Statement statement, @NonNull SqlChec @Override public List getSupportsDialectTypes() { return Arrays.asList(DialectType.MYSQL, DialectType.OB_MYSQL, DialectType.OB_ORACLE, - DialectType.ODP_SHARDING_OB_MYSQL); + DialectType.ODP_SHARDING_OB_MYSQL, DialectType.ORACLE); } private List builds(Stream stream) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/NoSpecificColumnExists.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/NoSpecificColumnExists.java index 5c6df2ebfd..ba0a510e08 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/NoSpecificColumnExists.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/NoSpecificColumnExists.java @@ -70,7 +70,7 @@ public SqlCheckRuleType getType() { @Override public List getSupportsDialectTypes() { return Arrays.asList(DialectType.OB_MYSQL, DialectType.MYSQL, DialectType.OB_ORACLE, - DialectType.ODP_SHARDING_OB_MYSQL); + DialectType.ODP_SHARDING_OB_MYSQL, DialectType.ORACLE); } private List build(String sql, Stream insertTables) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/NoValidWhereClause.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/NoValidWhereClause.java index a891fe9752..41762c3eac 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/NoValidWhereClause.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/NoValidWhereClause.java @@ -98,7 +98,7 @@ protected List getExpressionIsAlwaysTrueOrFalse(List whe @Override public List getSupportsDialectTypes() { return Arrays.asList(DialectType.OB_MYSQL, DialectType.MYSQL, DialectType.DORIS, DialectType.OB_ORACLE, - DialectType.ODP_SHARDING_OB_MYSQL); + DialectType.ODP_SHARDING_OB_MYSQL, DialectType.ORACLE); } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/NoWhereClauseExists.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/NoWhereClauseExists.java index 6dcc1b574e..635db195b7 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/NoWhereClauseExists.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/NoWhereClauseExists.java @@ -63,7 +63,7 @@ public List check(@NonNull Statement statement, @NonNull SqlChec @Override public List getSupportsDialectTypes() { return Arrays.asList(DialectType.OB_MYSQL, DialectType.MYSQL, DialectType.DORIS, DialectType.OB_ORACLE, - DialectType.ODP_SHARDING_OB_MYSQL); + DialectType.ODP_SHARDING_OB_MYSQL, DialectType.ORACLE); } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/NotNullColumnWithoutDefaultValue.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/NotNullColumnWithoutDefaultValue.java index 689f8473b0..828e48a140 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/NotNullColumnWithoutDefaultValue.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/NotNullColumnWithoutDefaultValue.java @@ -65,7 +65,7 @@ public List check(@NonNull Statement statement, @NonNull SqlChec @Override public List getSupportsDialectTypes() { return Arrays.asList(DialectType.OB_MYSQL, DialectType.MYSQL, DialectType.OB_ORACLE, - DialectType.ODP_SHARDING_OB_MYSQL); + DialectType.ODP_SHARDING_OB_MYSQL, DialectType.ORACLE); } private List builds(String sql, Stream stream) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleAffectedRowsExceedLimit.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleAffectedRowsExceedLimit.java new file mode 100644 index 0000000000..ed8fb11b75 --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleAffectedRowsExceedLimit.java @@ -0,0 +1,120 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.sqlcheck.rule; + +import java.util.Arrays; +import java.util.List; + +import org.springframework.jdbc.core.JdbcOperations; + +import com.oceanbase.odc.core.shared.constant.DialectType; +import com.oceanbase.tools.sqlparser.statement.Statement; +import com.oceanbase.tools.sqlparser.statement.delete.Delete; +import com.oceanbase.tools.sqlparser.statement.insert.Insert; +import com.oceanbase.tools.sqlparser.statement.update.Update; + +import lombok.NonNull; +import lombok.extern.slf4j.Slf4j; + +/** + * @description:{@link OracleAffectedRowsExceedLimit} + * + * @author: zijia.cj + * @date: 2024/10/24 13:06 + * @since: 4.3.3 + */ +@Slf4j +public class OracleAffectedRowsExceedLimit extends BaseAffectedRowsExceedLimit { + public static final String ODC_TEMP_EXPLAIN_STATEMENT_ID = "ODC_TEMP_EXPLAIN_STATEMENT_ID"; + + private final JdbcOperations jdbcOperations; + private final DialectType dialectType; + + public OracleAffectedRowsExceedLimit(@NonNull Long maxSqlAffectedRows, DialectType dialectType, + JdbcOperations jdbcOperations) { + super(maxSqlAffectedRows); + this.jdbcOperations = jdbcOperations; + this.dialectType = dialectType; + } + + /** + * Get supported database types + */ + @Override + public List getSupportsDialectTypes() { + return Arrays.asList(DialectType.OB_ORACLE, DialectType.ORACLE); + } + + /** + * Base method implemented by Oracle types + */ + @Override + public long getStatementAffectedRows(Statement statement) { + long affectedRows = 0; + if (statement instanceof Update || statement instanceof Delete || statement instanceof Insert) { + String originalSql = statement.getText(); + if (this.jdbcOperations == null) { + throw new IllegalStateException("JdbcOperations is null, please check your connection"); + } + switch (this.dialectType) { + case ORACLE: + affectedRows = getOracleAffectedRows(originalSql, this.jdbcOperations); + break; + case OB_ORACLE: + affectedRows = getOBAffectedRows(originalSql, this.jdbcOperations); + break; + default: + throw new UnsupportedOperationException("Unsupported dialect type: " + this.dialectType); + } + } + return affectedRows; + } + + private long getOracleAffectedRows(String originalSql, JdbcOperations jdbcOperations) { + /** + * 
+         *     Plan hash value: 775918519
+         *
+         * -------------------------------------------------------------------
+         * | Id  | Operation          | Name | Rows  | Cost (%CPU)| Time     |
+         * -------------------------------------------------------------------
+         * |   0 | DELETE STATEMENT   |      |    11 |     3   (0)| 00:00:01 |
+         * |   1 |  DELETE            | T1   |       |            |          |
+         * |   2 |   TABLE ACCESS FULL| T1   |    11 |     3   (0)| 00:00:01 |
+         * -------------------------------------------------------------------
+         *
+         * Query Block Name / Object Alias (identified by operation id):
+         * -------------------------------------------------------------
+         *
+         *    1 - DEL$1
+         *    2 - DEL$1 / T1@DEL$1
+         *
+         * Column Projection Information (identified by operation id):
+         * -----------------------------------------------------------
+         *
+         *    2 - "T1".ROWID[ROWID,10]
+         *
+ */ + String SetPlanSql = + "EXPLAIN PLAN SET STATEMENT_ID = '" + ODC_TEMP_EXPLAIN_STATEMENT_ID + "' FOR " + originalSql; + jdbcOperations.execute(SetPlanSql); + String getPlanSql = + "SELECT * FROM TABLE(DBMS_XPLAN.DISPLAY('PLAN_TABLE', '" + ODC_TEMP_EXPLAIN_STATEMENT_ID + "', 'ALL'))"; + List queryResults = jdbcOperations.query(getPlanSql, (rs, rowNum) -> rs.getString("PLAN_TABLE_OUTPUT")); + return getOBAndOracleAffectRowsFromResult(queryResults); + } + +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleColumnCalculation.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleColumnCalculation.java index 76398451ec..75c99cc5d3 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleColumnCalculation.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleColumnCalculation.java @@ -15,7 +15,7 @@ */ package com.oceanbase.odc.service.sqlcheck.rule; -import java.util.Collections; +import java.util.Arrays; import java.util.List; import com.oceanbase.odc.core.shared.constant.DialectType; @@ -40,7 +40,7 @@ protected boolean containsColumnReference(Expression expr) { @Override public List getSupportsDialectTypes() { - return Collections.singletonList(DialectType.OB_ORACLE); + return Arrays.asList(DialectType.OB_ORACLE, DialectType.ORACLE); } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleLeftFuzzyMatch.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleLeftFuzzyMatch.java index 1dc253e873..2a2fe98fd0 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleLeftFuzzyMatch.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleLeftFuzzyMatch.java @@ -16,7 +16,7 @@ package com.oceanbase.odc.service.sqlcheck.rule; import java.util.ArrayList; -import java.util.Collections; +import java.util.Arrays; import java.util.List; import org.apache.commons.lang3.StringUtils; @@ -51,7 +51,7 @@ protected boolean containsLeftFuzzy(Expression expr) { @Override public List getSupportsDialectTypes() { - return Collections.singletonList(DialectType.OB_ORACLE); + return Arrays.asList(DialectType.OB_ORACLE, DialectType.ORACLE); } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleMissingRequiredColumns.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleMissingRequiredColumns.java index fcc62babbf..5a087502b5 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleMissingRequiredColumns.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleMissingRequiredColumns.java @@ -15,7 +15,7 @@ */ package com.oceanbase.odc.service.sqlcheck.rule; -import java.util.Collections; +import java.util.Arrays; import java.util.List; import java.util.Set; @@ -44,7 +44,7 @@ protected String unquoteIdentifier(String identifier) { @Override public List getSupportsDialectTypes() { - return Collections.singletonList(DialectType.OB_ORACLE); + return Arrays.asList(DialectType.OB_ORACLE, DialectType.ORACLE); } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleNoColumnCommentExists.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleNoColumnCommentExists.java index 080567c187..a81f54f71a 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleNoColumnCommentExists.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleNoColumnCommentExists.java @@ -15,6 +15,7 @@ */ package com.oceanbase.odc.service.sqlcheck.rule; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; @@ -102,7 +103,7 @@ public List check(@NonNull Statement statement, @NonNull SqlChec @Override public List getSupportsDialectTypes() { - return Collections.singletonList(DialectType.OB_ORACLE); + return Arrays.asList(DialectType.OB_ORACLE, DialectType.ORACLE); } private String getKey(CreateTable c) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleNoNotNullAtInExpression.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleNoNotNullAtInExpression.java index a6954c40eb..97e1fbb7e7 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleNoNotNullAtInExpression.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleNoNotNullAtInExpression.java @@ -15,7 +15,7 @@ */ package com.oceanbase.odc.service.sqlcheck.rule; -import java.util.Collections; +import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; @@ -70,7 +70,7 @@ protected boolean containsNotNullForColumnReference(SelectBody selectBody) { @Override public List getSupportsDialectTypes() { - return Collections.singletonList(DialectType.OB_ORACLE); + return Arrays.asList(DialectType.OB_ORACLE, DialectType.ORACLE); } private String getColumnName(RelationReference reference) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleNoTableCommentExists.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleNoTableCommentExists.java index 2ddd188af2..b3d9ee6d6e 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleNoTableCommentExists.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleNoTableCommentExists.java @@ -15,8 +15,10 @@ */ package com.oceanbase.odc.service.sqlcheck.rule; +import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Objects; import java.util.Set; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -66,7 +68,11 @@ public List check(@NonNull Statement statement, @NonNull SqlChec if (statement instanceof SetComment) { setComments.add(new Pair<>((SetComment) statement, null)); } else if (statement instanceof CreateTable) { - createTables.add(new Pair<>((CreateTable) statement, null)); + CreateTable createTable = (CreateTable) statement; + if (Objects.nonNull(createTable.getLikeTable()) || Objects.nonNull(createTable.getAs())) { + return Collections.emptyList(); + } + createTables.add(new Pair<>(createTable, null)); } Set tblNames = setComments.stream().filter(s -> s.left.getTable() != null).map(s -> { RelationFactor t = s.left.getTable(); @@ -86,7 +92,7 @@ public List check(@NonNull Statement statement, @NonNull SqlChec @Override public List getSupportsDialectTypes() { - return Collections.singletonList(DialectType.OB_ORACLE); + return Arrays.asList(DialectType.OB_ORACLE, DialectType.ORACLE); } private String getKey(CreateTable c) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleOfflineDdlExists.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleOfflineDdlExists.java index 7392b5afd0..01d940d5f4 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleOfflineDdlExists.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleOfflineDdlExists.java @@ -17,6 +17,7 @@ import java.io.StringReader; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Objects; @@ -86,7 +87,7 @@ public List check(@NonNull Statement statement, @NonNull SqlChec @Override public List getSupportsDialectTypes() { - return Collections.singletonList(DialectType.OB_ORACLE); + return Arrays.asList(DialectType.OB_ORACLE, DialectType.ORACLE); } @Override diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleRestrictColumnNameCase.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleRestrictColumnNameCase.java index d48be225c8..44bb8e78f7 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleRestrictColumnNameCase.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleRestrictColumnNameCase.java @@ -15,6 +15,7 @@ */ package com.oceanbase.odc.service.sqlcheck.rule; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.stream.Collectors; @@ -76,7 +77,7 @@ public List check(@NonNull Statement statement, @NonNull SqlChec @Override public List getSupportsDialectTypes() { - return Collections.singletonList(DialectType.OB_ORACLE); + return Arrays.asList(DialectType.OB_ORACLE, DialectType.ORACLE); } private boolean verify(@NonNull String name) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleRestrictIndexDataTypes.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleRestrictIndexDataTypes.java index 81472b58c4..0649401dd1 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleRestrictIndexDataTypes.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleRestrictIndexDataTypes.java @@ -16,7 +16,7 @@ package com.oceanbase.odc.service.sqlcheck.rule; import java.io.StringReader; -import java.util.Collections; +import java.util.Arrays; import java.util.List; import java.util.Set; @@ -65,7 +65,7 @@ protected CreateTable getTableFromRemote(JdbcOperations jdbcOperations, String s @Override public List getSupportsDialectTypes() { - return Collections.singletonList(DialectType.OB_ORACLE); + return Arrays.asList(DialectType.OB_ORACLE, DialectType.ORACLE); } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleRestrictPKDataTypes.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleRestrictPKDataTypes.java index 912e4f672f..a70f83b8e7 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleRestrictPKDataTypes.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleRestrictPKDataTypes.java @@ -16,7 +16,7 @@ package com.oceanbase.odc.service.sqlcheck.rule; import java.io.StringReader; -import java.util.Collections; +import java.util.Arrays; import java.util.List; import java.util.Set; @@ -65,7 +65,7 @@ protected CreateTable getTableFromRemote(JdbcOperations jdbcOperations, String s @Override public List getSupportsDialectTypes() { - return Collections.singletonList(DialectType.OB_ORACLE); + return Arrays.asList(DialectType.OB_ORACLE, DialectType.ORACLE); } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleRestrictTableNameCase.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleRestrictTableNameCase.java index c590ccbd9b..a2d1a1fd86 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleRestrictTableNameCase.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleRestrictTableNameCase.java @@ -15,6 +15,7 @@ */ package com.oceanbase.odc.service.sqlcheck.rule; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.stream.Collectors; @@ -89,7 +90,7 @@ public List check(@NonNull Statement statement, @NonNull SqlChec @Override public List getSupportsDialectTypes() { - return Collections.singletonList(DialectType.OB_ORACLE); + return Arrays.asList(DialectType.OB_ORACLE, DialectType.ORACLE); } private boolean verify(@NonNull String name) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleTooManyAlterStatement.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleTooManyAlterStatement.java index 60b0701149..3477cab524 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleTooManyAlterStatement.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/OracleTooManyAlterStatement.java @@ -15,7 +15,7 @@ */ package com.oceanbase.odc.service.sqlcheck.rule; -import java.util.Collections; +import java.util.Arrays; import java.util.List; import com.oceanbase.odc.core.shared.constant.DialectType; @@ -43,7 +43,7 @@ protected String unquoteIdentifier(String identifier) { @Override public List getSupportsDialectTypes() { - return Collections.singletonList(DialectType.OB_ORACLE); + return Arrays.asList(DialectType.OB_ORACLE, DialectType.ORACLE); } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/PreferLocalOutOfLineIndex.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/PreferLocalOutOfLineIndex.java index 804c894b5d..bd6f88735b 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/PreferLocalOutOfLineIndex.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/PreferLocalOutOfLineIndex.java @@ -83,7 +83,7 @@ public SqlCheckRuleType getType() { @Override public List getSupportsDialectTypes() { return Arrays.asList(DialectType.OB_MYSQL, DialectType.MYSQL, DialectType.OB_ORACLE, - DialectType.ODP_SHARDING_OB_MYSQL); + DialectType.ODP_SHARDING_OB_MYSQL, DialectType.ORACLE); } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/ProhibitedDatatypeExists.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/ProhibitedDatatypeExists.java index 8c7a92a42b..50043e5da7 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/ProhibitedDatatypeExists.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/ProhibitedDatatypeExists.java @@ -72,7 +72,7 @@ public List check(@NonNull Statement statement, @NonNull SqlChec @Override public List getSupportsDialectTypes() { return Arrays.asList(DialectType.OB_MYSQL, DialectType.MYSQL, DialectType.OB_ORACLE, - DialectType.ODP_SHARDING_OB_MYSQL); + DialectType.ODP_SHARDING_OB_MYSQL, DialectType.ORACLE); } private List builds(String sql, Stream stream) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/RestrictColumnNotNull.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/RestrictColumnNotNull.java index b9a0a057e7..b79b1e2542 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/RestrictColumnNotNull.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/RestrictColumnNotNull.java @@ -73,7 +73,7 @@ public List check(@NonNull Statement statement, @NonNull SqlChec @Override public List getSupportsDialectTypes() { return Arrays.asList(DialectType.OB_ORACLE, DialectType.MYSQL, DialectType.OB_MYSQL, - DialectType.ODP_SHARDING_OB_MYSQL); + DialectType.ODP_SHARDING_OB_MYSQL, DialectType.ORACLE); } private boolean containsIgnoreCase(String name) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/RestrictDropObjectTypes.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/RestrictDropObjectTypes.java index 5e29bcae45..15e1801625 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/RestrictDropObjectTypes.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/RestrictDropObjectTypes.java @@ -93,7 +93,7 @@ action, getType(), new Object[] {getDeleteObjectType(action), allTypes})) @Override public List getSupportsDialectTypes() { return Arrays.asList(DialectType.OB_ORACLE, DialectType.MYSQL, - DialectType.OB_MYSQL, DialectType.ODP_SHARDING_OB_MYSQL); + DialectType.OB_MYSQL, DialectType.ODP_SHARDING_OB_MYSQL, DialectType.ORACLE); } private boolean notAllow(String objectType) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/RestrictIndexNaming.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/RestrictIndexNaming.java index 9506a17780..020029a77a 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/RestrictIndexNaming.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/RestrictIndexNaming.java @@ -124,7 +124,7 @@ public List check(@NonNull Statement statement, @NonNull SqlChec @Override public List getSupportsDialectTypes() { return Arrays.asList(DialectType.MYSQL, DialectType.OB_MYSQL, DialectType.OB_ORACLE, - DialectType.ODP_SHARDING_OB_MYSQL); + DialectType.ODP_SHARDING_OB_MYSQL, DialectType.ORACLE); } private boolean matches(String tableName, String indexName, List columns) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/RestrictPKNaming.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/RestrictPKNaming.java index 530a556e94..ea894c5f95 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/RestrictPKNaming.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/RestrictPKNaming.java @@ -109,7 +109,7 @@ public List check(@NonNull Statement statement, @NonNull SqlChec @Override public List getSupportsDialectTypes() { return Arrays.asList(DialectType.OB_ORACLE, DialectType.MYSQL, DialectType.OB_MYSQL, - DialectType.ODP_SHARDING_OB_MYSQL); + DialectType.ODP_SHARDING_OB_MYSQL, DialectType.ORACLE); } private boolean matches(String tableName, String indexName, List columns) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/RestrictUniqueIndexNaming.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/RestrictUniqueIndexNaming.java index b9e43f3a33..36239ecc92 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/RestrictUniqueIndexNaming.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/RestrictUniqueIndexNaming.java @@ -128,7 +128,7 @@ public List check(@NonNull Statement statement, @NonNull SqlChec @Override public List getSupportsDialectTypes() { return Arrays.asList(DialectType.MYSQL, DialectType.OB_MYSQL, DialectType.OB_ORACLE, - DialectType.ODP_SHARDING_OB_MYSQL); + DialectType.ODP_SHARDING_OB_MYSQL, DialectType.ORACLE); } private boolean matches(String tableName, String indexName, List columns) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/SelectStarExists.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/SelectStarExists.java index c6455080a5..da248144eb 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/SelectStarExists.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/SelectStarExists.java @@ -90,7 +90,7 @@ public List check(@NonNull Statement statement, @NonNull SqlChec @Override public List getSupportsDialectTypes() { return Arrays.asList(DialectType.OB_MYSQL, DialectType.MYSQL, DialectType.OB_ORACLE, - DialectType.ODP_SHARDING_OB_MYSQL); + DialectType.ODP_SHARDING_OB_MYSQL, DialectType.ORACLE); } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/SqlCheckRules.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/SqlCheckRules.java index 1bb755105b..03ab0de64a 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/SqlCheckRules.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/SqlCheckRules.java @@ -39,6 +39,8 @@ import com.oceanbase.odc.service.sqlcheck.factory.ColumnCharsetExistsFactory; import com.oceanbase.odc.service.sqlcheck.factory.ColumnCollationExistsFactory; import com.oceanbase.odc.service.sqlcheck.factory.ColumnNameInBlackListFactory; +import com.oceanbase.odc.service.sqlcheck.factory.CreateTableAsExistsFactory; +import com.oceanbase.odc.service.sqlcheck.factory.CreateTableLikeExistsFactory; import com.oceanbase.odc.service.sqlcheck.factory.ForeignConstraintExistsFactory; import com.oceanbase.odc.service.sqlcheck.factory.LeftFuzzyMatchFactory; import com.oceanbase.odc.service.sqlcheck.factory.MissingRequiredColumnsFactory; @@ -155,6 +157,8 @@ public static List getAllFactories(DialectType dialectType, rules.add(new TruncateTableExistsFactory()); rules.add(new SqlAffectedRowsFactory(jdbc)); rules.add(new Unable2JudgeAffectedRowsFactory(jdbc)); + rules.add(new CreateTableLikeExistsFactory()); + rules.add(new CreateTableAsExistsFactory()); return rules; } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/SyntaxErrorExists.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/SyntaxErrorExists.java index b47f1ef9b7..b9820451dd 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/SyntaxErrorExists.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/SyntaxErrorExists.java @@ -58,7 +58,7 @@ public List check(@NonNull Statement statement, @NonNull SqlChec @Override public List getSupportsDialectTypes() { return Arrays.asList(DialectType.OB_MYSQL, DialectType.MYSQL, DialectType.OB_ORACLE, - DialectType.ODP_SHARDING_OB_MYSQL); + DialectType.ODP_SHARDING_OB_MYSQL, DialectType.ORACLE); } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/TableNameInBlackList.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/TableNameInBlackList.java index 91f020a044..a043137165 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/TableNameInBlackList.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/TableNameInBlackList.java @@ -92,7 +92,7 @@ public List check(@NonNull Statement statement, @NonNull SqlChec @Override public List getSupportsDialectTypes() { return Arrays.asList(DialectType.OB_MYSQL, DialectType.MYSQL, DialectType.OB_ORACLE, - DialectType.ODP_SHARDING_OB_MYSQL); + DialectType.ODP_SHARDING_OB_MYSQL, DialectType.ORACLE); } private String unquoteIdentifier(String name) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/TooLongCharLength.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/TooLongCharLength.java index 7607a20f3e..e068949c69 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/TooLongCharLength.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/TooLongCharLength.java @@ -70,7 +70,7 @@ public List check(@NonNull Statement statement, @NonNull SqlChec @Override public List getSupportsDialectTypes() { return Arrays.asList(DialectType.OB_ORACLE, DialectType.OB_MYSQL, DialectType.MYSQL, - DialectType.ODP_SHARDING_OB_MYSQL); + DialectType.ODP_SHARDING_OB_MYSQL, DialectType.ORACLE); } private List builds(String sql, Stream stream) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/TooManyColumnDefinition.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/TooManyColumnDefinition.java index eee3efc366..f910ceded7 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/TooManyColumnDefinition.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/TooManyColumnDefinition.java @@ -81,7 +81,7 @@ public SqlCheckRuleType getType() { @Override public List getSupportsDialectTypes() { return Arrays.asList(DialectType.OB_MYSQL, DialectType.MYSQL, DialectType.OB_ORACLE, - DialectType.ODP_SHARDING_OB_MYSQL); + DialectType.ODP_SHARDING_OB_MYSQL, DialectType.ORACLE); } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/TooManyColumnRefInIndex.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/TooManyColumnRefInIndex.java index 4e9924cacd..3551000d31 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/TooManyColumnRefInIndex.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/TooManyColumnRefInIndex.java @@ -115,7 +115,7 @@ public List check(@NonNull Statement statement, @NonNull SqlChec @Override public List getSupportsDialectTypes() { return Arrays.asList(DialectType.OB_MYSQL, DialectType.MYSQL, DialectType.OB_ORACLE, - DialectType.ODP_SHARDING_OB_MYSQL); + DialectType.ODP_SHARDING_OB_MYSQL, DialectType.ORACLE); } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/TooManyColumnRefInPrimaryKey.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/TooManyColumnRefInPrimaryKey.java index 88942d0baa..6f392f139a 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/TooManyColumnRefInPrimaryKey.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/TooManyColumnRefInPrimaryKey.java @@ -93,7 +93,7 @@ public List check(@NonNull Statement statement, @NonNull SqlChec @Override public List getSupportsDialectTypes() { return Arrays.asList(DialectType.MYSQL, DialectType.OB_MYSQL, DialectType.OB_ORACLE, - DialectType.ODP_SHARDING_OB_MYSQL); + DialectType.ODP_SHARDING_OB_MYSQL, DialectType.ORACLE); } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/TooManyInExpression.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/TooManyInExpression.java index 824b9b35da..cd9c676ff3 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/TooManyInExpression.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/TooManyInExpression.java @@ -67,7 +67,7 @@ public SqlCheckRuleType getType() { @Override public List getSupportsDialectTypes() { return Arrays.asList(DialectType.OB_MYSQL, DialectType.MYSQL, DialectType.OB_ORACLE, - DialectType.ODP_SHARDING_OB_MYSQL); + DialectType.ODP_SHARDING_OB_MYSQL, DialectType.ORACLE); } protected List getTooManyInExprs(List wheres) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/TooManyOutOfLineIndex.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/TooManyOutOfLineIndex.java index 40bcf66ecb..0a24e41ade 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/TooManyOutOfLineIndex.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/TooManyOutOfLineIndex.java @@ -87,7 +87,7 @@ public SqlCheckRuleType getType() { @Override public List getSupportsDialectTypes() { return Arrays.asList(DialectType.OB_MYSQL, DialectType.MYSQL, DialectType.OB_ORACLE, - DialectType.ODP_SHARDING_OB_MYSQL); + DialectType.ODP_SHARDING_OB_MYSQL, DialectType.ORACLE); } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/TooManyTableJoin.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/TooManyTableJoin.java index d8f8925913..b612e82ea9 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/TooManyTableJoin.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/TooManyTableJoin.java @@ -88,7 +88,7 @@ public SqlCheckRuleType getType() { @Override public List getSupportsDialectTypes() { return Arrays.asList(DialectType.OB_MYSQL, DialectType.MYSQL, DialectType.OB_ORACLE, - DialectType.ODP_SHARDING_OB_MYSQL); + DialectType.ODP_SHARDING_OB_MYSQL, DialectType.ORACLE); } protected List getTooManyJoinRefs(List joins) { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/TruncateTableExists.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/TruncateTableExists.java index 130562ff78..2f7f656828 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/TruncateTableExists.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/sqlcheck/rule/TruncateTableExists.java @@ -41,7 +41,7 @@ public class TruncateTableExists implements SqlCheckRule { @Override public SqlCheckRuleType getType() { - return SqlCheckRuleType.TRUNCATE_TBLE_EXISTS; + return SqlCheckRuleType.TRUNCATE_TABLE_EXISTS; } @Override diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/state/StatefulRouteConfiguration.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/state/StatefulRouteConfiguration.java index 8da547e157..e5d2f0468a 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/state/StatefulRouteConfiguration.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/state/StatefulRouteConfiguration.java @@ -24,7 +24,7 @@ import org.springframework.context.annotation.Configuration; import com.oceanbase.odc.common.util.SystemUtils; -import com.oceanbase.odc.service.task.executor.server.TraceDecoratorThreadFactory; +import com.oceanbase.odc.service.task.executor.TraceDecoratorThreadFactory; @Configuration public class StatefulRouteConfiguration { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/structurecompare/StructureComparisonService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/structurecompare/StructureComparisonService.java index 2d9376e480..436fc68d46 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/structurecompare/StructureComparisonService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/structurecompare/StructureComparisonService.java @@ -106,7 +106,7 @@ public DBStructureComparisonResp getDBStructureComparisonResult(@NonNull Long id try { StorageObject storageObject = objectStorageFacade.loadObject("structure-comparison".concat(File.separator) - .concat(authenticationFacade.currentUserIdStr()), taskEntity.getStorageObjectId()); + .concat(String.valueOf(taskEntity.getCreatorId())), taskEntity.getStorageObjectId()); Validate.notNull(storageObject, "StorageObject can not be null"); Validate.notNull(storageObject.getMetadata(), "ObjectMetadata can not be null"); if (storageObject.getMetadata().getTotalLength() > MAX_TOTAL_SCRIPT_SIZE_BYTES) { @@ -147,6 +147,7 @@ public List batchCreateTaskResults(List flowInstance, false); + flowInstanceService.mapFlowInstanceWithReadPermission(taskEntity.getFlowInstanceId(), + flowInstance -> flowInstance); } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/task/ExceptionListener.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/ExceptionListener.java similarity index 93% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/task/ExceptionListener.java rename to server/odc-service/src/main/java/com/oceanbase/odc/service/task/ExceptionListener.java index 7aac18956a..6d2bed6f90 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/task/ExceptionListener.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/ExceptionListener.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.task.executor.task; +package com.oceanbase.odc.service.task; /** * listener for exception diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/task/Task.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/Task.java similarity index 70% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/task/Task.java rename to server/odc-service/src/main/java/com/oceanbase/odc/service/task/Task.java index ff1e8cdf4e..80665b34d0 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/task/Task.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/Task.java @@ -13,14 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -package com.oceanbase.odc.service.task.executor.task; +package com.oceanbase.odc.service.task; import java.util.Map; import com.oceanbase.odc.service.task.caller.JobContext; -import com.oceanbase.odc.service.task.enums.JobStatus; -import com.oceanbase.odc.service.task.executor.server.TaskExecutor; /** * Task interface. Each task should implement this interface @@ -31,14 +28,28 @@ public interface Task { /** - * Start current task. This method will be called by {@link TaskExecutor} for fire a task + * init task in runtime + * + * @param taskContext + * @throws Exception + */ + void init(TaskContext taskContext) throws Exception; + + /** + * Start current task. This method will be called by TaskExecutor for fire a task This method call + * must be blocked until job is done or failed + */ + boolean start() throws Exception; + + /** + * Stop current task. This method will be called TaskExecutor for stop a task */ - void start(TaskContext taskContext); + void stop() throws Exception; /** - * Stop current task. This method will be called by {@link TaskExecutor} for stop a task + * close and clean resource of task */ - boolean stop(); + void close() throws Exception; /** * Modify current task parameters @@ -59,13 +70,6 @@ public interface Task { */ JobContext getJobContext(); - /** - * Get task status - * - * @return {@link JobStatus} - */ - JobStatus getStatus(); - /** * Get task result */ diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/task/TaskContext.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/TaskContext.java similarity index 78% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/task/TaskContext.java rename to server/odc-service/src/main/java/com/oceanbase/odc/service/task/TaskContext.java index e5971bb421..981ea06f30 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/task/TaskContext.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/TaskContext.java @@ -13,8 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.task.executor.task; +package com.oceanbase.odc.service.task; +import com.oceanbase.odc.service.objectstorage.cloud.CloudObjectStorageService; import com.oceanbase.odc.service.task.caller.JobContext; /** @@ -37,4 +38,11 @@ public interface TaskContext { * @return */ JobContext getJobContext(); + + /** + * get shared storage for task upload or download file + * + * @return + */ + CloudObjectStorageService getSharedStorage(); } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/TaskEventListener.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/TaskEventListener.java new file mode 100644 index 0000000000..e68a2d8c97 --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/TaskEventListener.java @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.task; + +/** + * listen task event + * + * @author longpeng.zlp + * @date 2024/10/24 11:51 + */ +public interface TaskEventListener { + /** + * call when task start called + * + * @param task + */ + void onTaskStart(Task task); + + + /** + * call when task stop called + * + * @param task + */ + void onTaskStop(Task task); + + + /** + * call when task modify called + * + * @param task + */ + void onTaskModify(Task task); + + /** + * call when task final closed + * + * @param task + */ + void onTaskFinalize(Task task); + +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/TaskBase.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/TaskBase.java new file mode 100644 index 0000000000..c2aafeca1f --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/TaskBase.java @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.task.base; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; + +import com.oceanbase.odc.service.task.Task; +import com.oceanbase.odc.service.task.TaskContext; +import com.oceanbase.odc.service.task.caller.DefaultJobContext; +import com.oceanbase.odc.service.task.caller.JobContext; + +import lombok.extern.slf4j.Slf4j; + +/** + * base task for implement + * + * @author longpeng.zlp + * @date 2024/11/8 11:10 + */ +@Slf4j +public abstract class TaskBase implements Task { + protected TaskContext context; + protected DefaultJobContext jobContext; + + public void init(TaskContext taskContext) throws Exception { + this.context = taskContext; + jobContext = copyJobContext(taskContext); + log.info("Start task, id={}.", jobContext.getJobIdentity().getId()); + log.info("Init task parameters success, id={}.", jobContext.getJobIdentity().getId()); + doInit(jobContext); + } + + protected abstract void doInit(JobContext context) throws Exception; + + public JobContext getJobContext() { + return jobContext; + } + + // deep copy job context + protected DefaultJobContext copyJobContext(TaskContext taskContext) { + DefaultJobContext ret = new DefaultJobContext(); + JobContext src = taskContext.getJobContext(); + ret.setJobIdentity(src.getJobIdentity()); + if (null != src.getJobProperties()) { + ret.setJobProperties(new HashMap<>(src.getJobProperties())); + } + if (null != src.getJobParameters()) { + ret.setJobParameters(Collections.unmodifiableMap(new HashMap<>(src.getJobParameters()))); + } + ret.setJobClass(src.getJobClass()); + if (null != src.getHostUrls()) { + ret.setHostUrls(new ArrayList<>(src.getHostUrls())); + } + return ret; + } + + protected Long getJobId() { + return getJobContext().getJobIdentity().getId(); + } + + @Override + public boolean modify(Map jobParameters) { + if (Objects.isNull(jobParameters) || jobParameters.isEmpty()) { + log.warn("Job parameter cannot be null, id={}", getJobId()); + return false; + } + jobContext.setJobParameters(Collections.unmodifiableMap(jobParameters)); + return true; + } + +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/dataarchive/DataArchiveTask.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/dataarchive/DataArchiveTask.java new file mode 100644 index 0000000000..1b934ab46f --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/dataarchive/DataArchiveTask.java @@ -0,0 +1,267 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.task.base.dataarchive; + +import java.util.ArrayList; +import java.util.Date; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import com.oceanbase.odc.common.json.JsonUtils; +import com.oceanbase.odc.core.shared.constant.TaskStatus; +import com.oceanbase.odc.service.dlm.DLMJobFactory; +import com.oceanbase.odc.service.dlm.DLMJobStore; +import com.oceanbase.odc.service.dlm.DLMTableStructureSynchronizer; +import com.oceanbase.odc.service.dlm.model.DlmTableUnit; +import com.oceanbase.odc.service.dlm.model.DlmTableUnitParameters; +import com.oceanbase.odc.service.dlm.model.RateLimitConfiguration; +import com.oceanbase.odc.service.dlm.utils.DlmJobIdUtil; +import com.oceanbase.odc.service.schedule.job.DLMJobReq; +import com.oceanbase.odc.service.schedule.model.DlmTableUnitStatistic; +import com.oceanbase.odc.service.task.base.TaskBase; +import com.oceanbase.odc.service.task.caller.JobContext; +import com.oceanbase.odc.service.task.constants.JobParametersKeyConstants; +import com.oceanbase.odc.service.task.util.JobUtils; +import com.oceanbase.tools.migrator.common.enums.JobType; +import com.oceanbase.tools.migrator.core.meta.JobMeta; +import com.oceanbase.tools.migrator.job.Job; +import com.oceanbase.tools.migrator.limiter.LimiterConfig; +import com.oceanbase.tools.migrator.task.CheckMode; + +import lombok.extern.slf4j.Slf4j; + +/** + * @Author：tinker + * @Date: 2024/1/24 11:09 + * @Descripition: + */ + +@Slf4j +public class DataArchiveTask extends TaskBase> { + + private DLMJobFactory jobFactory; + private DLMJobStore jobStore; + private double progress = 0.0; + private Job job; + private List toDoList; + private int currentIndex = 0; + private boolean isToStop = false; + + public DataArchiveTask() {} + + @Override + protected void doInit(JobContext context) { + jobStore = new DLMJobStore(JobUtils.getMetaDBConnectionConfig()); + jobFactory = new DLMJobFactory(jobStore); + try { + DLMJobReq parameters = + JsonUtils.fromJson( + jobContext.getJobParameters().get(JobParametersKeyConstants.META_TASK_PARAMETER_JSON), + DLMJobReq.class); + initTableUnit(parameters); + } catch (Exception e) { + log.warn("Initialization of the DLM job was failed,jobIdentity={}", context.getJobIdentity(), e); + } + log.info("Initialization of the DLM job was successful. Number of tables to be processed = {},jobIdentity={}", + toDoList.size(), context.getJobIdentity()); + } + + @Override + public boolean start() throws Exception { + while (!isToStop && currentIndex < toDoList.size()) { + DlmTableUnit dlmTableUnit = toDoList.get(currentIndex); + if (dlmTableUnit.getStatus() == TaskStatus.DONE) { + log.info("The table had been completed,tableName={}", dlmTableUnit.getTableName()); + currentIndex++; + continue; + } + syncTableStructure(dlmTableUnit); + try { + jobStore.setDlmTableUnit(dlmTableUnit); + job = jobFactory.createJob(dlmTableUnit); + } catch (Throwable e) { + log.error("Failed to create job,dlmTableUnitId={}", dlmTableUnit.getDlmTableUnitId(), e); + dlmTableUnit.setStatus(isToStop ? TaskStatus.CANCELED : TaskStatus.FAILED); + currentIndex++; + continue; + } + log.info("Init {} job succeed,dlmTableUnitId={}", dlmTableUnit.getType(), dlmTableUnit.getDlmTableUnitId()); + try { + dlmTableUnit.setStatus(TaskStatus.RUNNING); + dlmTableUnit.setStartTime(new Date()); + job.run(); + log.info("{} job finished,dlmTableUnitId={}", dlmTableUnit.getType(), dlmTableUnit.getDlmTableUnitId()); + dlmTableUnit.setStatus(TaskStatus.DONE); + } catch (Throwable e) { + dlmTableUnit.setStatus(isToStop ? TaskStatus.CANCELED : TaskStatus.FAILED); + context.getExceptionListener().onException(e); + } + dlmTableUnit.setEndTime(new Date()); + currentIndex++; + } + log.info("All tables have been processed,jobIdentity={}.\n{}", jobContext.getJobIdentity(), buildReport()); + return true; + } + + private void syncTableStructure(DlmTableUnit tableUnit) { + if (tableUnit.getType() != JobType.MIGRATE) { + return; + } + try { + DLMTableStructureSynchronizer.sync(tableUnit.getSourceDatasourceInfo(), tableUnit.getTargetDatasourceInfo(), + tableUnit.getTableName(), tableUnit.getTargetTableName(), + tableUnit.getSyncTableStructure()); + } catch (Exception e) { + log.warn("Failed to sync target table structure,tableName={}", + tableUnit.getTableName(), e); + } + } + + private void initTableUnit(DLMJobReq req) { + List dlmTableUnits = new LinkedList<>(); + req.getTables().forEach(table -> { + DlmTableUnit dlmTableUnit = new DlmTableUnit(); + dlmTableUnit.setScheduleTaskId(req.getScheduleTaskId()); + DlmTableUnitParameters jobParameter = new DlmTableUnitParameters(); + jobParameter.setMigrateRule(table.getConditionExpression()); + jobParameter.setCheckMode(CheckMode.MULTIPLE_GET); + jobParameter.setReaderBatchSize(req.getRateLimit().getBatchSize()); + jobParameter.setWriterBatchSize(req.getRateLimit().getBatchSize()); + jobParameter.setMigrationInsertAction(req.getMigrationInsertAction()); + jobParameter.setMigratePartitions(table.getPartitions()); + jobParameter.setSyncDBObjectType(req.getSyncTableStructure()); + jobParameter.setShardingStrategy(req.getShardingStrategy()); + jobParameter.setPartName2MinKey(table.getPartName2MinKey()); + jobParameter.setPartName2MaxKey(table.getPartName2MaxKey()); + dlmTableUnit.setParameters(jobParameter); + dlmTableUnit.setDlmTableUnitId(DlmJobIdUtil.generateHistoryJobId(req.getJobName(), req.getJobType().name(), + req.getScheduleTaskId(), dlmTableUnits.size())); + dlmTableUnit.setTableName(table.getTableName()); + dlmTableUnit.setTargetTableName(table.getTargetTableName()); + dlmTableUnit.setSourceDatasourceInfo(req.getSourceDs()); + dlmTableUnit.setTargetDatasourceInfo(req.getTargetDs()); + dlmTableUnit.setFireTime(req.getFireTime()); + dlmTableUnit.setStatus(TaskStatus.PREPARING); + dlmTableUnit.setType(req.getJobType()); + dlmTableUnit.setStatistic(new DlmTableUnitStatistic()); + dlmTableUnit.setSyncTableStructure(req.getSyncTableStructure()); + LimiterConfig limiterConfig = new LimiterConfig(); + limiterConfig.setDataSizeLimit(req.getRateLimit().getDataSizeLimit()); + limiterConfig.setRowLimit(req.getRateLimit().getRowLimit()); + dlmTableUnit.setSourceLimitConfig(limiterConfig); + dlmTableUnit.setTargetLimitConfig(limiterConfig); + dlmTableUnits.add(dlmTableUnit); + }); + toDoList = new LinkedList<>(dlmTableUnits); + } + + private String buildReport() { + StringBuilder sb = new StringBuilder(); + sb.append("Job report:\n"); + sb.append("Total tables: ").append(toDoList.size()).append("\n"); + sb.append("Success tables: ") + .append(toDoList.stream().filter(t -> t.getStatus() == TaskStatus.DONE).map(DlmTableUnit::getTableName) + .collect( + Collectors.joining(","))) + .append("\n"); + sb.append("Failed tables: ") + .append(toDoList.stream().filter(t -> t.getStatus() == TaskStatus.FAILED) + .map(DlmTableUnit::getTableName).collect( + Collectors.joining(","))) + .append("\n"); + sb.append("Canceled tables: ") + .append(toDoList.stream().filter(t -> t.getStatus() == TaskStatus.CANCELED) + .map(DlmTableUnit::getTableName).collect( + Collectors.joining(","))) + .append("\n"); + return sb.toString(); + } + + @Override + public void stop() throws Exception { + isToStop = true; + if (job != null) { + try { + job.stop(); + toDoList.forEach(t -> { + if (!t.getStatus().isTerminated()) { + t.setStatus(TaskStatus.CANCELED); + } + }); + } catch (Exception e) { + log.warn("Update dlm table unit status failed,DlmTableUnitId={}", job.getJobMeta().getJobId()); + } + } + } + + @Override + public void close() throws Exception { + jobStore.destroy(); + } + + @Override + public boolean modify(Map jobParameters) { + if (!super.modify(jobParameters)) { + return false; + } + updateLimiter(jobParameters); + return true; + } + + public void updateLimiter(Map jobParameters) { + if (job == null || job.getJobMeta() == null) { + return; + } + JobMeta jobMeta = job.getJobMeta(); + try { + RateLimitConfiguration params; + if (jobParameters.containsKey(JobParametersKeyConstants.DLM_RATE_LIMIT_CONFIG)) { + params = JsonUtils.fromJson( + jobParameters.get(JobParametersKeyConstants.DLM_RATE_LIMIT_CONFIG), + RateLimitConfiguration.class); + } else { + DLMJobReq dlmJobReq = JsonUtils.fromJson( + jobParameters.get(JobParametersKeyConstants.META_TASK_PARAMETER_JSON), + DLMJobReq.class); + params = dlmJobReq.getRateLimit(); + } + if (params.getDataSizeLimit() != null) { + jobMeta.getSourceLimiterConfig().setDataSizeLimit(params.getDataSizeLimit()); + jobMeta.getTargetLimiterConfig().setDataSizeLimit(params.getDataSizeLimit()); + log.info("Update rate limit success,dataSizeLimit={}", params.getDataSizeLimit()); + } + if (params.getRowLimit() != null) { + jobMeta.getSourceLimiterConfig().setRowLimit(params.getRowLimit()); + jobMeta.getTargetLimiterConfig().setRowLimit(params.getRowLimit()); + log.info("Update rate limit success,rowLimit={}", params.getRowLimit()); + } + } catch (Exception e) { + log.warn("Update rate limit failed,errorMsg={}", e.getMessage()); + } + } + + @Override + public double getProgress() { + return progress; + } + + @Override + public List getTaskResult() { + return new ArrayList<>(toDoList); + } +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/runtime/DatabaseChangeTask.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/databasechange/DatabaseChangeTask.java similarity index 97% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/task/runtime/DatabaseChangeTask.java rename to server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/databasechange/DatabaseChangeTask.java index 2c7a8db251..25268324d1 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/runtime/DatabaseChangeTask.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/databasechange/DatabaseChangeTask.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.task.runtime; +package com.oceanbase.odc.service.task.base.databasechange; import java.io.ByteArrayInputStream; import java.io.File; @@ -91,12 +91,11 @@ import com.oceanbase.odc.service.session.factory.DefaultConnectSessionFactory; import com.oceanbase.odc.service.session.initializer.ConsoleTimeoutInitializer; import com.oceanbase.odc.service.session.model.SqlExecuteResult; +import com.oceanbase.odc.service.task.base.TaskBase; import com.oceanbase.odc.service.task.caller.JobContext; import com.oceanbase.odc.service.task.constants.JobParametersKeyConstants; import com.oceanbase.odc.service.task.constants.JobServerUrls; import com.oceanbase.odc.service.task.exception.JobException; -import com.oceanbase.odc.service.task.executor.task.BaseTask; -import com.oceanbase.odc.service.task.executor.task.TaskContext; import com.oceanbase.odc.service.task.util.HttpClientUtils; import com.oceanbase.odc.service.task.util.JobUtils; import com.oceanbase.tools.dbbrowser.parser.ParserUtil; @@ -113,7 +112,7 @@ */ @Slf4j -public class DatabaseChangeTask extends BaseTask { +public class DatabaseChangeTask extends TaskBase { private ConnectionSession connectionSession; private DatabaseChangeTaskParameters parameters; @@ -140,12 +139,15 @@ public class DatabaseChangeTask extends BaseTask { private volatile boolean canceled = false; private long taskId; + public DatabaseChangeTask() {} + @Override - protected void doInit(JobContext context) { + protected void doInit(JobContext jobContext) { taskId = getJobContext().getJobIdentity().getId(); log.info("Initiating database change task, taskId={}", taskId); - this.parameters = JobUtils.fromJson(getJobParameters().get(JobParametersKeyConstants.TASK_PARAMETER_JSON_KEY), - DatabaseChangeTaskParameters.class); + this.parameters = + JobUtils.fromJson(jobContext.getJobParameters().get(JobParametersKeyConstants.TASK_PARAMETER_JSON_KEY), + DatabaseChangeTaskParameters.class); this.databaseChangeParameters = JsonUtils.fromJson(this.parameters.getParameterJson(), DatabaseChangeParameters.class); log.info("Load database change task parameters successfully, taskId={}", taskId); @@ -162,7 +164,7 @@ protected void doInit(JobContext context) { try { SizeAwareInputStream sizeAwareInputStream = ObjectStorageUtils.loadObjectsForTask(this.parameters.getSqlFileObjectMetadatas(), - getCloudObjectStorageService(), JobUtils.getExecutorDataPath(), -1); + this.context.getSharedStorage(), JobUtils.getExecutorDataPath(), -1); sqlTotalBytes += sizeAwareInputStream.getTotalBytes(); sqlInputStream = sizeAwareInputStream.getInputStream(); } catch (IOException exception) { @@ -186,7 +188,7 @@ protected void doInit(JobContext context) { } @Override - protected boolean doStart(JobContext context, TaskContext taskContext) throws JobException { + public boolean start() throws JobException { try { int index = 0; while (sqlIterator.hasNext()) { @@ -244,7 +246,7 @@ protected boolean doStart(JobContext context, TaskContext taskContext) throws Jo aborted = true; break; } - taskContext.getExceptionListener().onException(e); + context.getExceptionListener().onException(e); } } writeZipFile(); @@ -260,14 +262,14 @@ protected boolean doStart(JobContext context, TaskContext taskContext) throws Jo } @Override - protected void doStop() { + public void stop() { tryExpireConnectionSession(); tryCloseInputStream(); canceled = true; } @Override - protected void doClose() throws Exception { + public void close() throws Exception { tryExpireConnectionSession(); tryCloseInputStream(); } @@ -429,7 +431,7 @@ private void writeZipFile() { OdcFileUtil.zip(String.format(zipFileRootPath), String.format("%s.zip", zipFileRootPath)); log.info("Database change task result set was saved as local zip file, file name={}", zipFileId); // Public cloud scenario, need to upload files to OSS - CloudObjectStorageService cloudObjectStorageService = getCloudObjectStorageService(); + CloudObjectStorageService cloudObjectStorageService = context.getSharedStorage(); if (Objects.nonNull(cloudObjectStorageService) && cloudObjectStorageService.supported()) { File tempZipFile = new File(String.format("%s.zip", zipFileRootPath)); try { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/runtime/DatabaseChangeTaskParameters.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/databasechange/DatabaseChangeTaskParameters.java similarity index 96% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/task/runtime/DatabaseChangeTaskParameters.java rename to server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/databasechange/DatabaseChangeTaskParameters.java index 5fcd10a439..45e1c0671c 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/runtime/DatabaseChangeTaskParameters.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/databasechange/DatabaseChangeTaskParameters.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.task.runtime; +package com.oceanbase.odc.service.task.base.databasechange; import java.util.List; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/runtime/QuerySensitiveColumnReq.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/databasechange/QuerySensitiveColumnReq.java similarity index 95% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/task/runtime/QuerySensitiveColumnReq.java rename to server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/databasechange/QuerySensitiveColumnReq.java index fdd0075481..f049e3b81e 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/runtime/QuerySensitiveColumnReq.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/databasechange/QuerySensitiveColumnReq.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.task.runtime; +package com.oceanbase.odc.service.task.base.databasechange; import java.util.List; import java.util.Set; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/runtime/QuerySensitiveColumnResp.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/databasechange/QuerySensitiveColumnResp.java similarity index 94% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/task/runtime/QuerySensitiveColumnResp.java rename to server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/databasechange/QuerySensitiveColumnResp.java index 3637f868b4..c443bdf708 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/runtime/QuerySensitiveColumnResp.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/databasechange/QuerySensitiveColumnResp.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.task.runtime; +package com.oceanbase.odc.service.task.base.databasechange; import java.util.List; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/task/LogicalDatabaseChangeTask.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/logicdatabasechange/LogicalDatabaseChangeTask.java similarity index 91% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/task/LogicalDatabaseChangeTask.java rename to server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/logicdatabasechange/LogicalDatabaseChangeTask.java index 71642aba59..e99cdac2e9 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/task/LogicalDatabaseChangeTask.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/logicdatabasechange/LogicalDatabaseChangeTask.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.task.executor.task; +package com.oceanbase.odc.service.task.base.logicdatabasechange; import java.util.ArrayList; import java.util.Collections; @@ -21,6 +21,7 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; @@ -52,6 +53,7 @@ import com.oceanbase.odc.service.connection.logicaldatabase.model.DetailLogicalTableResp; import com.oceanbase.odc.service.schedule.model.PublishLogicalDatabaseChangeReq; import com.oceanbase.odc.service.session.model.SqlExecuteResult; +import com.oceanbase.odc.service.task.base.TaskBase; import com.oceanbase.odc.service.task.caller.JobContext; import com.oceanbase.odc.service.task.constants.JobParametersKeyConstants; import com.oceanbase.tools.dbbrowser.parser.SqlParser; @@ -67,24 +69,26 @@ * @Description: [] */ @Slf4j -public class LogicalDatabaseChangeTask extends BaseTask>> { +public class LogicalDatabaseChangeTask extends TaskBase>> { private SqlRewriter sqlRewriter; private ExecutionGroupContext executionGroupContext; private PublishLogicalDatabaseChangeReq taskParameters; private List> executionGroups; private GroupExecutionEngine executorEngine; + public LogicalDatabaseChangeTask() {} + @Override - protected void doInit(JobContext context) throws Exception { + protected void doInit(JobContext context) { Map jobParameters = context.getJobParameters(); taskParameters = JsonUtils.fromJson(jobParameters.get(JobParametersKeyConstants.TASK_PARAMETER_JSON_KEY), PublishLogicalDatabaseChangeReq.class); sqlRewriter = new RelationFactorRewriter(); executionGroups = new ArrayList<>(); + initExecutorContext(); } - @Override - protected boolean doStart(JobContext context, TaskContext taskContext) throws Exception { + private void initExecutorContext() { try { DialectType dialectType = taskParameters.getLogicalDatabaseResp().getDialectType(); DetailLogicalDatabaseResp detailLogicalDatabaseResp = taskParameters.getLogicalDatabaseResp(); @@ -176,7 +180,14 @@ protected boolean doStart(JobContext context, TaskContext taskContext) throws Ex this.executionGroupContext = executorEngine.execute(executionGroups); } catch (Exception ex) { log.warn("start logical database change task failed, ", ex); - taskContext.getExceptionListener().onException(ex); + context.getExceptionListener().onException(ex); + } + } + + @Override + public boolean start() throws Exception { + if (Objects.isNull(this.executionGroupContext)) { + log.warn("logical database change task is not initialized"); return false; } while (!Thread.currentThread().isInterrupted()) { @@ -186,7 +197,7 @@ protected boolean doStart(JobContext context, TaskContext taskContext) throws Ex } if (CollectionUtils.isNotEmpty(this.executionGroupContext.getThrowables())) { log.warn("logical database change task failed, ", this.executionGroupContext.getThrowables()); - taskContext.getExceptionListener().onException(this.executionGroupContext.getThrowables().get(0)); + context.getExceptionListener().onException(this.executionGroupContext.getThrowables().get(0)); return false; } try { @@ -200,12 +211,12 @@ protected boolean doStart(JobContext context, TaskContext taskContext) throws Ex } @Override - protected void doStop() throws Exception { + public void stop() { this.executorEngine.terminateAll(); } @Override - protected void doClose() throws Exception { + public void close() throws Exception { if (this.executorEngine != null) { this.executorEngine.close(); } @@ -222,8 +233,16 @@ public Map> getTaskResult() { } @Override - protected void afterModifiedJobParameters() throws Exception { - Map currentJobParameters = getJobParameters(); + public boolean modify(Map jobParameters) { + if (!super.modify(jobParameters)) { + return false; + } + afterModifiedJobParameters(); + return true; + } + + protected void afterModifiedJobParameters() { + Map currentJobParameters = jobContext.getJobParameters(); if (currentJobParameters == null || currentJobParameters.isEmpty()) { return; } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/runtime/PreCheckTask.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/precheck/PreCheckTask.java similarity index 94% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/task/runtime/PreCheckTask.java rename to server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/precheck/PreCheckTask.java index 3557a7844c..9f5a2ea994 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/runtime/PreCheckTask.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/precheck/PreCheckTask.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.task.runtime; +package com.oceanbase.odc.service.task.base.precheck; import java.io.IOException; import java.io.InputStream; @@ -62,10 +62,9 @@ import com.oceanbase.odc.service.sqlcheck.SqlCheckRuleFactory; import com.oceanbase.odc.service.sqlcheck.model.CheckViolation; import com.oceanbase.odc.service.sqlcheck.rule.SqlCheckRules; +import com.oceanbase.odc.service.task.base.TaskBase; import com.oceanbase.odc.service.task.caller.JobContext; import com.oceanbase.odc.service.task.constants.JobParametersKeyConstants; -import com.oceanbase.odc.service.task.executor.task.BaseTask; -import com.oceanbase.odc.service.task.executor.task.TaskContext; import com.oceanbase.odc.service.task.util.JobUtils; import lombok.NonNull; @@ -76,7 +75,7 @@ * @date 2024/1/30 11:02 */ @Slf4j -public class PreCheckTask extends BaseTask { +public class PreCheckTask extends TaskBase { private PreCheckTaskParameters parameters; private List userInputSqls; @@ -88,20 +87,22 @@ public class PreCheckTask extends BaseTask { private SqlCheckTaskResult sqlCheckResult = null; private DatabasePermissionCheckResult permissionCheckResult = null; + public PreCheckTask() {} + @Override protected void doInit(JobContext context) throws Exception { this.taskId = getJobContext().getJobIdentity().getId(); log.info("Initiating pre-check task, taskId={}", taskId); - this.parameters = JobUtils.fromJson(getJobParameters().get(JobParametersKeyConstants.TASK_PARAMETER_JSON_KEY), - PreCheckTaskParameters.class); + this.parameters = + JobUtils.fromJson(jobContext.getJobParameters().get(JobParametersKeyConstants.TASK_PARAMETER_JSON_KEY), + PreCheckTaskParameters.class); log.info("Load pre-check task parameters successfully, taskId={}", taskId); loadUserInputSqlContent(); loadUploadFileInputStream(); log.info("Load sql content successfully, taskId={}", taskId); } - @Override - protected boolean doStart(JobContext context, TaskContext taskContext) throws Exception { + public boolean start() throws Exception { try { List sqls = new ArrayList<>(); this.overLimit = getSqlContentUntilOverLimit(sqls, this.parameters.getMaxReadContentBytes()); @@ -118,7 +119,7 @@ protected boolean doStart(JobContext context, TaskContext taskContext) throws Ex this.success = true; log.info("Pre-check task end up running, task id: {}", taskId); } catch (Throwable e) { - taskContext.getExceptionListener().onException(e); + context.getExceptionListener().onException(e); throw e; } finally { tryCloseInputStream(); @@ -127,12 +128,12 @@ protected boolean doStart(JobContext context, TaskContext taskContext) throws Ex } @Override - protected void doStop() throws Exception { + public void stop() { tryCloseInputStream(); } @Override - protected void doClose() throws Exception { + public void close() { tryCloseInputStream(); } @@ -197,7 +198,7 @@ private void loadUploadFileInputStream() throws IOException { List objectMetadataList = this.parameters.getSqlFileObjectMetadatas(); if (Objects.nonNull(params) && CollectionUtils.isNotEmpty(objectMetadataList)) { this.uploadFileInputStream = ObjectStorageUtils.loadObjectsForTask(objectMetadataList, - getCloudObjectStorageService(), JobUtils.getExecutorDataPath(), -1).getInputStream(); + context.getSharedStorage(), JobUtils.getExecutorDataPath(), -1).getInputStream(); this.uploadFileSqlIterator = SqlUtils.iterator(this.parameters.getConnectionConfig().getDialectType(), params.getDelimiter(), this.uploadFileInputStream, StandardCharsets.UTF_8); } @@ -243,7 +244,7 @@ private List checkViolations(List sqls) { } ConnectionConfig config = this.parameters.getConnectionConfig(); List rules = this.parameters.getRules(); - OBConsoleDataSourceFactory factory = new OBConsoleDataSourceFactory(config, true, false); + OBConsoleDataSourceFactory factory = new OBConsoleDataSourceFactory(config, true, false, false); factory.resetSchema(origin -> OBConsoleDataSourceFactory .getSchema(this.parameters.getRiskLevelDescriber().getDatabaseName(), config.getDialectType())); SqlCheckContext checkContext = new SqlCheckContext((long) sqls.size()); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/runtime/PreCheckTaskParameters.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/precheck/PreCheckTaskParameters.java similarity index 98% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/task/runtime/PreCheckTaskParameters.java rename to server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/precheck/PreCheckTaskParameters.java index bb0dd26324..12971d948a 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/runtime/PreCheckTaskParameters.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/precheck/PreCheckTaskParameters.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.task.runtime; +package com.oceanbase.odc.service.task.base.precheck; import java.io.Serializable; import java.util.List; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/runtime/RollbackPlanTask.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/rollback/RollbackPlanTask.java similarity index 94% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/task/runtime/RollbackPlanTask.java rename to server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/rollback/RollbackPlanTask.java index c391ec409c..41c586d817 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/runtime/RollbackPlanTask.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/rollback/RollbackPlanTask.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.task.runtime; +package com.oceanbase.odc.service.task.base.rollback; import java.io.File; import java.io.IOException; @@ -47,10 +47,9 @@ import com.oceanbase.odc.service.rollbackplan.UnsupportedSqlTypeForRollbackPlanException; import com.oceanbase.odc.service.rollbackplan.model.RollbackPlan; import com.oceanbase.odc.service.session.factory.DefaultConnectSessionFactory; +import com.oceanbase.odc.service.task.base.TaskBase; import com.oceanbase.odc.service.task.caller.JobContext; import com.oceanbase.odc.service.task.constants.JobParametersKeyConstants; -import com.oceanbase.odc.service.task.executor.task.BaseTask; -import com.oceanbase.odc.service.task.executor.task.TaskContext; import com.oceanbase.odc.service.task.util.JobUtils; import lombok.extern.slf4j.Slf4j; @@ -60,7 +59,7 @@ * @date 2024/2/6 11:03 */ @Slf4j -public class RollbackPlanTask extends BaseTask { +public class RollbackPlanTask extends TaskBase { private RollbackPlanTaskParameters parameters; private List userInputSqls; @@ -71,20 +70,22 @@ public class RollbackPlanTask extends BaseTask { private volatile boolean success = false; private volatile boolean aborted = false; + public RollbackPlanTask() {} + @Override protected void doInit(JobContext context) throws Exception { this.taskId = getJobContext().getJobIdentity().getId(); log.info("Initiating generate-rollback-plan task, taskId={}", taskId); - this.parameters = JobUtils.fromJson(getJobParameters().get(JobParametersKeyConstants.TASK_PARAMETER_JSON_KEY), - RollbackPlanTaskParameters.class); + this.parameters = + JobUtils.fromJson(jobContext.getJobParameters().get(JobParametersKeyConstants.TASK_PARAMETER_JSON_KEY), + RollbackPlanTaskParameters.class); log.info("Load generate-rollback-plan task parameters successfully, taskId={}", taskId); loadUserInputSqlContent(); loadUploadFileInputStream(); log.info("Load sql content successfully, taskId={}", taskId); } - @Override - protected boolean doStart(JobContext context, TaskContext taskContext) throws Exception { + public boolean start() throws Exception { try { long startTimeMills = System.currentTimeMillis(); ConnectionConfig connectionConfig = parameters.getConnectionConfig(); @@ -156,7 +157,7 @@ protected boolean doStart(JobContext context, TaskContext taskContext) throws Ex } } catch (Exception e) { rollbackPlanTaskResult = RollbackPlanTaskResult.fail(e.getMessage()); - taskContext.getExceptionListener().onException(e); + context.getExceptionListener().onException(e); throw e; } finally { tryCloseInputStream(); @@ -165,13 +166,13 @@ protected boolean doStart(JobContext context, TaskContext taskContext) throws Ex } @Override - protected void doStop() throws Exception { + public void stop() { this.aborted = true; tryCloseInputStream(); } @Override - protected void doClose() throws Exception { + public void close() { tryCloseInputStream(); } @@ -200,7 +201,7 @@ private void loadUploadFileInputStream() throws IOException { if (CollectionUtils.isNotEmpty(objectMetadataList)) { this.uploadFileInputStream = ObjectStorageUtils - .loadObjectsForTask(objectMetadataList, getCloudObjectStorageService(), + .loadObjectsForTask(objectMetadataList, context.getSharedStorage(), JobUtils.getExecutorDataPath(), parameters.getRollbackProperties().getMaxRollbackContentSizeBytes()) .getInputStream(); @@ -223,7 +224,7 @@ private void handleRollbackResult(String rollbackResult) { String resultFileId = StringUtils.uuid(); String filePath = String.format("%s/%s.sql", resultFileRootPath, resultFileId); FileUtils.writeStringToFile(new File(filePath), rollbackResult, StandardCharsets.UTF_8); - CloudObjectStorageService cloudObjectStorageService = getCloudObjectStorageService(); + CloudObjectStorageService cloudObjectStorageService = context.getSharedStorage(); if (Objects.nonNull(cloudObjectStorageService) && cloudObjectStorageService.supported()) { File tempFile = new File(filePath); try { diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/runtime/RollbackPlanTaskParameters.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/rollback/RollbackPlanTaskParameters.java similarity index 97% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/task/runtime/RollbackPlanTaskParameters.java rename to server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/rollback/RollbackPlanTaskParameters.java index 3427b84f66..5bb6386e82 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/runtime/RollbackPlanTaskParameters.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/rollback/RollbackPlanTaskParameters.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.task.runtime; +package com.oceanbase.odc.service.task.base.rollback; import java.io.Serializable; import java.util.List; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/task/SqlPlanTask.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/sqlplan/SqlPlanTask.java similarity index 96% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/task/SqlPlanTask.java rename to server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/sqlplan/SqlPlanTask.java index 7925167df1..11822ca176 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/task/SqlPlanTask.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/base/sqlplan/SqlPlanTask.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.task.executor.task; +package com.oceanbase.odc.service.task.base.sqlplan; import java.io.BufferedInputStream; import java.io.ByteArrayInputStream; @@ -69,6 +69,7 @@ import com.oceanbase.odc.service.session.initializer.ConsoleTimeoutInitializer; import com.oceanbase.odc.service.session.model.SqlExecuteResult; import com.oceanbase.odc.service.sqlplan.model.SqlPlanTaskResult; +import com.oceanbase.odc.service.task.base.TaskBase; import com.oceanbase.odc.service.task.caller.JobContext; import com.oceanbase.odc.service.task.constants.JobParametersKeyConstants; import com.oceanbase.odc.service.task.exception.JobException; @@ -81,7 +82,7 @@ import lombok.extern.slf4j.Slf4j; @Slf4j -public class SqlPlanTask extends BaseTask { +public class SqlPlanTask extends TaskBase { private PublishSqlPlanJobReq parameters; @@ -109,11 +110,14 @@ public class SqlPlanTask extends BaseTask { private final List csvFileMappers = new ArrayList<>(); + public SqlPlanTask() {} + @Override protected void doInit(JobContext context) { this.result = new SqlPlanTaskResult(); - this.parameters = JobUtils.fromJson(getJobParameters().get(JobParametersKeyConstants.META_TASK_PARAMETER_JSON), - PublishSqlPlanJobReq.class); + this.parameters = + JobUtils.fromJson(jobContext.getJobParameters().get(JobParametersKeyConstants.META_TASK_PARAMETER_JSON), + PublishSqlPlanJobReq.class); JobContext jobContext = getJobContext(); Map jobProperties = jobContext.getJobProperties(); this.taskId = jobContext.getJobIdentity().getId(); @@ -137,7 +141,7 @@ protected void doInit(JobContext context) { } @Override - protected boolean doStart(JobContext context, TaskContext taskContext) throws Exception { + public boolean start() throws Exception { try { int index = 0; initSqlInputStream(); @@ -180,7 +184,7 @@ protected boolean doStart(JobContext context, TaskContext taskContext) throws Ex break; } log.warn("Sql task execution failed, will continue to execute next statement.", e); - taskContext.getExceptionListener().onException(e); + context.getExceptionListener().onException(e); } } result.setTotalStatements(index); @@ -214,7 +218,7 @@ private void initSqlInputStream() { return; } - CloudObjectStorageService cloudObjectStorageService = getCloudObjectStorageService(); + CloudObjectStorageService cloudObjectStorageService = context.getSharedStorage(); if (Objects.isNull(cloudObjectStorageService) || !cloudObjectStorageService.supported()) { log.warn("Cloud object storage service not supported."); throw new UnexpectedException("Cloud object storage service not supported"); @@ -243,14 +247,13 @@ private void initSqlInputStream() { } } - @Override - protected void doStop() { + public void stop() { canceled = true; } @Override - protected void doClose() { + public void close() { tryExpireConnectionSession(); } @@ -322,7 +325,7 @@ private boolean executeSqlWithRetries(String sql) { private ConnectionSession generateSession() { ConnectionConfig connectionConfig = JobUtils.fromJson( - getJobParameters().get(JobParametersKeyConstants.CONNECTION_CONFIG), ConnectionConfig.class); + jobContext.getJobParameters().get(JobParametersKeyConstants.CONNECTION_CONFIG), ConnectionConfig.class); DefaultConnectSessionFactory sessionFactory = new DefaultConnectSessionFactory(connectionConfig); sessionFactory.setSessionTimeoutMillis(parameters.getTimeoutMillis()); ConnectionSession connectionSession = sessionFactory.generateSession(); @@ -467,7 +470,7 @@ private void upload() { private String uploadToOSS(String filePath) { // Public cloud scenario, need to upload files to OSS - CloudObjectStorageService cloudObjectStorageService = getCloudObjectStorageService(); + CloudObjectStorageService cloudObjectStorageService = context.getSharedStorage(); if (Objects.nonNull(cloudObjectStorageService) && cloudObjectStorageService.supported()) { File file = new File(filePath); String ossAddress; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/BaseJobCaller.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/BaseJobCaller.java index 82e1e863e8..17e285e573 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/BaseJobCaller.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/BaseJobCaller.java @@ -18,6 +18,7 @@ import com.oceanbase.odc.common.event.AbstractEvent; import com.oceanbase.odc.metadb.task.JobEntity; +import com.oceanbase.odc.service.resource.ResourceID; import com.oceanbase.odc.service.task.config.JobConfiguration; import com.oceanbase.odc.service.task.config.JobConfigurationHolder; import com.oceanbase.odc.service.task.config.JobConfigurationValidator; @@ -37,7 +38,6 @@ */ @Slf4j public abstract class BaseJobCaller implements JobCaller { - @Override public void start(JobContext context) throws JobException { JobConfigurationValidator.validComponent(); @@ -73,7 +73,7 @@ private void afterStartFailed(JobIdentity ji, ExecutorIdentifier executorIdentifier, Exception ex) throws JobException { if (executorIdentifier != null) { try { - destroy(ji); + finish(ji); } catch (JobException e) { // if destroy failed, domain job will destroy it log.warn("Destroy executor {} occur exception", executorIdentifier); @@ -92,9 +92,11 @@ public void stop(JobIdentity ji) throws JobException { JobEntity jobEntity = taskFrameworkService.find(ji.getId()); String executorEndpoint = jobEntity.getExecutorEndpoint(); + ExecutorIdentifier identifier = ExecutorIdentifierParser.parser(jobEntity.getExecutorIdentifier()); + ResourceID resourceID = ResourceIDUtil.getResourceID(identifier, jobEntity); try { if (executorEndpoint != null - && isExecutorExist(ExecutorIdentifierParser.parser(jobEntity.getExecutorIdentifier()))) { + && isExecutorExist(identifier, resourceID)) { taskExecutorClient.stop(executorEndpoint, ji); } afterStopSucceed(ji); @@ -125,7 +127,7 @@ public void modify(JobIdentity ji, String jobParametersJson) throws JobException } @Override - public void destroy(JobIdentity ji) throws JobException { + public void finish(JobIdentity ji) throws JobException { JobConfigurationValidator.validComponent(); JobConfiguration jobConfiguration = JobConfigurationHolder.getJobConfiguration(); TaskFrameworkService taskFrameworkService = jobConfiguration.getTaskFrameworkService(); @@ -138,12 +140,15 @@ public void destroy(JobIdentity ji) throws JobException { updateExecutorDestroyed(ji); return; } + ExecutorIdentifier identifier = ExecutorIdentifierParser.parser(executorIdentifier); + ResourceID resourceID = ResourceIDUtil.getResourceID(identifier, jobEntity); log.info("Preparing destroy,jobId={}, executorIdentifier={}.", ji.getId(), executorIdentifier); - doDestroy(ji, ExecutorIdentifierParser.parser(executorIdentifier)); + doFinish(ji, identifier, resourceID); } + @Override - public boolean canBeDestroy(JobIdentity ji) { + public boolean canBeFinish(JobIdentity ji) { JobConfiguration jobConfiguration = JobConfigurationHolder.getJobConfiguration(); TaskFrameworkService taskFrameworkService = jobConfiguration.getTaskFrameworkService(); JobEntity jobEntity = taskFrameworkService.find(ji.getId()); @@ -151,24 +156,30 @@ public boolean canBeDestroy(JobIdentity ji) { if (executorIdentifier == null) { return true; } - return canBeDestroy(ji, ExecutorIdentifierParser.parser(executorIdentifier)); + ExecutorIdentifier identifier = ExecutorIdentifierParser.parser(executorIdentifier); + ResourceID resourceID = ResourceIDUtil.getResourceID(identifier, jobEntity); + return canBeFinish(ji, identifier, resourceID); } - protected abstract void doDestroy(JobIdentity ji, ExecutorIdentifier ei) throws JobException; + /** + * detect if job on resource id can be finished + * + * @param ji + * @param ei + * @param resourceID resource id task working on + * @return + */ + protected abstract boolean canBeFinish(JobIdentity ji, ExecutorIdentifier ei, ResourceID resourceID); + + protected abstract void doFinish(JobIdentity ji, ExecutorIdentifier ei, ResourceID resourceID) + throws JobException; - protected abstract boolean canBeDestroy(JobIdentity ji, ExecutorIdentifier ei); private void publishEvent(T event) { JobConfiguration configuration = JobConfigurationHolder.getJobConfiguration(); configuration.getEventPublisher().publishEvent(event); } - protected void destroyInternal(ExecutorIdentifier identifier) throws JobException { - if (identifier == null || identifier.getExecutorName() == null) { - return; - } - doDestroyInternal(identifier); - } protected void updateExecutorDestroyed(JobIdentity ji) throws JobException { JobConfiguration jobConfiguration = JobConfigurationHolder.getJobConfiguration(); @@ -185,8 +196,6 @@ protected void updateExecutorDestroyed(JobIdentity ji) throws JobException { protected abstract void doStop(JobIdentity ji) throws JobException; - protected abstract void doDestroyInternal(ExecutorIdentifier identifier) throws JobException; - - protected abstract boolean isExecutorExist(ExecutorIdentifier identifier) throws JobException; - + protected abstract boolean isExecutorExist(ExecutorIdentifier identifier, ResourceID resourceID) + throws JobException; } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/DefaultExecutorIdentifier.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/DefaultExecutorIdentifier.java index 113c492a89..a5f15817cf 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/DefaultExecutorIdentifier.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/DefaultExecutorIdentifier.java @@ -16,6 +16,7 @@ package com.oceanbase.odc.service.task.caller; import com.oceanbase.odc.common.util.StringUtils; +import com.oceanbase.odc.service.common.util.UrlUtils; import lombok.Builder; import lombok.Data; @@ -56,11 +57,10 @@ public String toString() { .append(port); if (StringUtils.isNotBlank(namespace)) { sb.append("/"); - sb.append(namespace); + sb.append(UrlUtils.encode(namespace)); } sb.append("/") - .append(executorName); + .append(UrlUtils.encode(executorName)); return sb.toString(); } - } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/ExecutorIdentifierParser.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/ExecutorIdentifierParser.java index 77df720018..b4508105da 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/ExecutorIdentifierParser.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/ExecutorIdentifierParser.java @@ -17,6 +17,7 @@ import org.springframework.web.util.UriComponents; +import com.oceanbase.odc.common.util.StringUtils; import com.oceanbase.odc.service.common.util.UrlUtils; import com.oceanbase.odc.service.task.exception.TaskRuntimeException; @@ -35,12 +36,19 @@ public static ExecutorIdentifier parser(String identifierString) { throw new TaskRuntimeException("Illegal executor name : " + path); } - String namespace = path.substring(0, nameIndex).replace("/", ""); + String tmpStr = path.substring(0, nameIndex); + String[] regionAndNamespace = StringUtils.split(tmpStr, "/"); + String namespace = null; + // new version + if (regionAndNamespace.length == 1) { + // old version + namespace = regionAndNamespace[0]; + } return DefaultExecutorIdentifier.builder().host(uriComponents.getHost()) .port(uriComponents.getPort()) .protocol(uriComponents.getScheme()) - .namespace(namespace.length() == 0 ? null : namespace) - .executorName(path.substring(nameIndex).replace("/", "")) + .namespace(StringUtils.isEmpty(namespace) ? null : UrlUtils.decode(namespace)) + .executorName(UrlUtils.decode(path.substring(nameIndex).replace("/", ""))) .build(); } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/ExecutorProcessBuilderFactory.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/ExecutorProcessBuilderFactory.java index 2c13d54127..2721b03a70 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/ExecutorProcessBuilderFactory.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/ExecutorProcessBuilderFactory.java @@ -43,15 +43,18 @@ public ProcessBuilder getProcessBuilder(ProcessConfig processConfig, long jobId, commands.add("-D" + JobUtils.generateExecutorSelectorOnProcess(executorName)); commands.addAll(jvmOptions(processConfig, jobId)); if (ODC_SERVER_EXECUTABLE_JAR.matcher(runtimeMxBean.getClassPath()).matches()) { - // start odc executor by java -jar - commands.add("-jar"); + // start odc executor by java -cp + commands.add("-cp"); // set jar package file name in commands commands.add(runtimeMxBean.getClassPath()); + commands.add("-Dloader.main=" + JobConstants.ODC_AGENT_CLASS_NAME); + commands.add("org.springframework.boot.loader.PropertiesLauncher"); } else { // start odc executor by java -classpath - commands.add("-classpath"); + commands.add("-cp"); commands.add(runtimeMxBean.getClassPath()); - commands.add(JobConstants.ODC_SERVER_CLASS_NAME); + commands.add(JobConstants.ODC_AGENT_CLASS_NAME); + // commands.add("org.springframework.boot.loader.PropertiesLauncher"); } pb.command(commands); pb.directory(new File(".")); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/JobCaller.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/JobCaller.java index 2ada663165..2bdf81e671 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/JobCaller.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/JobCaller.java @@ -52,14 +52,19 @@ public interface JobCaller { */ void modify(JobIdentity ji, String jobParametersJson) throws JobException; - /** - * destroy a odc job executor - * - * @param ji job identity - * @throws JobException throws JobException when stop job failed + * complete the job, process should be quit resource can be released as well + * + * @param ji + * @throws JobException */ - void destroy(JobIdentity ji) throws JobException; + void finish(JobIdentity ji) throws JobException; - boolean canBeDestroy(JobIdentity ji); + /** + * if job can be finished + * + * @param ji + * @return false is job/resource in unknown state + */ + boolean canBeFinish(JobIdentity ji); } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/JobCallerBuilder.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/JobCallerBuilder.java index bf21acf357..0ffe215036 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/JobCallerBuilder.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/JobCallerBuilder.java @@ -22,12 +22,14 @@ import org.apache.commons.io.FileUtils; import com.oceanbase.odc.common.util.StringUtils; +import com.oceanbase.odc.service.resource.ResourceManager; import com.oceanbase.odc.service.task.config.JobConfigurationHolder; import com.oceanbase.odc.service.task.config.TaskFrameworkProperties; import com.oceanbase.odc.service.task.constants.JobEnvKeyConstants; import com.oceanbase.odc.service.task.enums.TaskMonitorMode; import com.oceanbase.odc.service.task.enums.TaskRunMode; import com.oceanbase.odc.service.task.jasypt.JasyptEncryptorConfigProperties; +import com.oceanbase.odc.service.task.resource.PodConfig; import com.oceanbase.odc.service.task.util.JobPropertiesUtils; import com.oceanbase.odc.service.task.util.JobUtils; @@ -38,8 +40,14 @@ */ public class JobCallerBuilder { - public static JobCaller buildProcessCaller(JobContext context) { - Map environments = new JobEnvironmentFactory().build(context, TaskRunMode.PROCESS); + /** + * build process caller with given env + * + * @param context + * @param environments env for process builder + * @return + */ + public static JobCaller buildProcessCaller(JobContext context, Map environments) { JobUtils.encryptEnvironments(environments); /** * write JobContext to file in case of exceeding the environments size limit; set the file path in @@ -69,12 +77,15 @@ public static JobCaller buildProcessCaller(JobContext context) { return new ProcessJobCaller(config); } - public static JobCaller buildK8sJobCaller(K8sJobClient k8sJobClient, PodConfig podConfig, JobContext context) { + /** + * build k8s start env + * + * @param context + * @return + */ + public static Map buildK8sEnv(JobContext context) { Map environments = new JobEnvironmentFactory().build(context, TaskRunMode.K8S); - // common environment variables - environments.put(JobEnvKeyConstants.ODC_LOG_DIRECTORY, podConfig.getMountPath()); - Map jobProperties = context.getJobProperties(); // executor listen port @@ -98,11 +109,18 @@ public static JobCaller buildK8sJobCaller(K8sJobClient k8sJobClient, PodConfig p environments.put(JobEnvKeyConstants.ODC_PROPERTY_ENCRYPTION_PREFIX, jasyptProperties.getPrefix()); environments.put(JobEnvKeyConstants.ODC_PROPERTY_ENCRYPTION_SUFFIX, jasyptProperties.getSuffix()); environments.put(JobEnvKeyConstants.ODC_PROPERTY_ENCRYPTION_SALT, jasyptProperties.getSalt()); + return environments; + } + public static JobCaller buildK8sJobCaller(PodConfig podConfig, JobContext context, + ResourceManager resourceManager) { + Map environments = buildK8sEnv(context); + // common environment variables + environments.put(JobEnvKeyConstants.ODC_LOG_DIRECTORY, podConfig.getMountPath()); // do encryption for sensitive information JobUtils.encryptEnvironments(environments); podConfig.setEnvironments(environments); - return new K8sJobCaller(k8sJobClient, podConfig); + return new K8sJobCaller(podConfig, resourceManager); } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/K8sJobCaller.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/K8sJobCaller.java index f6a8df5610..973d14b5db 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/K8sJobCaller.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/K8sJobCaller.java @@ -16,14 +16,18 @@ package com.oceanbase.odc.service.task.caller; -import static com.oceanbase.odc.service.task.constants.JobConstants.ODC_EXECUTOR_CANNOT_BE_DESTROYED; - import java.util.Optional; -import com.oceanbase.odc.metadb.task.JobEntity; -import com.oceanbase.odc.service.task.config.JobConfiguration; -import com.oceanbase.odc.service.task.config.JobConfigurationHolder; +import com.oceanbase.odc.service.resource.ResourceID; +import com.oceanbase.odc.service.resource.ResourceLocation; +import com.oceanbase.odc.service.resource.ResourceManager; +import com.oceanbase.odc.service.resource.ResourceState; +import com.oceanbase.odc.service.resource.ResourceWithID; import com.oceanbase.odc.service.task.exception.JobException; +import com.oceanbase.odc.service.task.resource.DefaultResourceOperatorBuilder; +import com.oceanbase.odc.service.task.resource.K8sPodResource; +import com.oceanbase.odc.service.task.resource.K8sResourceContext; +import com.oceanbase.odc.service.task.resource.PodConfig; import com.oceanbase.odc.service.task.schedule.JobIdentity; import com.oceanbase.odc.service.task.util.JobUtils; @@ -36,90 +40,72 @@ */ @Slf4j public class K8sJobCaller extends BaseJobCaller { - - private final K8sJobClient client; - private final PodConfig podConfig; - - public K8sJobCaller(K8sJobClient client, PodConfig podConfig) { - this.client = client; - this.podConfig = podConfig; + /** + * base job config + */ + private final PodConfig defaultPodConfig; + private final ResourceManager resourceManager; + + public K8sJobCaller(PodConfig podConfig, ResourceManager resourceManager) { + this.defaultPodConfig = podConfig; + this.resourceManager = resourceManager; } @Override public ExecutorIdentifier doStart(JobContext context) throws JobException { - String jobName = JobUtils.generateExecutorName(context.getJobIdentity()); + try { + ResourceLocation resourceLocation = buildResourceLocation(context); + ResourceWithID resource = + resourceManager.create(resourceLocation, buildK8sResourceContext(context, resourceLocation)); + String arn = resource.getResource().resourceID().getIdentifier(); + return DefaultExecutorIdentifier.builder().namespace(resource.getResource().getNamespace()) + .executorName(arn).build(); + } catch (Throwable e) { + throw new JobException("doStart failed for " + context, e); + } + } - String arn = client.create(podConfig.getNamespace(), jobName, podConfig.getImage(), - podConfig.getCommand(), podConfig); + protected K8sResourceContext buildK8sResourceContext(JobContext context, ResourceLocation resourceLocation) { + String jobName = JobUtils.generateExecutorName(context.getJobIdentity()); + return new K8sResourceContext(defaultPodConfig, jobName, resourceLocation.getRegion(), + resourceLocation.getGroup(), + DefaultResourceOperatorBuilder.CLOUD_K8S_POD_TYPE, context); + } - return DefaultExecutorIdentifier.builder().namespace(podConfig.getNamespace()) - .executorName(arn).build(); + protected ResourceLocation buildResourceLocation(JobContext context) { + // TODO(tianke): confirm is this correct? + String region = ResourceIDUtil.checkAndGetJobProperties(context.getJobProperties(), + ResourceIDUtil.REGION_PROP_NAME, ResourceIDUtil.DEFAULT_PROP_VALUE); + String group = ResourceIDUtil.checkAndGetJobProperties(context.getJobProperties(), + ResourceIDUtil.GROUP_PROP_NAME, ResourceIDUtil.DEFAULT_PROP_VALUE); + return new ResourceLocation(region, group); } @Override public void doStop(JobIdentity ji) throws JobException {} @Override - protected void doDestroy(JobIdentity ji, ExecutorIdentifier ei) throws JobException { + protected void doFinish(JobIdentity ji, ExecutorIdentifier ei, ResourceID resourceID) + throws JobException { + resourceManager.release(resourceID); updateExecutorDestroyed(ji); - Optional k8sJobResponse = client.get(ei.getNamespace(), ei.getExecutorName()); - if (k8sJobResponse.isPresent()) { - if (PodStatus.PENDING == PodStatus.of(k8sJobResponse.get().getResourceStatus())) { - JobConfiguration jobConfiguration = JobConfigurationHolder.getJobConfiguration(); - JobEntity jobEntity = jobConfiguration.getTaskFrameworkService().find(ji.getId()); - if ((System.currentTimeMillis() - jobEntity.getStartedTime().getTime()) / 1000 > podConfig - .getPodPendingTimeoutSeconds()) { - log.info("Pod pending timeout, will be deleted, jobId={}, pod={}, " - + "podPendingTimeoutSeconds={}.", ji.getId(), ei.getExecutorName(), - podConfig.getPodPendingTimeoutSeconds()); - } else { - // Pod cannot be deleted when pod pending is not timeout, - // so throw exception representative delete failed - throw new JobException(ODC_EXECUTOR_CANNOT_BE_DESTROYED + - "Destroy pod failed, jodId={0}, identifier={1}, podStatus={2}", - ji.getId(), ei.getExecutorName(), k8sJobResponse.get().getResourceStatus()); - } - } - log.info("Found pod, delete it, jobId={}, pod={}.", ji.getId(), ei.getExecutorName()); - destroyInternal(ei); - } } @Override - protected boolean canBeDestroy(JobIdentity ji, ExecutorIdentifier ei) { - Optional k8sJobResponse = null; - try { - k8sJobResponse = client.get(ei.getNamespace(), ei.getExecutorName()); - } catch (JobException e) { - log.warn("Get k8s pod occur error, jobId={}", ji.getId(), e); - return false; - } - if (k8sJobResponse.isPresent()) { - if (PodStatus.PENDING == PodStatus.of(k8sJobResponse.get().getResourceStatus())) { - JobConfiguration jobConfiguration = JobConfigurationHolder.getJobConfiguration(); - JobEntity jobEntity = jobConfiguration.getTaskFrameworkService().find(ji.getId()); - if ((System.currentTimeMillis() - jobEntity.getStartedTime().getTime()) / 1000 <= podConfig - .getPodPendingTimeoutSeconds()) { - // Pod cannot be deleted when pod pending is not timeout, - // so throw exception representative cannot delete - log.warn("Cannot destroy pod, pending is not timeout, jodId={}, identifier={}, podStatus={}", - ji.getId(), ei.getExecutorName(), k8sJobResponse.get().getResourceStatus()); - return false; - } - } - } - return true; + protected boolean canBeFinish(JobIdentity ji, ExecutorIdentifier ei, ResourceID resourceID) { + return resourceManager.canBeDestroyed(resourceID); } @Override - protected void doDestroyInternal(ExecutorIdentifier identifier) throws JobException { - client.delete(podConfig.getNamespace(), identifier.getExecutorName()); - } - - @Override - protected boolean isExecutorExist(ExecutorIdentifier identifier) throws JobException { - Optional executorOptional = client.get(identifier.getNamespace(), identifier.getExecutorName()); - return executorOptional.isPresent() && - PodStatus.of(executorOptional.get().getResourceStatus()) != PodStatus.TERMINATING; + protected boolean isExecutorExist(ExecutorIdentifier identifier, ResourceID resourceID) + throws JobException { + try { + Optional executorOptional = + resourceManager.query(resourceID); + return executorOptional.isPresent() && !ResourceState.isDestroying( + executorOptional.get().getResourceState()); + } catch (Throwable e) { + throw new JobException("invoke isExecutor failed", e); + } } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/K8sJobClient.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/K8sJobClient.java deleted file mode 100644 index 7ece29a027..0000000000 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/K8sJobClient.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright (c) 2023 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oceanbase.odc.service.task.caller; - -import java.util.List; -import java.util.Optional; - -import com.oceanbase.odc.service.task.exception.JobException; - -/** - * K8sJobClient is a client to CRUD k8s job in different environment. eg: native or cloud k8s - * - * @author yaobin - * @date 2023-11-15 - * @since 4.2.4 - */ -public interface K8sJobClient { - - /** - * create job in k8s namespace with pod name and use a specific image - * - * @param namespace namespace name - * @param name pod name - * @param image image name - * @param command image start command - * @param podConfig pod config - * @return arn string - * @throws JobException throws exception when create job failed - */ - String create(String namespace, String name, String image, List command, - PodConfig podConfig) throws JobException; - - /** - * get job by serial number in k8s namespace - * - * @param namespace namespace name - * @param arn arn string - * @return job serial number - * @throws JobException throws exception when get job failed - */ - Optional get(String namespace, String arn) throws JobException; - - /** - * delete job by serial number in k8s namespace - * - * @param namespace namespace name - * @param arn arn string - * @return job serial number - * @throws JobException throws exception when delete job failed - */ - String delete(String namespace, String arn) throws JobException; -} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/K8sJobResponse.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/K8sJobResponse.java index 5507cf4a5b..b85f617c07 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/K8sJobResponse.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/K8sJobResponse.java @@ -22,7 +22,6 @@ */ public interface K8sJobResponse { - /** * deploy region */ diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/NativeK8sJobClient.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/NativeK8sJobClient.java deleted file mode 100644 index af86426d70..0000000000 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/NativeK8sJobClient.java +++ /dev/null @@ -1,194 +0,0 @@ -/* - * Copyright (c) 2023 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oceanbase.odc.service.task.caller; - -import static com.oceanbase.odc.service.task.constants.JobConstants.FIELD_SELECTOR_METADATA_NAME; -import static com.oceanbase.odc.service.task.constants.JobConstants.RESTART_POLICY_NEVER; -import static com.oceanbase.odc.service.task.constants.JobConstants.TEMPLATE_API_VERSION; -import static com.oceanbase.odc.service.task.constants.JobConstants.TEMPLATE_KIND_POD; - -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStreamReader; -import java.io.Reader; -import java.util.Collections; -import java.util.List; -import java.util.Optional; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; - -import org.apache.commons.collections4.CollectionUtils; - -import com.oceanbase.odc.common.util.EncodeUtils; -import com.oceanbase.odc.common.util.StringUtils; -import com.oceanbase.odc.core.shared.PreConditions; -import com.oceanbase.odc.core.shared.Verify; -import com.oceanbase.odc.core.shared.constant.ErrorCodes; -import com.oceanbase.odc.service.task.config.K8sProperties; -import com.oceanbase.odc.service.task.exception.JobException; - -import io.kubernetes.client.openapi.ApiClient; -import io.kubernetes.client.openapi.ApiException; -import io.kubernetes.client.openapi.Configuration; -import io.kubernetes.client.openapi.apis.CoreV1Api; -import io.kubernetes.client.openapi.models.V1Container; -import io.kubernetes.client.openapi.models.V1EnvVar; -import io.kubernetes.client.openapi.models.V1ObjectMeta; -import io.kubernetes.client.openapi.models.V1Pod; -import io.kubernetes.client.openapi.models.V1PodList; -import io.kubernetes.client.openapi.models.V1PodSpec; -import io.kubernetes.client.util.ClientBuilder; -import io.kubernetes.client.util.Config; -import io.kubernetes.client.util.KubeConfig; -import lombok.NonNull; - -/** - * @author yaobin - * @date 2023-11-15 - * @since 4.2.4 - */ -public class NativeK8sJobClient implements K8sJobClient { - - private final K8sProperties k8sProperties; - private static final long TIMEOUT_MILLS = 60000; - - public NativeK8sJobClient(K8sProperties k8sProperties) throws IOException { - this.k8sProperties = k8sProperties; - ApiClient apiClient = null; - if (StringUtils.isNotBlank(k8sProperties.getKubeConfig())) { - byte[] kubeConfigBytes = EncodeUtils.base64DecodeFromString(k8sProperties.getKubeConfig()); - Verify.notNull(kubeConfigBytes, "kube config"); - try (Reader targetReader = new InputStreamReader(new ByteArrayInputStream(kubeConfigBytes))) { - KubeConfig kubeConfig = KubeConfig.loadKubeConfig(targetReader); - apiClient = ClientBuilder.kubeconfig(kubeConfig).build(); - } - } else if (StringUtils.isNotBlank(k8sProperties.getKubeUrl())) { - apiClient = Config.defaultClient().setBasePath(k8sProperties.getKubeUrl()); - } - Verify.notNull(apiClient, "k8s api client"); - apiClient.setHttpClient(apiClient - .getHttpClient() - .newBuilder() - .readTimeout(TIMEOUT_MILLS, TimeUnit.MILLISECONDS) - .connectTimeout(TIMEOUT_MILLS, TimeUnit.MILLISECONDS) - .pingInterval(1, TimeUnit.MINUTES) - .build()); - Configuration.setDefaultApiClient(apiClient); - } - - @Override - public String create(@NonNull String namespace, @NonNull String name, @NonNull String image, - List command, @NonNull PodConfig podConfig) throws JobException { - validK8sProperties(); - - V1Pod job = getV1Pod(name, image, command, podConfig); - CoreV1Api api = new CoreV1Api(); - try { - V1Pod createdJob = api.createNamespacedPod(namespace, job, null, null, - null, null); - return createdJob.getMetadata().getName(); - } catch (ApiException e) { - if (e.getResponseBody() != null) { - throw new JobException(e.getResponseBody(), e); - } else { - throw new JobException("Create job occur error:", e); - } - } - } - - @Override - public Optional get(@NonNull String namespace, @NonNull String arn) throws JobException { - validK8sProperties(); - CoreV1Api api = new CoreV1Api(); - V1PodList job = null; - try { - job = api.listNamespacedPod(namespace, null, null, null, - FIELD_SELECTOR_METADATA_NAME + "=" + arn, - null, null, null, null, null, null, false); - } catch (ApiException e) { - throw new JobException(e.getResponseBody(), e); - } - Optional v1PodOptional = job.getItems().stream().findAny(); - if (!v1PodOptional.isPresent()) { - return Optional.empty(); - } - V1Pod v1Pod = v1PodOptional.get(); - DefaultK8sJobResponse response = new DefaultK8sJobResponse(); - response.setArn(arn); - response.setName(arn); - response.setRegion(k8sProperties.getRegion()); - response.setResourceStatus(v1Pod.getStatus().getPhase()); - response.setPodIpAddress(v1Pod.getStatus().getPodIP()); - return Optional.of(response); - } - - @Override - public String delete(@NonNull String namespace, @NonNull String arn) throws JobException { - validK8sProperties(); - CoreV1Api api = new CoreV1Api(); - V1Pod pod = null; - try { - pod = api.deleteNamespacedPod(arn, namespace, null, null, - null, null, null, null); - } catch (ApiException e) { - throw new JobException(e.getResponseBody(), e); - } - return pod.getMetadata().getName(); - } - - private V1Pod getV1Pod(String jobName, String image, List command, PodConfig podParam) { - V1Container container = new V1Container() - .name(jobName) - .image(image) - .imagePullPolicy(podParam.getImagePullPolicy()); - - if (CollectionUtils.isNotEmpty(command)) { - container.setCommand(command); - } - - if (podParam.getEnvironments().size() > 0) { - List envVars = podParam.getEnvironments().entrySet().stream() - .map(entry -> new V1EnvVar().name(entry.getKey()).value(entry.getValue())) - .collect(Collectors.toList()); - container.setEnv(envVars); - } - - V1PodSpec v1PodSpec = new V1PodSpec() - .containers(Collections.singletonList(container)) - .restartPolicy(RESTART_POLICY_NEVER); - - return new V1Pod() - .apiVersion(getVersion()).kind(getKind()) - .metadata(new V1ObjectMeta().name(jobName)) - .spec(v1PodSpec); - } - - private void validK8sProperties() { - PreConditions.validArgumentState(k8sProperties.getKubeConfig() != null - || k8sProperties.getKubeUrl() != null, - ErrorCodes.BadArgument, - new Object[] {}, "Target k8s is not set"); - } - - private String getVersion() { - return TEMPLATE_API_VERSION; - } - - private String getKind() { - return TEMPLATE_KIND_POD; - } -} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/PodStatus.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/PodStatus.java deleted file mode 100644 index d1292c739d..0000000000 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/PodStatus.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright (c) 2023 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.oceanbase.odc.service.task.caller; - -/** - * @author yaobin - * @date 2024-04-03 - * @since 4.2.4 - */ -public enum PodStatus { - - PENDING("Pending", "INIT"), - RUNNING("Running", "ALLOCATED"), - TERMINATING("Terminating", "PENDING_DELETE"), - UNKNOWN("Known", "UNKNOWN"); - - - private final String name; - private final String alias; - - PodStatus(String name, String alias) { - this.name = name; - this.alias = alias; - } - - public String getName() { - return this.name; - } - - public String getAlias() { - return this.alias; - } - - - public static PodStatus of(String s) { - for (PodStatus status : PodStatus.values()) { - if (status.name().equalsIgnoreCase(s) || status.getAlias().equalsIgnoreCase(s)) { - return status; - } - } - return UNKNOWN; - } -} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/ProcessJobCaller.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/ProcessJobCaller.java index f8b81fe2e1..c976c672c6 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/ProcessJobCaller.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/ProcessJobCaller.java @@ -18,15 +18,19 @@ import static com.oceanbase.odc.service.task.constants.JobConstants.ODC_EXECUTOR_CANNOT_BE_DESTROYED; +import java.io.File; import java.io.IOException; +import java.lang.ProcessBuilder.Redirect; import java.text.MessageFormat; import java.util.Objects; import java.util.Optional; import com.fasterxml.jackson.core.type.TypeReference; +import com.oceanbase.odc.common.json.JsonUtils; import com.oceanbase.odc.common.util.SystemUtils; import com.oceanbase.odc.metadb.task.JobEntity; import com.oceanbase.odc.service.common.response.OdcResult; +import com.oceanbase.odc.service.resource.ResourceID; import com.oceanbase.odc.service.task.config.JobConfiguration; import com.oceanbase.odc.service.task.config.JobConfigurationHolder; import com.oceanbase.odc.service.task.enums.JobStatus; @@ -52,11 +56,15 @@ public ProcessJobCaller(ProcessConfig processConfig) { } @Override - protected ExecutorIdentifier doStart(JobContext context) throws JobException { + public ExecutorIdentifier doStart(JobContext context) throws JobException { String executorName = JobUtils.generateExecutorName(context.getJobIdentity()); ProcessBuilder pb = new ExecutorProcessBuilderFactory().getProcessBuilder( processConfig, context.getJobIdentity().getId(), executorName); + log.info("start task with processConfig={}, env={}", JobUtils.toJson(processConfig), + JsonUtils.toJson(pb.environment())); + pb.redirectErrorStream(true); + pb.redirectOutput(Redirect.appendTo(new File("process-call.log"))); Process process; try { process = pb.start(); @@ -93,14 +101,15 @@ protected ExecutorIdentifier doStart(JobContext context) throws JobException { protected void doStop(JobIdentity ji) throws JobException {} @Override - protected void doDestroy(JobIdentity ji, ExecutorIdentifier ei) throws JobException { - if (isExecutorExist(ei)) { + protected void doFinish(JobIdentity ji, ExecutorIdentifier ei, ResourceID resourceID) + throws JobException { + if (isExecutorExist(ei, resourceID)) { long pid = Long.parseLong(ei.getNamespace()); log.info("Found process, try kill it, pid={}.", pid); // first update destroy time, second destroy executor. // if executor failed update will be rollback, ensure distributed transaction atomicity. updateExecutorDestroyed(ji); - destroyInternal(ei); + doDestroyInternal(ei); return; } @@ -132,9 +141,8 @@ protected void doDestroy(JobIdentity ji, ExecutorIdentifier ei) throws JobExcept + " may not on this machine, jodId={0}, identifier={1}", ji.getId(), ei); } - @Override - public boolean canBeDestroy(JobIdentity ji, ExecutorIdentifier ei) { - if (isExecutorExist(ei)) { + public boolean canBeFinish(JobIdentity ji, ExecutorIdentifier ei, ResourceID resourceID) { + if (isExecutorExist(ei, resourceID)) { log.info("Executor be found, jobId={}, identifier={}", ji.getId(), ei); return true; } @@ -152,7 +160,6 @@ public boolean canBeDestroy(JobIdentity ji, ExecutorIdentifier ei) { return false; } - @Override protected void doDestroyInternal(ExecutorIdentifier identifier) throws JobException { long pid = Long.parseLong(identifier.getNamespace()); boolean result = SystemUtils.killProcessByPid(pid); @@ -165,7 +172,7 @@ protected void doDestroyInternal(ExecutorIdentifier identifier) throws JobExcept } @Override - protected boolean isExecutorExist(ExecutorIdentifier identifier) { + protected boolean isExecutorExist(ExecutorIdentifier identifier, ResourceID resourceID) { long pid = Long.parseLong(identifier.getNamespace()); boolean result = SystemUtils.isProcessRunning(pid, JobUtils.generateExecutorSelectorOnProcess(identifier.getExecutorName())); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/ResourceIDUtil.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/ResourceIDUtil.java new file mode 100644 index 0000000000..c22d3d7617 --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/ResourceIDUtil.java @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.task.caller; + +import java.util.Map; + +import com.oceanbase.odc.metadb.task.JobEntity; +import com.oceanbase.odc.service.resource.ResourceID; +import com.oceanbase.odc.service.resource.ResourceLocation; +import com.oceanbase.odc.service.task.resource.DefaultResourceOperatorBuilder; + +import lombok.extern.slf4j.Slf4j; + +/** + * util to help build resource id from job entity and executor identifier + * + * @author longpeng.zlp + * @date 2024/9/2 11:58 + */ +@Slf4j +public class ResourceIDUtil { + public static final String REGION_PROP_NAME = "regionName"; + public static final String GROUP_PROP_NAME = "cloudProvider"; + public static final String DEFAULT_PROP_VALUE = "local"; + + /** + * get with log is missing + * + * @param jobParameters + * @param propName + * @param defaultValue + * @return defaultValue if key is absent in jobParameters + */ + public static String checkAndGetJobProperties(Map jobParameters, String propName, + String defaultValue) { + if (null == jobParameters) { + log.warn("get propName={} failed from job context={} failed, use default value={}", propName, jobParameters, + defaultValue); + return defaultValue; + } + String ret = jobParameters.get(propName); + if (null == ret) { + log.warn("get propName={} failed from job context={} failed, use default value={}", propName, jobParameters, + defaultValue); + return defaultValue; + } else { + return ret; + } + } + + /** + * resolve resourceID with default region and group name + * + * @param executorIdentifier + * @param jobProperties + * @return + */ + public static ResourceID getResourceID(ExecutorIdentifier executorIdentifier, + Map jobProperties) { + String region = checkAndGetJobProperties(jobProperties, REGION_PROP_NAME, DEFAULT_PROP_VALUE); + String group = checkAndGetJobProperties(jobProperties, GROUP_PROP_NAME, DEFAULT_PROP_VALUE); + return new ResourceID(new ResourceLocation(region, group), DefaultResourceOperatorBuilder.CLOUD_K8S_POD_TYPE, + executorIdentifier.getNamespace(), + executorIdentifier.getExecutorName()); + } + + /** + * get resource id by jobEntity and executor identifier + * + * @param executorIdentifier + * @param jobEntity + * @return + */ + public static ResourceID getResourceID(ExecutorIdentifier executorIdentifier, JobEntity jobEntity) { + return getResourceID(executorIdentifier, jobEntity.getJobProperties()); + } +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/config/DefaultJobConfiguration.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/config/DefaultJobConfiguration.java index 61ebb60f01..3169842dae 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/config/DefaultJobConfiguration.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/config/DefaultJobConfiguration.java @@ -22,9 +22,8 @@ import com.oceanbase.odc.service.common.model.HostProperties; import com.oceanbase.odc.service.connection.ConnectionService; import com.oceanbase.odc.service.objectstorage.cloud.model.CloudEnvConfigurations; -import com.oceanbase.odc.service.schedule.ScheduleTaskService; +import com.oceanbase.odc.service.resource.ResourceManager; import com.oceanbase.odc.service.task.TaskService; -import com.oceanbase.odc.service.task.caller.K8sJobClientSelector; import com.oceanbase.odc.service.task.dispatch.JobDispatcher; import com.oceanbase.odc.service.task.jasypt.JasyptEncryptorConfigProperties; import com.oceanbase.odc.service.task.schedule.JobCredentialProvider; @@ -56,15 +55,13 @@ public abstract class DefaultJobConfiguration implements JobConfiguration { protected TaskService taskService; - protected ScheduleTaskService scheduleTaskService; - protected ConnectionService connectionService; protected JobDispatcher jobDispatcher; protected Scheduler daemonScheduler; - protected K8sJobClientSelector k8sJobClientSelector; + protected ResourceManager resourceManager; protected HostUrlProvider hostUrlProvider; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/config/DefaultSpringJobConfiguration.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/config/DefaultSpringJobConfiguration.java index a5638563ea..11745a7253 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/config/DefaultSpringJobConfiguration.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/config/DefaultSpringJobConfiguration.java @@ -27,9 +27,8 @@ import com.oceanbase.odc.service.common.model.HostProperties; import com.oceanbase.odc.service.connection.ConnectionService; import com.oceanbase.odc.service.objectstorage.cloud.model.CloudEnvConfigurations; -import com.oceanbase.odc.service.schedule.ScheduleTaskService; +import com.oceanbase.odc.service.resource.ResourceManager; import com.oceanbase.odc.service.task.TaskService; -import com.oceanbase.odc.service.task.caller.K8sJobClientSelector; import com.oceanbase.odc.service.task.dispatch.ImmediateJobDispatcher; import com.oceanbase.odc.service.task.jasypt.JasyptEncryptorConfigProperties; import com.oceanbase.odc.service.task.schedule.DefaultTaskFrameworkDisabledHandler; @@ -62,9 +61,9 @@ public void afterPropertiesSet() { setJobImageNameProvider(new DefaultJobImageNameProvider(this::getTaskFrameworkProperties)); setConnectionService(ctx.getBean(ConnectionService.class)); setTaskService(ctx.getBean(TaskService.class)); - setScheduleTaskService(ctx.getBean(ScheduleTaskService.class)); setDaemonScheduler((Scheduler) ctx.getBean("taskFrameworkSchedulerFactoryBean")); - setJobDispatcher(new ImmediateJobDispatcher()); + setJobDispatcher(new ImmediateJobDispatcher(ctx.getBean(ResourceManager.class))); + setResourceManager(ctx.getBean(ResourceManager.class)); LocalEventPublisher publisher = new LocalEventPublisher(); TaskFrameworkService tfs = ctx.getBean(TaskFrameworkService.class); if (tfs instanceof StdTaskFrameworkService) { @@ -92,10 +91,6 @@ public TaskFrameworkProperties getTaskFrameworkProperties() { return ctx.getBean(TaskFrameworkProperties.class); } - @Override - public K8sJobClientSelector getK8sJobClientSelector() { - return ctx.getBean(K8sJobClientSelector.class); - } private void initJobRateLimiter() { StartJobRateLimiterSupport limiterSupport = new StartJobRateLimiterSupport(); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/config/DefaultTaskFrameworkProperties.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/config/DefaultTaskFrameworkProperties.java index 766abd0386..e6fd835c54 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/config/DefaultTaskFrameworkProperties.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/config/DefaultTaskFrameworkProperties.java @@ -96,4 +96,8 @@ public class DefaultTaskFrameworkProperties implements TaskFrameworkProperties { private String destroyExecutorJobCronExpression; private String pullTaskResultJobCronExpression; + /** + * local k8s debug mode, use process builder mock k8s + */ + private boolean enableK8sLocalDebugMode; } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/config/JobConfiguration.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/config/JobConfiguration.java index 0736c0a235..9ff8d7be81 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/config/JobConfiguration.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/config/JobConfiguration.java @@ -22,9 +22,8 @@ import com.oceanbase.odc.service.common.model.HostProperties; import com.oceanbase.odc.service.connection.ConnectionService; import com.oceanbase.odc.service.objectstorage.cloud.model.CloudEnvConfigurations; -import com.oceanbase.odc.service.schedule.ScheduleTaskService; +import com.oceanbase.odc.service.resource.ResourceManager; import com.oceanbase.odc.service.task.TaskService; -import com.oceanbase.odc.service.task.caller.K8sJobClientSelector; import com.oceanbase.odc.service.task.dispatch.JobDispatcher; import com.oceanbase.odc.service.task.jasypt.JasyptEncryptorConfigProperties; import com.oceanbase.odc.service.task.schedule.JobCredentialProvider; @@ -48,9 +47,9 @@ public interface JobConfiguration { CloudEnvConfigurations getCloudEnvConfigurations(); - TaskService getTaskService(); + ResourceManager getResourceManager(); - ScheduleTaskService getScheduleTaskService(); + TaskService getTaskService(); ConnectionService getConnectionService(); @@ -58,8 +57,6 @@ public interface JobConfiguration { JobDispatcher getJobDispatcher(); - K8sJobClientSelector getK8sJobClientSelector(); - HostUrlProvider getHostUrlProvider(); TaskFrameworkService getTaskFrameworkService(); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/config/K8sProperties.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/config/K8sProperties.java index 64ad181206..e558152f10 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/config/K8sProperties.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/config/K8sProperties.java @@ -32,7 +32,6 @@ public class K8sProperties { private String kubeConfig; private String region; private String group; - /** * pod image name with version, odc job will be running in this image */ diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/config/TaskFrameworkConfiguration.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/config/TaskFrameworkConfiguration.java index c76dc30999..d35c11087b 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/config/TaskFrameworkConfiguration.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/config/TaskFrameworkConfiguration.java @@ -16,7 +16,6 @@ package com.oceanbase.odc.service.task.config; -import java.io.IOException; import java.util.Map; import java.util.concurrent.TimeUnit; @@ -31,13 +30,8 @@ import org.springframework.scheduling.quartz.SchedulerFactoryBean; import com.oceanbase.odc.common.event.EventPublisher; -import com.oceanbase.odc.common.util.StringUtils; import com.oceanbase.odc.service.common.ConditionOnServer; import com.oceanbase.odc.service.objectstorage.cloud.model.CloudEnvConfigurations; -import com.oceanbase.odc.service.task.caller.DefaultK8sJobClientSelector; -import com.oceanbase.odc.service.task.caller.K8sJobClientSelector; -import com.oceanbase.odc.service.task.caller.NativeK8sJobClient; -import com.oceanbase.odc.service.task.caller.NullK8sJobClientSelector; import com.oceanbase.odc.service.task.exception.JobException; import com.oceanbase.odc.service.task.jasypt.DefaultJasyptEncryptorConfigProperties; import com.oceanbase.odc.service.task.jasypt.JasyptEncryptorConfigProperties; @@ -69,22 +63,6 @@ public JobCredentialProvider jobCredentialProvider(CloudEnvConfigurations cloudE return new DefaultJobCredentialProvider(cloudEnvConfigurations); } - @Lazy - @Bean - @ConditionalOnMissingBean(K8sJobClientSelector.class) - public K8sJobClientSelector k8sJobClientSelector(@Autowired TaskFrameworkProperties taskFrameworkProperties) - throws IOException { - K8sProperties k8sProperties = taskFrameworkProperties.getK8sProperties(); - if (StringUtils.isBlank(k8sProperties.getKubeUrl())) { - log.info("local task k8s cluster is not enabled."); - return new NullK8sJobClientSelector(); - } - log.info("build k8sJobClientSelector, kubeUrl={}, namespace={}", - k8sProperties.getKubeUrl(), k8sProperties.getNamespace()); - NativeK8sJobClient nativeK8sJobClient = new NativeK8sJobClient(k8sProperties); - return new DefaultK8sJobClientSelector(nativeK8sJobClient); - } - @Bean @ConditionalOnBean(TaskFrameworkService.class) public StartJobRateLimiter monitorProcessRateLimiter(@Autowired TaskFrameworkService taskFrameworkService) { @@ -167,5 +145,4 @@ public EventPublisher getEventPublisher() { } }; } - } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/config/TaskFrameworkProperties.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/config/TaskFrameworkProperties.java index bda6d91b87..823a27e287 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/config/TaskFrameworkProperties.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/config/TaskFrameworkProperties.java @@ -72,4 +72,6 @@ public interface TaskFrameworkProperties { String getDestroyExecutorJobCronExpression(); + boolean isEnableK8sLocalDebugMode(); + } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/constants/JobConstants.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/constants/JobConstants.java index 3d1dc1690e..2d16737f6a 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/constants/JobConstants.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/constants/JobConstants.java @@ -56,6 +56,8 @@ public class JobConstants { public static final String ODC_SERVER_CLASS_NAME = "com.oceanbase.odc.server.OdcServer"; + public static final String ODC_AGENT_CLASS_NAME = "com.oceanbase.odc.agent.OdcAgent"; + public static final String ODC_EXECUTOR_DEFAULT_MOUNT_PATH = "/data/logs/odc-task/runtime"; public static final String ODC_EXECUTOR_PROCESS_PROPERTIES_KEY = "odc.executor"; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/dispatch/ImmediateJobDispatcher.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/dispatch/ImmediateJobDispatcher.java index a5c54e672d..bb1073383a 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/dispatch/ImmediateJobDispatcher.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/dispatch/ImmediateJobDispatcher.java @@ -22,11 +22,11 @@ import com.oceanbase.odc.common.util.StringUtils; import com.oceanbase.odc.common.util.SystemUtils; import com.oceanbase.odc.metadb.task.JobEntity; +import com.oceanbase.odc.service.resource.ResourceManager; import com.oceanbase.odc.service.task.caller.JobCaller; import com.oceanbase.odc.service.task.caller.JobCallerBuilder; import com.oceanbase.odc.service.task.caller.JobContext; -import com.oceanbase.odc.service.task.caller.K8sJobClient; -import com.oceanbase.odc.service.task.caller.PodConfig; +import com.oceanbase.odc.service.task.caller.JobEnvironmentFactory; import com.oceanbase.odc.service.task.config.JobConfiguration; import com.oceanbase.odc.service.task.config.JobConfigurationHolder; import com.oceanbase.odc.service.task.config.JobConfigurationValidator; @@ -36,6 +36,7 @@ import com.oceanbase.odc.service.task.constants.JobEnvKeyConstants; import com.oceanbase.odc.service.task.enums.TaskRunMode; import com.oceanbase.odc.service.task.exception.JobException; +import com.oceanbase.odc.service.task.resource.PodConfig; import com.oceanbase.odc.service.task.schedule.DefaultJobContextBuilder; import com.oceanbase.odc.service.task.schedule.JobIdentity; import com.oceanbase.odc.service.task.schedule.provider.JobImageNameProvider; @@ -50,6 +51,11 @@ * @since 4.2.4 */ public class ImmediateJobDispatcher implements JobDispatcher { + private final ResourceManager resourceManager; + + public ImmediateJobDispatcher(ResourceManager resourceManager) { + this.resourceManager = resourceManager; + } @Override public void start(JobContext context) throws JobException { @@ -64,21 +70,22 @@ public void stop(JobIdentity ji) throws JobException { } @Override - public void modify(JobIdentity ji, String jobParametersJson) throws JobException { + public void modify(JobIdentity ji, String jobParametersJson) + throws JobException { JobCaller jobCaller = getJobCaller(ji, null); jobCaller.modify(ji, jobParametersJson); } @Override - public void destroy(JobIdentity ji) throws JobException { + public void finish(JobIdentity ji) throws JobException { JobCaller jobCaller = getJobCaller(ji, null); - jobCaller.destroy(ji); + jobCaller.finish(ji); } @Override - public boolean canBeDestroy(JobIdentity ji) { + public boolean canBeFinish(JobIdentity ji) { JobCaller jobCaller = getJobCaller(ji, null); - return jobCaller.canBeDestroy(ji); + return jobCaller.canBeFinish(ji); } private JobCaller getJobCaller(JobIdentity ji, JobContext context) { @@ -92,7 +99,6 @@ private JobCaller getJobCaller(JobIdentity ji, JobContext context) { context = new DefaultJobContextBuilder().build(je); } if (je.getRunMode() == TaskRunMode.K8S) { - K8sJobClient k8sJobClient = config.getK8sJobClientSelector().select(context); PodConfig podConfig = createDefaultPodConfig(config.getTaskFrameworkProperties()); Map labels = JobPropertiesUtils.getLabels(context.getJobProperties()); podConfig.setLabels(labels); @@ -100,9 +106,11 @@ private JobCaller getJobCaller(JobIdentity ji, JobContext context) { if (StringUtils.isNotBlank(regionName)) { podConfig.setRegion(regionName); } - return JobCallerBuilder.buildK8sJobCaller(k8sJobClient, podConfig, context); + return JobCallerBuilder.buildK8sJobCaller(podConfig, context, resourceManager); + } else { + return JobCallerBuilder.buildProcessCaller(context, + new JobEnvironmentFactory().build(context, TaskRunMode.PROCESS)); } - return JobCallerBuilder.buildProcessCaller(context); } private PodConfig createDefaultPodConfig(TaskFrameworkProperties taskFrameworkProperties) { @@ -133,5 +141,4 @@ private PodConfig createDefaultPodConfig(TaskFrameworkProperties taskFrameworkPr podConfig.setPodPendingTimeoutSeconds(k8s.getPodPendingTimeoutSeconds()); return podConfig; } - } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/dummy/DummyTask.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/dummy/DummyTask.java new file mode 100644 index 0000000000..8d77096c38 --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/dummy/DummyTask.java @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.task.dummy; + +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; + +import com.oceanbase.odc.service.task.base.TaskBase; +import com.oceanbase.odc.service.task.caller.JobContext; + +import lombok.extern.slf4j.Slf4j; + +/** + * @author longpeng.zlp + * @date 2024/8/28 10:51 + */ +@Slf4j +public class DummyTask extends TaskBase { + + private AtomicBoolean stopped = new AtomicBoolean(false); + private AtomicLong loopCount = new AtomicLong(0); + private final long maxLoopCount = 1000000; + + public DummyTask() {} + + @Override + public double getProgress() { + return (loopCount.get() / maxLoopCount) * 100; + } + + @Override + public String getTaskResult() { + return "has loop count" + loopCount.get(); + } + + @Override + protected void doInit(JobContext context) throws Exception {} + + @Override + public boolean start() throws Exception { + while (!stopped.get() && loopCount.get() < maxLoopCount) { + Thread.sleep(1000); + log.info("dummy task loop for to {}", loopCount.get()); + } + return !stopped.get(); + } + + @Override + public void stop() throws Exception { + stopped.set(true); + } + + @Override + public void close() throws Exception { + stopped.set(true); + } +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/dummy/LocalMockK8sJobClient.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/dummy/LocalMockK8sJobClient.java new file mode 100644 index 0000000000..921ba8ab3b --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/dummy/LocalMockK8sJobClient.java @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.task.dummy; + +import java.util.Date; +import java.util.Optional; + +import com.oceanbase.odc.common.util.SystemUtils; +import com.oceanbase.odc.service.resource.ResourceState; +import com.oceanbase.odc.service.task.caller.DefaultExecutorIdentifier; +import com.oceanbase.odc.service.task.caller.DefaultJobContext; +import com.oceanbase.odc.service.task.caller.JobCallerBuilder; +import com.oceanbase.odc.service.task.caller.JobContext; +import com.oceanbase.odc.service.task.caller.ProcessJobCaller; +import com.oceanbase.odc.service.task.caller.ResourceIDUtil; +import com.oceanbase.odc.service.task.exception.JobException; +import com.oceanbase.odc.service.task.resource.DefaultResourceOperatorBuilder; +import com.oceanbase.odc.service.task.resource.K8sPodResource; +import com.oceanbase.odc.service.task.resource.K8sResourceContext; +import com.oceanbase.odc.service.task.resource.client.K8sJobClient; +import com.oceanbase.odc.service.task.resource.client.K8sJobClientSelector; +import com.oceanbase.odc.service.task.schedule.JobIdentity; + +/** + * use local process to mock k8s command + * + * @author longpeng.zlp + * @date 2024/8/28 11:18 + */ +public class LocalMockK8sJobClient implements K8sJobClientSelector { + @Override + public K8sJobClient select(String resourceGroup) { + return new LocalProcessClient(); + } + + private static final class LocalProcessClient implements K8sJobClient { + @Override + public K8sPodResource create(K8sResourceContext k8sResourceContext) throws JobException { + JobContext jobContext = getJobContext(k8sResourceContext.getExtraData()); + ProcessJobCaller jobCaller = (ProcessJobCaller) JobCallerBuilder.buildProcessCaller(jobContext, + JobCallerBuilder.buildK8sEnv(jobContext)); + DefaultExecutorIdentifier executorIdentifier = (DefaultExecutorIdentifier) jobCaller.doStart(jobContext); + return new K8sPodResource(k8sResourceContext.getRegion(), k8sResourceContext.getGroup(), + k8sResourceContext.type(), + executorIdentifier.getNamespace(), + executorIdentifier.getExecutorName(), ResourceState.AVAILABLE, + "127.0.0.1:" + executorIdentifier.getPort(), new Date(System.currentTimeMillis())); + } + + private JobContext getJobContext(Object extraData) { + JobContext ret = (JobContext) extraData; + if (null == ret) { + DefaultJobContext jobContext = new DefaultJobContext(); + jobContext.setJobClass(DummyTask.class.getName()); + JobIdentity jobEntity = new JobIdentity(); + jobEntity.setId(1L); + jobContext.setJobIdentity(jobEntity); + ret = jobContext; + } + return ret; + } + + + @Override + public Optional get(String namespace, String arn) throws JobException { + K8sPodResource ret = new K8sPodResource(ResourceIDUtil.REGION_PROP_NAME, + ResourceIDUtil.GROUP_PROP_NAME, DefaultResourceOperatorBuilder.CLOUD_K8S_POD_TYPE, + namespace, arn, ResourceState.AVAILABLE, + "127.0.0.1", new Date(System.currentTimeMillis())); + return Optional.of(ret); + } + + @Override + public String delete(String namespace, String arn) throws JobException { + long pid = Long.parseLong(namespace); + SystemUtils.killProcessByPid(pid); + return namespace + ":" + arn; + } + } +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/server/HeartbeatRequest.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/HeartbeatRequest.java similarity index 93% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/server/HeartbeatRequest.java rename to server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/HeartbeatRequest.java index b7535bc1f8..57798bc01d 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/server/HeartbeatRequest.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/HeartbeatRequest.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.task.executor.server; +package com.oceanbase.odc.service.task.executor; import com.oceanbase.odc.service.task.schedule.JobIdentity; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/task/TaskDescription.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/TaskDescription.java similarity index 100% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/task/TaskDescription.java rename to server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/TaskDescription.java diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/task/DefaultTaskResult.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/TaskResult.java similarity index 72% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/task/DefaultTaskResult.java rename to server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/TaskResult.java index 01ad0e8ced..df24042dfd 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/task/DefaultTaskResult.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/TaskResult.java @@ -13,12 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -package com.oceanbase.odc.service.task.executor.task; +package com.oceanbase.odc.service.task.executor; import java.util.Map; -import com.oceanbase.odc.service.task.enums.JobStatus; +import com.oceanbase.odc.common.util.MapUtils; +import com.oceanbase.odc.common.util.StringUtils; +import com.oceanbase.odc.core.shared.constant.TaskStatus; import com.oceanbase.odc.service.task.schedule.JobIdentity; import lombok.Data; @@ -29,11 +30,11 @@ * @since 4.2.4 */ @Data -public class DefaultTaskResult implements TaskResult { +public class TaskResult { private JobIdentity jobIdentity; - private JobStatus status; + private TaskStatus status; private String resultJson; @@ -45,7 +46,7 @@ public class DefaultTaskResult implements TaskResult { private Map logMetadata; - public boolean progressChanged(DefaultTaskResult previous) { + public boolean isProgressChanged(TaskResult previous) { if (previous == null) { return true; } @@ -55,13 +56,9 @@ public boolean progressChanged(DefaultTaskResult previous) { if (Double.compare(progress, previous.getProgress()) != 0) { return true; } - if (logMetadata != null && !logMetadata.equals(previous.getLogMetadata())) { - return true; - } - if (resultJson != null && !resultJson.equals(previous.getResultJson())) { + if (!MapUtils.isEqual(logMetadata, previous.logMetadata, String::equals)) { return true; } - return false; + return !StringUtils.equals(resultJson, previous.getResultJson()); } - } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/server/TraceDecoratorThreadFactory.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/TraceDecoratorThreadFactory.java similarity index 95% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/server/TraceDecoratorThreadFactory.java rename to server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/TraceDecoratorThreadFactory.java index 26a9ffb6e6..2b73cc40fe 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/server/TraceDecoratorThreadFactory.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/TraceDecoratorThreadFactory.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.task.executor.server; +package com.oceanbase.odc.service.task.executor; import java.util.concurrent.ThreadFactory; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/server/TraceDecoratorUtils.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/TraceDecoratorUtils.java similarity index 94% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/server/TraceDecoratorUtils.java rename to server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/TraceDecoratorUtils.java index e300f66d8f..e516918f82 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/server/TraceDecoratorUtils.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/TraceDecoratorUtils.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.task.executor.server; +package com.oceanbase.odc.service.task.executor; import com.oceanbase.odc.common.trace.TraceDecorator; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/task/BaseTask.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/task/BaseTask.java deleted file mode 100644 index c90350407a..0000000000 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/task/BaseTask.java +++ /dev/null @@ -1,208 +0,0 @@ -/* - * Copyright (c) 2023 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.oceanbase.odc.service.task.executor.task; - -import java.util.Collections; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicReference; - -import com.oceanbase.odc.service.objectstorage.cloud.CloudObjectStorageService; -import com.oceanbase.odc.service.objectstorage.cloud.model.ObjectStorageConfiguration; -import com.oceanbase.odc.service.task.caller.DefaultJobContext; -import com.oceanbase.odc.service.task.caller.JobContext; -import com.oceanbase.odc.service.task.enums.JobStatus; -import com.oceanbase.odc.service.task.executor.server.TaskMonitor; -import com.oceanbase.odc.service.task.util.CloudObjectStorageServiceBuilder; -import com.oceanbase.odc.service.task.util.JobUtils; - -import lombok.Getter; -import lombok.extern.slf4j.Slf4j; - -/** - * @author gaoda.xy - * @date 2023/11/22 20:16 - */ -@Slf4j -public abstract class BaseTask implements Task, ExceptionListener { - - private final AtomicBoolean closed = new AtomicBoolean(false); - private JobContext context; - private Map jobParameters; - private volatile JobStatus status = JobStatus.PREPARING; - private CloudObjectStorageService cloudObjectStorageService; - // only save latest exception if any - // it will be cleaned if been fetched - protected AtomicReference latestException = new AtomicReference<>(); - - @Getter - private TaskMonitor taskMonitor; - - @Override - public void start(TaskContext taskContext) { - this.context = taskContext.getJobContext(); - log.info("Start task, id={}.", context.getJobIdentity().getId()); - - this.jobParameters = Collections.unmodifiableMap(context.getJobParameters()); - log.info("Init task parameters success, id={}.", context.getJobIdentity().getId()); - - try { - initCloudObjectStorageService(); - } catch (Exception e) { - log.warn("Init cloud object storage service failed, id={}.", getJobId(), e); - } - - this.taskMonitor = createTaskMonitor(); - try { - doInit(context); - updateStatus(JobStatus.RUNNING); - taskMonitor.monitor(); - if (doStart(context, taskContext)) { - updateStatus(JobStatus.DONE); - } else { - updateStatus(JobStatus.FAILED); - } - } catch (Throwable e) { - log.warn("Task failed, id={}.", getJobId(), e); - updateStatus(JobStatus.FAILED); - taskContext.getExceptionListener().onException(e); - } finally { - close(); - } - } - - protected TaskMonitor createTaskMonitor() { - return new TaskMonitor(this, cloudObjectStorageService); - } - - @Override - public boolean stop() { - try { - if (getStatus().isTerminated()) { - log.warn("Task is already finished and cannot be canceled, id={}, status={}.", getJobId(), getStatus()); - } else { - doStop(); - // doRefresh cannot execute if update status to 'canceled'. - updateStatus(JobStatus.CANCELING); - } - return true; - } catch (Throwable e) { - log.warn("Stop task failed, id={}", getJobId(), e); - return false; - } finally { - close(); - } - } - - @Override - public boolean modify(Map jobParameters) { - if (Objects.isNull(jobParameters) || jobParameters.isEmpty()) { - log.warn("Job parameter cannot be null, id={}", getJobId()); - return false; - } - if (getStatus().isTerminated()) { - log.warn("Task is already finished, cannot modify parameters, id={}", getJobId()); - return false; - } - DefaultJobContext ctx = (DefaultJobContext) getJobContext(); - ctx.setJobParameters(jobParameters); - this.jobParameters = Collections.unmodifiableMap(jobParameters); - try { - afterModifiedJobParameters(); - } catch (Exception e) { - log.warn("Do after modified job parameters failed", e); - } - return true; - } - - private void initCloudObjectStorageService() { - Optional storageConfig = JobUtils.getObjectStorageConfiguration(); - storageConfig.ifPresent(osc -> this.cloudObjectStorageService = CloudObjectStorageServiceBuilder.build(osc)); - } - - private void close() { - if (closed.compareAndSet(false, true)) { - try { - doClose(); - } catch (Throwable e) { - // do nothing - } - log.info("Task completed, id={}, status={}.", getJobId(), getStatus()); - taskMonitor.finalWork(); - } - } - - protected CloudObjectStorageService getCloudObjectStorageService() { - return cloudObjectStorageService; - } - - @Override - public JobStatus getStatus() { - return status; - } - - @Override - public JobContext getJobContext() { - return context; - } - - protected void updateStatus(JobStatus status) { - log.info("Update task status, id={}, status={}.", getJobId(), status); - this.status = status; - } - - protected Map getJobParameters() { - return this.jobParameters; - } - - private Long getJobId() { - return getJobContext().getJobIdentity().getId(); - } - - protected abstract void doInit(JobContext context) throws Exception; - - /** - * start a task return succeed or failed after completed. - * - * @return return true if execute succeed, else return false - */ - protected abstract boolean doStart(JobContext context, TaskContext taskContext) throws Exception; - - protected abstract void doStop() throws Exception; - - /** - * task can release relational resource in this method - */ - protected abstract void doClose() throws Exception; - - protected void afterModifiedJobParameters() throws Exception { - // do nothing - } - - public Throwable getError() { - Throwable e = latestException.getAndSet(null); - log.info("retrieve exception = {}", null == e ? null : e.getMessage()); - return e; - } - - public void onException(Throwable e) { - log.info("found exception", e); - this.latestException.set(e); - } -} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/task/DataArchiveTask.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/task/DataArchiveTask.java deleted file mode 100644 index a465abeb50..0000000000 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/executor/task/DataArchiveTask.java +++ /dev/null @@ -1,225 +0,0 @@ -/* - * Copyright (c) 2023 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.oceanbase.odc.service.task.executor.task; - -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Date; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; - -import com.oceanbase.odc.common.json.JsonUtils; -import com.oceanbase.odc.core.shared.constant.TaskStatus; -import com.oceanbase.odc.service.dlm.DLMJobFactory; -import com.oceanbase.odc.service.dlm.DLMJobStore; -import com.oceanbase.odc.service.dlm.DLMTableStructureSynchronizer; -import com.oceanbase.odc.service.dlm.DataSourceInfoMapper; -import com.oceanbase.odc.service.dlm.model.DlmTableUnit; -import com.oceanbase.odc.service.dlm.model.DlmTableUnitParameters; -import com.oceanbase.odc.service.dlm.utils.DlmJobIdUtil; -import com.oceanbase.odc.service.schedule.job.DLMJobReq; -import com.oceanbase.odc.service.schedule.model.DlmTableUnitStatistic; -import com.oceanbase.odc.service.task.caller.JobContext; -import com.oceanbase.odc.service.task.constants.JobParametersKeyConstants; -import com.oceanbase.odc.service.task.util.JobUtils; -import com.oceanbase.tools.migrator.common.enums.JobType; -import com.oceanbase.tools.migrator.job.Job; -import com.oceanbase.tools.migrator.task.CheckMode; - -import lombok.extern.slf4j.Slf4j; - -/** - * @Author：tinker - * @Date: 2024/1/24 11:09 - * @Descripition: - */ - -@Slf4j -public class DataArchiveTask extends BaseTask> { - - private DLMJobFactory jobFactory; - private DLMJobStore jobStore; - private double progress = 0.0; - private Job job; - private Map result; - private boolean isToStop = false; - - - @Override - protected void doInit(JobContext context) { - jobStore = new DLMJobStore(JobUtils.getMetaDBConnectionConfig()); - jobFactory = new DLMJobFactory(jobStore); - log.info("Init data-archive job env succeed,jobIdentity={}", context.getJobIdentity()); - } - - @Override - protected boolean doStart(JobContext context, TaskContext taskContext) throws Exception { - - jobStore.setJobParameters(getJobParameters()); - DLMJobReq parameters = - JsonUtils.fromJson(getJobParameters().get(JobParametersKeyConstants.META_TASK_PARAMETER_JSON), - DLMJobReq.class); - if (parameters.getFireTime() == null) { - parameters.setFireTime(new Date()); - } - try { - result = getDlmTableUnits(parameters).stream() - .collect(Collectors.toMap(DlmTableUnit::getDlmTableUnitId, o -> o)); - jobStore.setDlmTableUnits(result); - } catch (Exception e) { - log.warn("Get dlm job failed!", e); - taskContext.getExceptionListener().onException(e); - return false; - } - Set dlmTableUnitIds = result.keySet(); - - for (String dlmTableUnitId : dlmTableUnitIds) { - DlmTableUnit dlmTableUnit = result.get(dlmTableUnitId); - if (isToStop) { - log.info("Job is terminated,jobIdentity={}", context.getJobIdentity()); - break; - } - if (dlmTableUnit.getStatus() == TaskStatus.DONE) { - log.info("The table had been completed,tableName={}", dlmTableUnit.getTableName()); - continue; - } - startTableUnit(dlmTableUnitId); - if (parameters.getJobType() == JobType.MIGRATE) { - try { - DLMTableStructureSynchronizer.sync( - DataSourceInfoMapper.toConnectionConfig(parameters.getSourceDs()), - DataSourceInfoMapper.toConnectionConfig(parameters.getTargetDs()), - dlmTableUnit.getTableName(), dlmTableUnit.getTargetTableName(), - parameters.getSyncTableStructure()); - } catch (Exception e) { - log.warn("Failed to sync target table structure,table will be ignored,tableName={}", - dlmTableUnit.getTableName(), e); - if (!parameters.getSyncTableStructure().isEmpty()) { - finishTableUnit(dlmTableUnitId, TaskStatus.FAILED); - continue; - } - } - } - try { - job = jobFactory.createJob(dlmTableUnit); - log.info("Init {} job succeed,DLMJobId={}", job.getJobMeta().getJobType(), job.getJobMeta().getJobId()); - log.info("{} job start,DLMJobId={}", job.getJobMeta().getJobType(), job.getJobMeta().getJobId()); - if (isToStop) { - finishTableUnit(dlmTableUnitId, TaskStatus.CANCELED); - job.stop(); - log.info("The task has stopped."); - break; - } else { - job.run(); - } - log.info("{} job finished,DLMJobId={}", dlmTableUnit.getType(), dlmTableUnitId); - finishTableUnit(dlmTableUnitId, TaskStatus.DONE); - } catch (Throwable e) { - log.error("{} job failed,DLMJobId={},errorMsg={}", dlmTableUnit.getType(), dlmTableUnitId, e); - // set task status to failed if any job failed. - if (job != null && job.getJobMeta().isToStop()) { - finishTableUnit(dlmTableUnitId, TaskStatus.CANCELED); - } else { - finishTableUnit(dlmTableUnitId, TaskStatus.FAILED); - taskContext.getExceptionListener().onException(e); - } - } - } - return true; - } - - private void startTableUnit(String dlmTableUnitId) { - result.get(dlmTableUnitId).setStatus(TaskStatus.RUNNING); - result.get(dlmTableUnitId).setStartTime(new Date()); - } - - private void finishTableUnit(String dlmTableUnitId, TaskStatus status) { - result.get(dlmTableUnitId).setStatus(status); - result.get(dlmTableUnitId).setEndTime(new Date()); - } - - private List getDlmTableUnits(DLMJobReq req) throws SQLException { - List dlmTableUnits = new LinkedList<>(); - req.getTables().forEach(table -> { - DlmTableUnit dlmTableUnit = new DlmTableUnit(); - dlmTableUnit.setScheduleTaskId(req.getScheduleTaskId()); - DlmTableUnitParameters jobParameter = new DlmTableUnitParameters(); - jobParameter.setMigrateRule(table.getConditionExpression()); - jobParameter.setCheckMode(CheckMode.MULTIPLE_GET); - jobParameter.setReaderBatchSize(req.getRateLimit().getBatchSize()); - jobParameter.setWriterBatchSize(req.getRateLimit().getBatchSize()); - jobParameter.setMigrationInsertAction(req.getMigrationInsertAction()); - jobParameter.setMigratePartitions(table.getPartitions()); - jobParameter.setSyncDBObjectType(req.getSyncTableStructure()); - jobParameter.setShardingStrategy(req.getShardingStrategy()); - dlmTableUnit.setParameters(jobParameter); - dlmTableUnit.setDlmTableUnitId(DlmJobIdUtil.generateHistoryJobId(req.getJobName(), req.getJobType().name(), - req.getScheduleTaskId(), dlmTableUnits.size())); - dlmTableUnit.setTableName(table.getTableName()); - dlmTableUnit.setTargetTableName(table.getTargetTableName()); - dlmTableUnit.setSourceDatasourceInfo(req.getSourceDs()); - dlmTableUnit.setTargetDatasourceInfo(req.getTargetDs()); - dlmTableUnit.setFireTime(req.getFireTime()); - dlmTableUnit.setStatus(TaskStatus.PREPARING); - dlmTableUnit.setType(req.getJobType()); - dlmTableUnit.setStatistic(new DlmTableUnitStatistic()); - dlmTableUnits.add(dlmTableUnit); - }); - return dlmTableUnits; - } - - @Override - protected void doStop() throws Exception { - isToStop = true; - if (job != null) { - try { - job.stop(); - result.forEach((k, v) -> { - if (!v.getStatus().isTerminated()) { - v.setStatus(TaskStatus.CANCELED); - } - }); - } catch (Exception e) { - log.warn("Update dlm table unit status failed,DlmTableUnitId={}", job.getJobMeta().getJobId()); - } - } - } - - @Override - protected void doClose() throws Exception { - jobStore.destroy(); - } - - @Override - protected void afterModifiedJobParameters() throws Exception { - if (jobStore != null) { - jobStore.setJobParameters(getJobParameters()); - } - } - - @Override - public double getProgress() { - return progress; - } - - @Override - public List getTaskResult() { - return new ArrayList<>(result.values()); - } -} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/listener/DefaultJobProcessUpdateEvent.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/listener/DefaultJobProcessUpdateEvent.java index 4bc8adf801..a7cb8ad49a 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/listener/DefaultJobProcessUpdateEvent.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/listener/DefaultJobProcessUpdateEvent.java @@ -16,7 +16,7 @@ package com.oceanbase.odc.service.task.listener; import com.oceanbase.odc.common.event.AbstractEvent; -import com.oceanbase.odc.service.task.executor.task.TaskResult; +import com.oceanbase.odc.service.task.executor.TaskResult; import lombok.Getter; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/listener/DefaultJobProcessUpdateListener.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/listener/DefaultJobProcessUpdateListener.java index 5651c1e43a..59e9780f73 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/listener/DefaultJobProcessUpdateListener.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/listener/DefaultJobProcessUpdateListener.java @@ -25,7 +25,7 @@ import com.oceanbase.odc.core.shared.constant.TaskStatus; import com.oceanbase.odc.metadb.task.JobEntity; import com.oceanbase.odc.service.schedule.ScheduleTaskService; -import com.oceanbase.odc.service.task.executor.task.TaskResult; +import com.oceanbase.odc.service.task.executor.TaskResult; import com.oceanbase.odc.service.task.schedule.JobIdentity; import com.oceanbase.odc.service.task.service.TaskFrameworkService; @@ -56,7 +56,6 @@ public void onEvent(DefaultJobProcessUpdateEvent event) { updateScheduleTaskStatus(taskEntity.getId(), TaskStatus.RUNNING, TaskStatus.PREPARING); } }); - } private void updateScheduleTaskStatus(Long id, TaskStatus status, TaskStatus previousStatus) { @@ -68,7 +67,6 @@ private void updateScheduleTaskStatus(Long id, TaskStatus status, TaskStatus pre log.warn("Update scheduleTask status from {} to {} failed, scheduleTaskId={}", previousStatus, status, id); } - } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/processor/LoadDataResultProcessor.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/processor/LoadDataResultProcessor.java deleted file mode 100644 index c0277ccff4..0000000000 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/processor/LoadDataResultProcessor.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright (c) 2023 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.oceanbase.odc.service.task.processor; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - -import com.oceanbase.odc.metadb.task.JobRepository; -import com.oceanbase.odc.service.task.executor.task.TaskDescription; -import com.oceanbase.odc.service.task.executor.task.TaskResult; -import com.oceanbase.odc.service.task.processor.result.ResultProcessor; - -import lombok.extern.slf4j.Slf4j; - -/** - * @author: liuyizhuo.lyz - * @date: 2024/10/21 - */ -@Slf4j -@Component -public class LoadDataResultProcessor implements ResultProcessor { - - @Autowired - private JobRepository jobRepository; - - @Override - public void process(TaskResult result) { - try { - jobRepository.updateResultJson(result.getResultJson(), result.getJobIdentity().getId()); - log.info("Refresh loaddata task result successfully, jobIdentity={}", result.getJobIdentity()); - } catch (Exception e) { - log.error("Refresh loaddata task result failed, jobIdentity={}", result.getJobIdentity(), e); - } - } - - @Override - public boolean interested(String type) { - return TaskDescription.LOAD_DATA.matched(type); - } -} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/processor/result/DLMResultProcessor.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/processor/result/DLMResultProcessor.java index 343bbe2b9d..50c428e1bb 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/processor/result/DLMResultProcessor.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/processor/result/DLMResultProcessor.java @@ -25,7 +25,7 @@ import com.oceanbase.odc.service.dlm.DLMService; import com.oceanbase.odc.service.dlm.model.DlmTableUnit; import com.oceanbase.odc.service.schedule.ScheduleTaskService; -import com.oceanbase.odc.service.task.executor.task.TaskResult; +import com.oceanbase.odc.service.task.executor.TaskResult; import com.oceanbase.odc.service.task.processor.matcher.DLMProcessorMatcher; import lombok.extern.slf4j.Slf4j; @@ -48,7 +48,7 @@ public class DLMResultProcessor extends DLMProcessorMatcher implements ResultPro @Override public void process(TaskResult result) { - log.info("Start refresh result,result={}", result.getResultJson()); + log.info("Start refresh result,jobIdentity={}", result.getJobIdentity()); try { List dlmTableUnits = JsonUtils.fromJson(result.getResultJson(), new TypeReference>() {}); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/processor/result/LogicalDBChangeResultProcessor.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/processor/result/LogicalDBChangeResultProcessor.java index eb48bdd754..33f9c12a87 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/processor/result/LogicalDBChangeResultProcessor.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/processor/result/LogicalDBChangeResultProcessor.java @@ -32,7 +32,7 @@ import com.oceanbase.odc.service.connection.logicaldatabase.core.executor.sql.SqlExecutionResultWrapper; import com.oceanbase.odc.service.connection.logicaldatabase.core.model.LogicalDBChangeExecutionUnit; import com.oceanbase.odc.service.schedule.ScheduleTaskService; -import com.oceanbase.odc.service.task.executor.task.TaskResult; +import com.oceanbase.odc.service.task.executor.TaskResult; import com.oceanbase.odc.service.task.processor.matcher.LogicalDBChangeProcessorMatcher; import lombok.extern.slf4j.Slf4j; diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/processor/result/ResultProcessor.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/processor/result/ResultProcessor.java index b882a0eeaa..6c78763b79 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/processor/result/ResultProcessor.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/processor/result/ResultProcessor.java @@ -15,7 +15,7 @@ */ package com.oceanbase.odc.service.task.processor.result; -import com.oceanbase.odc.service.task.executor.task.TaskResult; +import com.oceanbase.odc.service.task.executor.TaskResult; import com.oceanbase.odc.service.task.processor.ProcessorMatcher; /** diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/resource/k8s/DefaultResourceOperatorBuilder.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/resource/DefaultResourceOperatorBuilder.java similarity index 57% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/resource/k8s/DefaultResourceOperatorBuilder.java rename to server/odc-service/src/main/java/com/oceanbase/odc/service/task/resource/DefaultResourceOperatorBuilder.java index 8d3afed440..16004d4213 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/resource/k8s/DefaultResourceOperatorBuilder.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/resource/DefaultResourceOperatorBuilder.java @@ -13,37 +13,81 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.resource.k8s; +package com.oceanbase.odc.service.task.resource; +import java.io.IOException; import java.util.Optional; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + import com.oceanbase.odc.common.util.StringUtils; import com.oceanbase.odc.metadb.resource.ResourceEntity; import com.oceanbase.odc.metadb.resource.ResourceRepository; import com.oceanbase.odc.service.resource.ResourceID; import com.oceanbase.odc.service.resource.ResourceLocation; import com.oceanbase.odc.service.resource.ResourceOperatorBuilder; -import com.oceanbase.odc.service.resource.k8s.client.K8sJobClientSelector; +import com.oceanbase.odc.service.task.config.K8sProperties; +import com.oceanbase.odc.service.task.config.TaskFrameworkProperties; +import com.oceanbase.odc.service.task.dummy.LocalMockK8sJobClient; +import com.oceanbase.odc.service.task.resource.client.DefaultK8sJobClientSelector; +import com.oceanbase.odc.service.task.resource.client.K8sJobClientSelector; +import com.oceanbase.odc.service.task.resource.client.NativeK8sJobClient; +import com.oceanbase.odc.service.task.resource.client.NullK8sJobClientSelector; -import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; /** * default impl for k8s resource operator - * + * * @author longpeng.zlp * @date 2024/9/2 17:33 */ -@AllArgsConstructor +@Component +@Slf4j public class DefaultResourceOperatorBuilder implements ResourceOperatorBuilder { public static final String CLOUD_K8S_POD_TYPE = "cloudK8sPod"; - private final K8sJobClientSelector k8sJobClientSelector; - private final long podPendingTimeoutSeconds; - private final ResourceRepository resourceRepository; + protected K8sJobClientSelector k8sJobClientSelector; + protected K8sProperties k8sProperties; + protected ResourceRepository resourceRepository; + + public DefaultResourceOperatorBuilder(@Autowired TaskFrameworkProperties taskFrameworkProperties, + @Autowired ResourceRepository resourceRepository) throws IOException { + this.k8sProperties = taskFrameworkProperties.getK8sProperties(); + this.resourceRepository = resourceRepository; + this.k8sJobClientSelector = buildK8sJobSelector(taskFrameworkProperties); + } + + /** + * build k8s job selector + */ + protected K8sJobClientSelector buildK8sJobSelector( + TaskFrameworkProperties taskFrameworkProperties) throws IOException { + K8sProperties k8sProperties = taskFrameworkProperties.getK8sProperties(); + K8sJobClientSelector k8sJobClientSelector; + if (taskFrameworkProperties.isEnableK8sLocalDebugMode()) { + // k8s use in local debug mode + log.info("local debug k8s cluster enabled."); + k8sJobClientSelector = new LocalMockK8sJobClient(); + } else if (StringUtils.isBlank(k8sProperties.getKubeUrl())) { + log.info("local task k8s cluster is not enabled."); + k8sJobClientSelector = new NullK8sJobClientSelector(); + } else { + // normal mode + log.info("build k8sJobClientSelector, kubeUrl={}, namespace={}", + k8sProperties.getKubeUrl(), k8sProperties.getNamespace()); + NativeK8sJobClient nativeK8sJobClient = new NativeK8sJobClient(k8sProperties); + k8sJobClientSelector = new DefaultK8sJobClientSelector(nativeK8sJobClient); + } + return k8sJobClientSelector; + } + + @Override public K8sResourceOperator build(ResourceLocation resourceLocation) { return new K8sResourceOperator(new K8sResourceOperatorContext(k8sJobClientSelector, - this::getResourceCreateTimeInSeconds, podPendingTimeoutSeconds)); + this::getResourceCreateTimeInSeconds, k8sProperties.getPodPendingTimeoutSeconds())); } /** @@ -95,7 +139,7 @@ public K8sPodResource toResource(ResourceEntity resourceEntity, Optional jobClass) { TriggerConfig config = new TriggerConfig(); config.setTriggerStrategy(TriggerStrategy.CRON); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/schedule/daemon/CheckRunningJob.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/schedule/daemon/CheckRunningJob.java index 309c05462f..af71061231 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/schedule/daemon/CheckRunningJob.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/schedule/daemon/CheckRunningJob.java @@ -26,6 +26,7 @@ import org.springframework.data.domain.Page; import com.oceanbase.odc.common.util.SilentExecutor; +import com.oceanbase.odc.common.util.StringUtils; import com.oceanbase.odc.core.alarm.AlarmEventNames; import com.oceanbase.odc.core.alarm.AlarmUtils; import com.oceanbase.odc.metadb.task.JobEntity; @@ -79,76 +80,102 @@ private void handleJobRetryingOrFailed(JobEntity jobEntity) { private void doHandleJobRetryingOrFailed(JobEntity jobEntity) { log.info("Start to handle heartbeat timeout job, jobId={}.", jobEntity.getId()); TaskFrameworkService taskFrameworkService = getConfiguration().getTaskFrameworkService(); - JobEntity a = taskFrameworkService.findWithPessimisticLock(jobEntity.getId()); - - if (a.getStatus() != JobStatus.RUNNING) { - log.warn("Current job is not RUNNING, abort continue, jobId={}.", a.getId()); + // query from db + JobEntity refreshedJobEntity = taskFrameworkService.findWithPessimisticLock(jobEntity.getId()); + // TODO(lx): confirm this logic why job not in running status not handle + if (refreshedJobEntity.getStatus() != JobStatus.RUNNING) { + log.warn("Current job is not RUNNING, abort continue, jobId={}.", refreshedJobEntity.getId()); return; } - boolean isNeedRetry = checkJobIfRetryNecessary(a); + try { + tryFinishJob(taskFrameworkService, refreshedJobEntity); + } finally { + // mark resource as released to let resource collector collect resource + if (TaskRunMode.K8S == refreshedJobEntity.getRunMode()) { + ResourceManagerUtil.markResourceReleased(refreshedJobEntity, refreshedJobEntity.getExecutorIdentifier(), + getConfiguration().getResourceManager()); + log.info("CheckRunningJob release resource for job = {}", jobEntity); + } + } + } + + private void tryFinishJob(TaskFrameworkService taskFrameworkService, JobEntity jobEntity) { + boolean isNeedRetry = checkJobIfRetryNecessary(jobEntity); if (isNeedRetry) { log.info("Need to restart job, try to set status to RETRYING, jobId={}, oldStatus={}.", - a.getId(), a.getStatus()); + jobEntity.getId(), jobEntity.getStatus()); int rows; - if (TaskRunMode.K8S == a.getRunMode()) { - rows = taskFrameworkService.updateExecutorEndpoint(a.getId(), null); - log.info("Clear executor endpoint why retry task, jobId={}, rows={}", a.getId(), rows); + if (TaskRunMode.K8S == jobEntity.getRunMode()) { + rows = taskFrameworkService.updateExecutorEndpoint(jobEntity.getId(), null); + log.info("Clear executor endpoint why retry task, jobId={}, rows={}", jobEntity.getId(), rows); } rows = taskFrameworkService - .updateStatusDescriptionByIdOldStatus(a.getId(), JobStatus.RUNNING, + .updateStatusDescriptionByIdOldStatus(jobEntity.getId(), JobStatus.RUNNING, JobStatus.RETRYING, "Heart timeout and retrying job"); if (rows > 0) { - log.info("Set job status to RETRYING, jobId={}, oldStatus={}.", a.getId(), a.getStatus()); + log.info("Set job status to RETRYING, jobId={}, oldStatus={}.", jobEntity.getId(), + jobEntity.getStatus()); } else { throw new TaskRuntimeException("Set job status to RETRYING failed, jobId=" + jobEntity.getId()); } } else { log.info("No need to restart job, try to set status to FAILED, jobId={},oldStatus={}.", - a.getId(), a.getStatus()); + jobEntity.getId(), jobEntity.getStatus()); TaskFrameworkProperties taskFrameworkProperties = getConfiguration().getTaskFrameworkProperties(); int rows = taskFrameworkService - .updateStatusToFailedWhenHeartTimeout(a.getId(), + .updateStatusToFailedWhenHeartTimeout(jobEntity.getId(), taskFrameworkProperties.getJobHeartTimeoutSeconds(), "Heart timeout and set job to status FAILED."); if (rows > 0) { - log.info("Set job status to FAILED accomplished, jobId={}, oldStatus={}.", a.getId(), a.getStatus()); + log.info("Set job status to FAILED accomplished, jobId={}, oldStatus={}.", jobEntity.getId(), + jobEntity.getStatus()); Map eventMessage = AlarmUtils.createAlarmMapBuilder() .item(AlarmUtils.ORGANIZATION_NAME, Optional.ofNullable(jobEntity.getOrganizationId()).map( Object::toString).orElse(StrUtil.EMPTY)) - .item(AlarmUtils.TASK_JOB_ID_NAME, jobEntity.getId().toString()) + .item(AlarmUtils.TASK_JOB_ID_NAME, String.valueOf(jobEntity.getId())) .item(AlarmUtils.MESSAGE_NAME, - MessageFormat.format("Job running failed due to heart timeout, jobId={0}", a.getId())) + MessageFormat.format("Job running failed due to heart timeout, jobId={0}", + jobEntity.getId())) .build(); AlarmUtils.alarm(AlarmEventNames.TASK_HEARTBEAT_TIMEOUT, eventMessage); } else { throw new TaskRuntimeException("Set job status to FAILED failed, jobId=" + jobEntity.getId()); } } - if (!getConfiguration().getJobDispatcher().canBeDestroy(JobIdentity.of(a.getId()))) { - log.info("Cannot destroy executor, jobId={}.", a.getId()); + if (!getConfiguration().getJobDispatcher().canBeFinish(JobIdentity.of(jobEntity.getId()))) { + log.info("Cannot destroy executor, jobId={}.", jobEntity.getId()); throw new TaskRuntimeException("Cannot destroy executor, jobId={}" + jobEntity.getId()); } // First try to stop remote job try { - log.info("Try to stop remote job, jobId={}.", a.getId()); - getConfiguration().getJobDispatcher().stop(JobIdentity.of(a.getId())); + log.info("Try to stop remote job, jobId={}.", jobEntity.getId()); + if (StringUtils.isEmpty(jobEntity.getExecutorIdentifier())) { + log.info("found invalid job = {}, resource destroy not confirmed, set status to failed", jobEntity); + taskFrameworkService + .updateStatusDescriptionByIdOldStatus(jobEntity.getId(), JobStatus.RUNNING, + JobStatus.FAILED, "old job not determinate resource has created"); + return; + } + getConfiguration().getJobDispatcher().stop(JobIdentity.of(jobEntity.getId())); } catch (JobException e) { // Process will continue if stop failed and not rollback transaction - log.warn("Try to stop remote failed, jobId={}.", a.getId(), e); + log.warn("Try to stop remote failed, jobId={}.", jobEntity.getId(), e); } - // Second destroy executor + // Second finish job and clean it all try { - log.info("Try to destroy executor, jobId={}.", a.getId()); - getConfiguration().getJobDispatcher().destroy(JobIdentity.of(a.getId())); + log.info("Try to destroy executor, jobId={}.", jobEntity.getId()); + getConfiguration().getJobDispatcher().finish(JobIdentity.of(jobEntity.getId())); } catch (JobException e) { throw new TaskRuntimeException(e); } if (!isNeedRetry) { + // set status to destroyed + taskFrameworkService.updateExecutorToDestroyed(jobEntity.getId()); getConfiguration().getEventPublisher().publishEvent( - new JobTerminateEvent(JobIdentity.of(a.getId()), JobStatus.FAILED)); + new JobTerminateEvent(JobIdentity.of(jobEntity.getId()), JobStatus.FAILED)); } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/schedule/daemon/DestroyExecutorJob.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/schedule/daemon/DestroyExecutorJob.java index 606fcf1b1a..0e9ae03fd4 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/schedule/daemon/DestroyExecutorJob.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/schedule/daemon/DestroyExecutorJob.java @@ -78,7 +78,7 @@ private void destroyExecutor(TaskFrameworkService taskFrameworkService, JobEntit log.info("Job prepare destroy executor, jobId={},status={}.", lockedEntity.getId(), lockedEntity.getStatus()); try { - getConfiguration().getJobDispatcher().destroy(JobIdentity.of(lockedEntity.getId())); + getConfiguration().getJobDispatcher().finish(JobIdentity.of(lockedEntity.getId())); } catch (JobException e) { log.warn("Destroy executor occur error, jobId={}: ", lockedEntity.getId(), e); if (e.getMessage() != null && @@ -87,7 +87,7 @@ private void destroyExecutor(TaskFrameworkService taskFrameworkService, JobEntit .item(AlarmUtils.ORGANIZATION_NAME, Optional.ofNullable(jobEntity.getOrganizationId()).map( Object::toString).orElse(StrUtil.EMPTY)) - .item(AlarmUtils.TASK_JOB_ID_NAME, jobEntity.getId().toString()) + .item(AlarmUtils.TASK_JOB_ID_NAME, String.valueOf(jobEntity.getId())) .item(AlarmUtils.MESSAGE_NAME, MessageFormat.format("Job executor destroy failed, jobId={0}, message={1}", lockedEntity.getId(), e.getMessage())) diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/schedule/daemon/DestroyResourceJob.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/schedule/daemon/DestroyResourceJob.java new file mode 100644 index 0000000000..36075d4746 --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/schedule/daemon/DestroyResourceJob.java @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.task.schedule.daemon; + +import java.text.MessageFormat; +import java.util.Map; + +import org.quartz.DisallowConcurrentExecution; +import org.quartz.Job; +import org.quartz.JobExecutionContext; +import org.quartz.JobExecutionException; +import org.springframework.data.domain.Page; + +import com.oceanbase.odc.core.alarm.AlarmEventNames; +import com.oceanbase.odc.core.alarm.AlarmUtils; +import com.oceanbase.odc.metadb.resource.ResourceEntity; +import com.oceanbase.odc.service.resource.ResourceID; +import com.oceanbase.odc.service.resource.ResourceLocation; +import com.oceanbase.odc.service.task.config.JobConfiguration; +import com.oceanbase.odc.service.task.config.JobConfigurationHolder; +import com.oceanbase.odc.service.task.config.JobConfigurationValidator; +import com.oceanbase.odc.service.task.config.TaskFrameworkProperties; +import com.oceanbase.odc.service.task.constants.JobConstants; +import com.oceanbase.odc.service.task.exception.TaskRuntimeException; +import com.oceanbase.odc.service.task.service.TaskFrameworkService; + +import lombok.extern.slf4j.Slf4j; + +/** + * @author yaobin + * @date 2024-01-22 + * @since 4.2.4 + */ +@Slf4j +@DisallowConcurrentExecution +public class DestroyResourceJob implements Job { + + private JobConfiguration configuration; + + @Override + public void execute(JobExecutionContext context) throws JobExecutionException { + configuration = JobConfigurationHolder.getJobConfiguration(); + JobConfigurationValidator.validComponent(); + + // scan terminate job + TaskFrameworkService taskFrameworkService = configuration.getTaskFrameworkService(); + TaskFrameworkProperties taskFrameworkProperties = configuration.getTaskFrameworkProperties(); + Page resources = taskFrameworkService.findAbandonedResource(0, + taskFrameworkProperties.getSingleFetchDestroyExecutorJobRows()); + resources.forEach(a -> { + try { + destroyResource(a); + } catch (Throwable e) { + log.warn("Try to destroy failed, jobId={}.", a.getId(), e); + } + }); + } + + private void destroyResource(ResourceEntity resourceEntity) { + getConfiguration().getTransactionManager().doInTransactionWithoutResult(() -> { + ResourceID resourceID = new ResourceID(new ResourceLocation(resourceEntity.getRegion(), + resourceEntity.getGroupName()), resourceEntity.getResourceType(), resourceEntity.getNamespace(), + resourceEntity.getResourceName()); + try { + configuration.getResourceManager().destroy(resourceID); + } catch (Throwable e) { + log.warn("DestroyResourceJob destroy resource = {} failed", resourceEntity, e); + if (e.getMessage() != null && + !e.getMessage().startsWith(JobConstants.ODC_EXECUTOR_CANNOT_BE_DESTROYED)) { + Map eventMessage = AlarmUtils.createAlarmMapBuilder() + .item(AlarmUtils.ORGANIZATION_NAME, AlarmUtils.ODC_RESOURCE) + .item(AlarmUtils.RESOURCE_ID_NAME, String.valueOf(resourceEntity.getId())) + .item(AlarmUtils.RESOURCE_TYPE, resourceEntity.getResourceType()) + .item(AlarmUtils.MESSAGE_NAME, + MessageFormat.format("Job resource destroy failed, resourceID={0}, message={1}", + resourceEntity.getId(), e.getMessage())) + .build(); + AlarmUtils.alarm(AlarmEventNames.TASK_EXECUTOR_DESTROY_FAILED, eventMessage); + } + throw new TaskRuntimeException(e); + } + log.info("Job destroy resource succeed, resource={}", resourceEntity); + }); + } + + private JobConfiguration getConfiguration() { + return configuration; + } +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/schedule/daemon/DoCancelingJob.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/schedule/daemon/DoCancelingJob.java index 9342f73a3e..831326cfb1 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/schedule/daemon/DoCancelingJob.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/schedule/daemon/DoCancelingJob.java @@ -27,6 +27,7 @@ import com.oceanbase.odc.service.task.config.JobConfigurationValidator; import com.oceanbase.odc.service.task.config.TaskFrameworkProperties; import com.oceanbase.odc.service.task.enums.JobStatus; +import com.oceanbase.odc.service.task.enums.TaskRunMode; import com.oceanbase.odc.service.task.exception.TaskRuntimeException; import com.oceanbase.odc.service.task.listener.JobTerminateEvent; import com.oceanbase.odc.service.task.schedule.JobIdentity; @@ -67,20 +68,17 @@ private void cancelJob(TaskFrameworkService taskFrameworkService, JobEntity jobE getConfiguration().getTransactionManager().doInTransactionWithoutResult(() -> { JobEntity lockedEntity = taskFrameworkService.findWithPessimisticLock(jobEntity.getId()); if (lockedEntity.getStatus() == JobStatus.CANCELING) { - if (!configuration.getTaskFrameworkService().refreshLogMeta(lockedEntity.getId())) { + if (!configuration.getTaskFrameworkService().refreshLogMetaForCancelJob(lockedEntity.getId())) { log.info( "Job is canceling but log have not uploaded, continue monitor result, jobId={}, currentStatus={}", lockedEntity.getId(), lockedEntity.getStatus()); return; } - JobStatus currentStatus = taskFrameworkService.find(lockedEntity.getId()).getStatus(); - if (currentStatus.isTerminated()) { - // the job terminated before we update it to CANCELED - log.info("Job is already terminated, jobId={},currentStatus={}", lockedEntity.getId(), - currentStatus); - getConfiguration().getEventPublisher().publishEvent( - new JobTerminateEvent(JobIdentity.of(lockedEntity.getId()), currentStatus)); - return; + // mark resource as released + if (TaskRunMode.K8S == lockedEntity.getRunMode()) { + ResourceManagerUtil.markResourceReleased(lockedEntity, lockedEntity.getExecutorIdentifier(), + getConfiguration().getResourceManager()); + log.info("DoCancelingJob release resource for job = {}", jobEntity); } // For transaction atomic, first update to CANCELED, then stop remote job in executor, // if stop remote failed, transaction will be rollback @@ -100,6 +98,9 @@ private void cancelJob(TaskFrameworkService taskFrameworkService, JobEntity jobE // MessageFormat.format("Cancel job failed, jobId={0}", lockedEntity.getId())); // throw new TaskRuntimeException(e); // } + // set status to destroyed + // TODO(lx): do finish action before release resource + taskFrameworkService.updateExecutorToDestroyed(jobEntity.getId()); log.info("Job be cancelled successfully, jobId={}, oldStatus={}.", lockedEntity.getId(), lockedEntity.getStatus()); getConfiguration().getEventPublisher().publishEvent( diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/schedule/daemon/ResourceManagerUtil.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/schedule/daemon/ResourceManagerUtil.java new file mode 100644 index 0000000000..7fed362876 --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/schedule/daemon/ResourceManagerUtil.java @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.task.schedule.daemon; + +import com.oceanbase.odc.common.util.StringUtils; +import com.oceanbase.odc.metadb.task.JobEntity; +import com.oceanbase.odc.service.resource.ResourceID; +import com.oceanbase.odc.service.resource.ResourceManager; +import com.oceanbase.odc.service.task.caller.ExecutorIdentifier; +import com.oceanbase.odc.service.task.caller.ExecutorIdentifierParser; +import com.oceanbase.odc.service.task.caller.ResourceIDUtil; + +/** + * @author longpeng.zlp + * @date 2024/8/15 17:54 + */ +public class ResourceManagerUtil { + public static void markResourceReleased(JobEntity jobEntity, String executorIdentifierStr, + ResourceManager resourceManager) { + // mark resource as released to let resource collector collect resource + if (StringUtils.isNotEmpty(executorIdentifierStr)) { + ExecutorIdentifier executorIdentifier = ExecutorIdentifierParser.parser(executorIdentifierStr); + ResourceID resourceID = ResourceIDUtil.getResourceID(executorIdentifier, jobEntity); + resourceManager.release(resourceID); + } + } +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/schedule/daemon/StartPreparingJob.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/schedule/daemon/StartPreparingJob.java index 76d0ea36e6..18c46070de 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/schedule/daemon/StartPreparingJob.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/schedule/daemon/StartPreparingJob.java @@ -68,7 +68,8 @@ public void execute(JobExecutionContext context) throws JobExecutionException { configuration.getTaskFrameworkDisabledHandler().handleJobToFailed(); return; } - if (!ResourceDetectUtil.isResourceAvailable(configuration.getTaskFrameworkProperties())) { + // check if local compute resource is enough for process mode + if (!ResourceDetectUtil.isProcessResourceAvailable(configuration.getTaskFrameworkProperties())) { return; } TaskFrameworkProperties taskFrameworkProperties = configuration.getTaskFrameworkProperties(); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/service/ExecutorEndpointManager.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/service/ExecutorEndpointManager.java index 61c9453d80..24d03e3cbf 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/service/ExecutorEndpointManager.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/service/ExecutorEndpointManager.java @@ -26,14 +26,15 @@ import com.oceanbase.odc.core.authority.util.SkipAuthorize; import com.oceanbase.odc.metadb.task.JobEntity; import com.oceanbase.odc.service.cloud.model.CloudProvider; +import com.oceanbase.odc.service.resource.ResourceID; +import com.oceanbase.odc.service.resource.ResourceManager; import com.oceanbase.odc.service.task.caller.ExecutorIdentifier; import com.oceanbase.odc.service.task.caller.ExecutorIdentifierParser; import com.oceanbase.odc.service.task.caller.JobContext; -import com.oceanbase.odc.service.task.caller.K8sJobClient; -import com.oceanbase.odc.service.task.caller.K8sJobClientSelector; -import com.oceanbase.odc.service.task.caller.K8sJobResponse; +import com.oceanbase.odc.service.task.caller.ResourceIDUtil; import com.oceanbase.odc.service.task.enums.JobStatus; import com.oceanbase.odc.service.task.enums.TaskRunMode; +import com.oceanbase.odc.service.task.resource.K8sPodResource; import com.oceanbase.odc.service.task.schedule.DefaultJobContextBuilder; import com.oceanbase.odc.service.task.util.JobPropertiesUtils; @@ -44,10 +45,10 @@ @Service @SkipAuthorize("odc internal usage") public class ExecutorEndpointManager { - @Autowired - private K8sJobClientSelector k8sJobClientSelector; @Autowired private TaskFrameworkService taskFrameworkService; + @Autowired + private ResourceManager resourceManager; private ExecutorHostAdapter hostAdapter = null; @@ -72,7 +73,6 @@ public String getExecutorEndpoint(@NonNull JobEntity je) { // here TaskRunMode.K8S == je.getRunMode() JobContext jobContext = new DefaultJobContextBuilder().build(je); ExecutorIdentifier executorIdentifier = ExecutorIdentifierParser.parser(je.getExecutorIdentifier()); - K8sJobClient k8sJobClient = k8sJobClientSelector.select(jobContext); Map jobProperties = jobContext.getJobProperties(); int executorListenPort = JobPropertiesUtils.getExecutorListenPort(jobProperties); if (executorListenPort <= 0) { @@ -80,10 +80,11 @@ public String getExecutorEndpoint(@NonNull JobEntity je) { + ", executorListenPort=" + executorListenPort); } try { - Optional responseOptional = k8sJobClient.get(executorIdentifier.getNamespace(), - executorIdentifier.getExecutorName()); - if (responseOptional.isPresent()) { - K8sJobResponse response = responseOptional.get(); + ResourceID resourceID = ResourceIDUtil.getResourceID(executorIdentifier, je); + Optional resourceOptional = + resourceManager.query(resourceID); + if (resourceOptional.isPresent()) { + K8sPodResource response = resourceOptional.get(); String podIpAddress = response.getPodIpAddress(); if (StringUtils.isNotBlank(podIpAddress)) { String adaptedHost = adaptHost(podIpAddress, jobProperties); @@ -92,7 +93,7 @@ public String getExecutorEndpoint(@NonNull JobEntity je) { return executorEndpoint; } else { throw new RuntimeException( - "Failed to get executor endpoint, pod status=" + response.getResourceStatus()); + "Failed to get executor endpoint, pod status=" + response.getResourceState()); } } else { throw new RuntimeException("Failed to get executor endpoint, pod not exists"); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/service/StdTaskFrameworkService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/service/StdTaskFrameworkService.java index a18199b3df..07a0b8819b 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/service/StdTaskFrameworkService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/service/StdTaskFrameworkService.java @@ -16,6 +16,7 @@ package com.oceanbase.odc.service.task.service; +import java.text.MessageFormat; import java.util.Collections; import java.util.Date; import java.util.List; @@ -51,32 +52,42 @@ import com.oceanbase.odc.common.util.ExceptionUtils; import com.oceanbase.odc.common.util.StringUtils; import com.oceanbase.odc.common.util.SystemUtils; +import com.oceanbase.odc.core.alarm.AlarmEventNames; +import com.oceanbase.odc.core.alarm.AlarmUtils; import com.oceanbase.odc.core.authority.util.SkipAuthorize; import com.oceanbase.odc.core.shared.PreConditions; import com.oceanbase.odc.core.shared.constant.ResourceType; +import com.oceanbase.odc.core.shared.constant.TaskStatus; import com.oceanbase.odc.core.shared.exception.NotFoundException; +import com.oceanbase.odc.metadb.resource.ResourceEntity; +import com.oceanbase.odc.metadb.resource.ResourceRepository; import com.oceanbase.odc.metadb.task.JobAttributeEntity; import com.oceanbase.odc.metadb.task.JobAttributeRepository; import com.oceanbase.odc.metadb.task.JobEntity; import com.oceanbase.odc.metadb.task.JobRepository; +import com.oceanbase.odc.service.resource.ResourceManager; +import com.oceanbase.odc.service.resource.ResourceState; import com.oceanbase.odc.service.task.config.TaskFrameworkProperties; import com.oceanbase.odc.service.task.constants.JobAttributeEntityColumn; import com.oceanbase.odc.service.task.constants.JobEntityColumn; import com.oceanbase.odc.service.task.enums.JobStatus; import com.oceanbase.odc.service.task.enums.TaskRunMode; import com.oceanbase.odc.service.task.exception.JobException; -import com.oceanbase.odc.service.task.executor.server.HeartbeatRequest; -import com.oceanbase.odc.service.task.executor.task.DefaultTaskResult; -import com.oceanbase.odc.service.task.executor.task.TaskResult; +import com.oceanbase.odc.service.task.executor.HeartbeatRequest; +import com.oceanbase.odc.service.task.executor.TaskResult; import com.oceanbase.odc.service.task.listener.DefaultJobProcessUpdateEvent; import com.oceanbase.odc.service.task.listener.JobTerminateEvent; import com.oceanbase.odc.service.task.processor.result.ResultProcessor; +import com.oceanbase.odc.service.task.resource.DefaultResourceOperatorBuilder; import com.oceanbase.odc.service.task.schedule.JobDefinition; import com.oceanbase.odc.service.task.schedule.JobIdentity; +import com.oceanbase.odc.service.task.state.JobStatusFsm; import com.oceanbase.odc.service.task.util.JobDateUtils; import com.oceanbase.odc.service.task.util.JobPropertiesUtils; import com.oceanbase.odc.service.task.util.TaskExecutorClient; +import cn.hutool.core.text.CharSequenceUtil; +import cn.hutool.core.util.StrUtil; import lombok.NonNull; import lombok.Setter; import lombok.extern.slf4j.Slf4j; @@ -97,6 +108,10 @@ public class StdTaskFrameworkService implements TaskFrameworkService { @Autowired private JobRepository jobRepository; @Autowired + private ResourceRepository resourceRepository; + @Autowired + private ResourceManager resourceManager; + @Autowired private JobAttributeRepository jobAttributeRepository; @Setter private EventPublisher publisher; @@ -118,6 +133,8 @@ public class StdTaskFrameworkService implements TaskFrameworkService { private TaskExecutorClient taskExecutorClient; @Autowired private ExecutorEndpointManager executorEndpointManager; + // default impl + private JobStatusFsm jobStatusFsm = new JobStatusFsm(); @Autowired private List resultProcessors; @@ -162,6 +179,17 @@ public Page findTerminalJob(int page, int size) { return page(condition, page, size); } + @Override + public Page findAbandonedResource(int page, int size) { + Specification specification = SpecificationUtil.columnLate(ResourceEntity.CREATE_TIME, + JobDateUtils.getCurrentDateSubtractDays(RECENT_DAY)); + Specification condition = Specification.where(specification) + .and(SpecificationUtil.columnEqual(ResourceEntity.STATUS, ResourceState.ABANDONED)) + .and(SpecificationUtil.columnEqual(ResourceEntity.TYPE, + DefaultResourceOperatorBuilder.CLOUD_K8S_POD_TYPE)); + return resourceRepository.findAll(condition, PageRequest.of(page, size)); + } + @Override public Page findHeartTimeTimeoutJobs(int timeoutSeconds, int page, int size) { Specification condition = Specification.where(getRecentDaySpec(RECENT_DAY)) @@ -294,7 +322,7 @@ public void handleResult(TaskResult taskResult) { log.warn("Job identity is null"); return; } - if (taskResult.getStatus() == JobStatus.CANCELED) { + if (taskResult.getStatus() == TaskStatus.CANCELED) { log.warn("Job is canceled by odc server, this result is ignored."); return; } @@ -303,8 +331,8 @@ public void handleResult(TaskResult taskResult) { log.warn("Job identity is not exists by id {}", taskResult.getJobIdentity().getId()); return; } + // that's may be a dangerous operation if task report too frequent saveOrUpdateLogMetadata(taskResult, je.getId(), je.getStatus()); - if (je.getStatus().isTerminated() || je.getStatus() == JobStatus.CANCELING) { log.warn("Job is finished, ignore result, jobId={}, currentStatus={}", je.getId(), je.getStatus()); return; @@ -317,56 +345,7 @@ public void handleResult(TaskResult taskResult) { log.warn("Update executor endpoint failed, jobId={}", je.getId()); } } - // TODO: update task entity only when progress changed - int rows = updateTaskResult(taskResult, je); - if (rows > 0) { - taskResultPublisherExecutor - .execute(() -> publisher.publishEvent(new DefaultJobProcessUpdateEvent(taskResult))); - if (publisher != null && taskResult.getStatus() != null && taskResult.getStatus().isTerminated()) { - taskResultPublisherExecutor.execute(() -> publisher - .publishEvent(new JobTerminateEvent(taskResult.getJobIdentity(), taskResult.getStatus(), - taskResult.getErrorMessage()))); - } - } - } - - @Override - public void refreshResult(Long id) { - taskResultPullerExecutor.execute(() -> { - try { - doRefreshResult(id); - } catch (Exception e) { - log.warn("Refresh job result failed, jobId={}, causeReason={}", - id, ExceptionUtils.getRootCauseReason(e)); - } - }); - } - - @Override - public boolean refreshLogMeta(Long id) { - JobEntity je = find(id); - // CANCELING is also a state within the running phase - if (JobStatus.RUNNING != je.getStatus() && JobStatus.CANCELING != je.getStatus()) { - log.warn("Job is not running, don't need to refresh log meta, jobId={}, currentStatus={}", id, - je.getStatus()); - return true; - } - try { - String executorEndpoint = executorEndpointManager.getExecutorEndpoint(je); - DefaultTaskResult result = taskExecutorClient.getResult(executorEndpoint, JobIdentity.of(id)); - - if (je.getRunMode().isK8s() && MapUtils.isEmpty(result.getLogMetadata())) { - log.info("Refresh log failed due to log have not uploaded, jobId={}, currentStatus={}", je.getId(), - je.getStatus()); - return false; - } - saveOrUpdateLogMetadata(result, je.getId(), je.getStatus()); - handleTaskResult(je.getJobType(), result); - return true; - } catch (Exception exception) { - log.warn("Refresh log meta failed,errorMsg={}", exception.getMessage()); - return false; - } + handleTaskResultInner(je, taskResult); } private void doRefreshResult(Long id) throws JobException { @@ -378,18 +357,18 @@ private void doRefreshResult(Long id) throws JobException { } String executorEndpoint = executorEndpointManager.getExecutorEndpoint(je); - DefaultTaskResult result = taskExecutorClient.getResult(executorEndpoint, JobIdentity.of(id)); - if (result.getStatus() == JobStatus.PREPARING) { + TaskResult result = taskExecutorClient.getResult(executorEndpoint, JobIdentity.of(id)); + if (result.getStatus() == TaskStatus.PREPARING) { log.info("Job is preparing, ignore refresh, jobId={}, currentStatus={}", id, result.getStatus()); return; } - DefaultTaskResult previous = JsonUtils.fromJson(je.getResultJson(), DefaultTaskResult.class); + TaskResult previous = JsonUtils.fromJson(je.getResultJson(), TaskResult.class); if (!updateHeartbeatTime(id)) { log.warn("Update lastHeartbeatTime failed, the job may finished or deleted already, jobId={}", id); return; } - if (!result.progressChanged(previous)) { + if (!result.isProgressChanged(previous)) { log.info("Progress not changed, skip update result to metadb, jobId={}, currentProgress={}", id, result.getProgress()); return; @@ -403,10 +382,14 @@ private void doRefreshResult(Long id) throws JobException { je.getId(), je.getStatus()); return; } + handleTaskResultInner(je, result); + } - int rows = updateTaskResult(result, je); + protected void handleTaskResultInner(JobEntity jobEntity, TaskResult result) { + JobStatus expectedJobStatus = jobStatusFsm.determinateJobStatus(jobEntity.getStatus(), result.getStatus()); + int rows = updateTaskResult(result, jobEntity, expectedJobStatus); if (rows == 0) { - log.warn("Update task result failed, the job may finished or deleted already, jobId={}", id); + log.warn("Update task result failed, the job may finished or deleted already, jobId={}", jobEntity.getId()); return; } taskResultPublisherExecutor @@ -414,8 +397,65 @@ private void doRefreshResult(Long id) throws JobException { if (publisher != null && result.getStatus() != null && result.getStatus().isTerminated()) { taskResultPublisherExecutor.execute(() -> publisher - .publishEvent(new JobTerminateEvent(result.getJobIdentity(), result.getStatus(), - result.getErrorMessage()))); + .publishEvent(new JobTerminateEvent(result.getJobIdentity(), expectedJobStatus))); + + // TODO maybe we can destroy the pod there. + if (result.getStatus() == TaskStatus.FAILED) { + Map eventMessage = AlarmUtils.createAlarmMapBuilder() + .item(AlarmUtils.ORGANIZATION_NAME, Optional.ofNullable(jobEntity.getOrganizationId()).map( + Object::toString).orElse(StrUtil.EMPTY)) + .item(AlarmUtils.TASK_JOB_ID_NAME, String.valueOf(jobEntity.getId())) + .item(AlarmUtils.MESSAGE_NAME, + MessageFormat.format("Job execution failed, jobId={0}", + result.getJobIdentity().getId())) + .item(AlarmUtils.FAILED_REASON_NAME, + CharSequenceUtil.nullToDefault(result.getErrorMessage(), + CharSequenceUtil.EMPTY)) + .build(); + AlarmUtils.alarm(AlarmEventNames.TASK_EXECUTION_FAILED, eventMessage); + } + } + } + + @Override + public void refreshResult(Long id) { + taskResultPullerExecutor.execute(() -> { + try { + doRefreshResult(id); + } catch (Exception e) { + log.warn("Refresh job result failed, jobId={}, causeReason={}", + id, ExceptionUtils.getRootCauseReason(e)); + } + }); + } + + @Override + public boolean refreshLogMetaForCancelJob(Long id) { + JobEntity je = find(id); + // CANCELING is also a state within the running phase + if (JobStatus.RUNNING != je.getStatus() && JobStatus.CANCELING != je.getStatus()) { + log.warn("Job is not running, don't need to refresh log meta, jobId={}, currentStatus={}", id, + je.getStatus()); + return true; + } + try { + String executorEndpoint = executorEndpointManager.getExecutorEndpoint(je); + TaskResult result = taskExecutorClient.getResult(executorEndpoint, JobIdentity.of(id)); + + if (je.getRunMode().isK8s() && MapUtils.isEmpty(result.getLogMetadata())) { + log.info("Refresh log failed due to log have not uploaded, jobId={}, currentStatus={}", je.getId(), + je.getStatus()); + return false; + } + // force update result json once + // TODO(tianke): move this logic to event listener + jobRepository.updateResultJson(JsonUtils.toJson(result), result.getJobIdentity().getId()); + saveOrUpdateLogMetadata(result, je.getId(), je.getStatus()); + handleTaskResult(je.getJobType(), result); + return true; + } catch (Exception exception) { + log.warn("Refresh log meta failed,errorMsg={}", exception.getMessage()); + return false; } } @@ -458,11 +498,11 @@ private int updateExecutorEndpoint(Long id, String executorEndpoint, JobEntity c return jobRepository.updateExecutorEndpoint(id, executorEndpoint, currentJob.getStatus()); } - private int updateTaskResult(TaskResult taskResult, JobEntity currentJob) { + private int updateTaskResult(TaskResult taskResult, JobEntity currentJob, JobStatus expectedStatus) { JobEntity jse = new JobEntity(); handleTaskResult(currentJob.getJobType(), taskResult); - jse.setResultJson(taskResult.getResultJson()); - jse.setStatus(taskResult.getStatus()); + jse.setResultJson(JsonUtils.toJson(taskResult)); + jse.setStatus(expectedStatus); jse.setProgressPercentage(taskResult.getProgress()); jse.setLastReportTime(JobDateUtils.getCurrentDate()); if (taskResult.getStatus() != null && taskResult.getStatus().isTerminated()) { @@ -627,14 +667,6 @@ public Optional findByJobIdAndAttributeKey(Long jobId, String attributeK return Objects.isNull(attributeEntity) ? Optional.empty() : Optional.of(attributeEntity.getAttributeValue()); } - private void handleTaskResult(String jobType, TaskResult taskResult) { - for (ResultProcessor processor : resultProcessors) { - if (processor.interested(jobType)) { - processor.process(taskResult); - } - } - } - @Override public Map getJobAttributes(Long jobId) { List attributeEntityList = jobAttributeRepository.findByJobId(jobId); @@ -643,4 +675,11 @@ public Map getJobAttributes(Long jobId) { JobAttributeEntity::getAttributeValue)); } + private void handleTaskResult(String jobType, TaskResult taskResult) { + for (ResultProcessor processor : resultProcessors) { + if (processor.interested(jobType)) { + processor.process(taskResult); + } + } + } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/service/TaskFrameworkService.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/service/TaskFrameworkService.java index 7f149b955d..71712ac9fc 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/service/TaskFrameworkService.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/service/TaskFrameworkService.java @@ -22,11 +22,12 @@ import org.springframework.data.domain.Page; +import com.oceanbase.odc.metadb.resource.ResourceEntity; import com.oceanbase.odc.metadb.task.JobEntity; import com.oceanbase.odc.service.task.enums.JobStatus; import com.oceanbase.odc.service.task.enums.TaskRunMode; -import com.oceanbase.odc.service.task.executor.server.HeartbeatRequest; -import com.oceanbase.odc.service.task.executor.task.TaskResult; +import com.oceanbase.odc.service.task.executor.HeartbeatRequest; +import com.oceanbase.odc.service.task.executor.TaskResult; import com.oceanbase.odc.service.task.schedule.JobDefinition; /** @@ -42,7 +43,7 @@ public interface TaskFrameworkService { void refreshResult(Long id); - boolean refreshLogMeta(Long id); + boolean refreshLogMetaForCancelJob(Long id); void handleHeart(HeartbeatRequest heart); @@ -52,6 +53,8 @@ public interface TaskFrameworkService { Page findTerminalJob(int page, int size); + Page findAbandonedResource(int page, int size); + JobEntity findWithPessimisticLock(Long id); Page find(JobStatus status, int page, int size); diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/state/JobStatusFsm.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/state/JobStatusFsm.java new file mode 100644 index 0000000000..f99162bfb5 --- /dev/null +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/state/JobStatusFsm.java @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.task.state; + +import com.oceanbase.odc.core.shared.constant.TaskStatus; +import com.oceanbase.odc.service.task.enums.JobStatus; + +/** + * default fsm for job status transfer according job current status and task status + * + * @author longpeng.zlp + * @date 2024/10/22 14:15 + */ +public class JobStatusFsm { + /** + * determinate job status by task status and current job status + * + * @param currentStatus + * @param taskStatus + * @return + */ + public JobStatus determinateJobStatus(JobStatus currentStatus, TaskStatus taskStatus) { + switch (taskStatus) { + // prepare is task init status, job should expected in running status + // running Job status is expected, if job = canceling, outside may set to cancel when task timeout + case PREPARING: + // running meaning task has running normally, job should expected in running status + case RUNNING: + return currentStatus; + // if task status is canceled, it must be canceled by job who owns it + case CANCELED: + return JobStatus.CANCELED; + // any task failed will cause the failed status of job + case ABNORMAL: + case FAILED: + return JobStatus.FAILED; + // task done will drive job status to done + case DONE: + return JobStatus.DONE; + default: + throw new IllegalStateException("status " + taskStatus + " not handled"); + } + } +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/util/JobPropertiesUtils.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/util/JobPropertiesUtils.java index a0418ea523..124f323649 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/util/JobPropertiesUtils.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/util/JobPropertiesUtils.java @@ -21,6 +21,7 @@ import com.oceanbase.odc.common.util.MapUtils; import com.oceanbase.odc.common.util.StringUtils; import com.oceanbase.odc.service.cloud.model.CloudProvider; +import com.oceanbase.odc.service.task.caller.ResourceIDUtil; import com.oceanbase.odc.service.task.enums.TaskMonitorMode; import lombok.NonNull; @@ -40,20 +41,20 @@ public static Map getLabels(@NonNull Map jobProp public static void setCloudProvider(@NonNull Map jobProperties, @NonNull CloudProvider cloudProvider) { - jobProperties.put("cloudProvider", cloudProvider.toString()); + jobProperties.put(ResourceIDUtil.GROUP_PROP_NAME, cloudProvider.toString()); } public static CloudProvider getCloudProvider(@NonNull Map jobProperties) { - String cloudProvider = jobProperties.get("cloudProvider"); + String cloudProvider = jobProperties.get(ResourceIDUtil.GROUP_PROP_NAME); return StringUtils.isBlank(cloudProvider) ? CloudProvider.NONE : CloudProvider.valueOf(cloudProvider); } public static void setRegionName(@NonNull Map jobProperties, @NonNull String regionName) { - jobProperties.put("regionName", regionName); + jobProperties.put(ResourceIDUtil.REGION_PROP_NAME, regionName); } public static String getRegionName(@NonNull Map jobProperties) { - return jobProperties.get("regionName"); + return jobProperties.get(ResourceIDUtil.REGION_PROP_NAME); } public static void setMonitorMode(@NonNull Map jobProperties, diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/util/JobUtils.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/util/JobUtils.java index 9f02bff787..e80ee3861f 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/util/JobUtils.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/util/JobUtils.java @@ -30,12 +30,14 @@ import com.oceanbase.odc.common.util.SystemUtils; import com.oceanbase.odc.core.shared.Verify; import com.oceanbase.odc.core.shared.constant.ConnectType; +import com.oceanbase.odc.metadb.task.JobEntity; import com.oceanbase.odc.service.connection.model.ConnectionConfig; import com.oceanbase.odc.service.objectstorage.cloud.model.ObjectStorageConfiguration; import com.oceanbase.odc.service.task.caller.JobEnvironmentEncryptor; import com.oceanbase.odc.service.task.constants.JobConstants; import com.oceanbase.odc.service.task.constants.JobEnvKeyConstants; import com.oceanbase.odc.service.task.enums.TaskRunMode; +import com.oceanbase.odc.service.task.executor.TaskResult; import com.oceanbase.odc.service.task.jasypt.AccessEnvironmentJasyptEncryptorConfigProperties; import com.oceanbase.odc.service.task.jasypt.DefaultJasyptEncryptor; import com.oceanbase.odc.service.task.jasypt.JasyptEncryptorConfigProperties; @@ -188,4 +190,9 @@ public static String encrypt(String key, String salt, String raw) { public static String decrypt(String key, String salt, String encrypted) { return new JobEnvironmentEncryptor().decrypt(key, salt, encrypted); } + + public static String retrieveJobResultStr(JobEntity jobEntity) { + TaskResult taskResult = JsonUtils.fromJson(jobEntity.getResultJson(), TaskResult.class); + return null == taskResult ? null : taskResult.getResultJson(); + } } diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/util/TaskExecutorClient.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/util/TaskExecutorClient.java index 5a8f1e2c38..242f298062 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/util/TaskExecutorClient.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/task/util/TaskExecutorClient.java @@ -28,8 +28,7 @@ import com.oceanbase.odc.service.schedule.ScheduleLogProperties; import com.oceanbase.odc.service.task.constants.JobExecutorUrls; import com.oceanbase.odc.service.task.exception.JobException; -import com.oceanbase.odc.service.task.executor.server.ExecutorRequestHandler; -import com.oceanbase.odc.service.task.executor.task.DefaultTaskResult; +import com.oceanbase.odc.service.task.executor.TaskResult; import com.oceanbase.odc.service.task.model.OdcTaskLogLevel; import com.oceanbase.odc.service.task.schedule.JobIdentity; @@ -38,7 +37,7 @@ /** * task-executor api calling encapsulation
- * see @{@link JobExecutorUrls} @{@link ExecutorRequestHandler} + * see @{@link JobExecutorUrls} */ @Slf4j @Component @@ -111,12 +110,12 @@ public void modifyJobParameters(@NonNull String executorEndpoint, @NonNull JobId } } - public DefaultTaskResult getResult(@NonNull String executorEndpoint, @NonNull JobIdentity ji) throws JobException { + public TaskResult getResult(@NonNull String executorEndpoint, @NonNull JobIdentity ji) throws JobException { String url = executorEndpoint + String.format(JobExecutorUrls.GET_RESULT, ji.getId()); log.info("Try query job result from executor, jobId={}, url={}", ji.getId(), url); try { - SuccessResponse response = - HttpClientUtils.request("GET", url, new TypeReference>() {}); + SuccessResponse response = + HttpClientUtils.request("GET", url, new TypeReference>() {}); if (response != null && response.getSuccessful()) { return response.getData(); } else { diff --git a/server/odc-service/src/test/java/com/oceanbase/odc/service/dlm/utils/DataArchiveConditionUtilTest.java b/server/odc-service/src/test/java/com/oceanbase/odc/service/dlm/utils/DataArchiveConditionUtilTest.java index 6c8dd523f7..bf2fdd21fb 100644 --- a/server/odc-service/src/test/java/com/oceanbase/odc/service/dlm/utils/DataArchiveConditionUtilTest.java +++ b/server/odc-service/src/test/java/com/oceanbase/odc/service/dlm/utils/DataArchiveConditionUtilTest.java @@ -46,7 +46,7 @@ public void testParseCondition() { String condition = "select * from test where gmt_create>'${start}' and gmt_create<'${end}'"; OffsetConfig config = new OffsetConfig(); config.setName("start"); - config.setPattern("yyyy-MM-dd HH:mm:ss|+10m"); + config.setPattern("yyyy-MM-dd|+10M"); OffsetConfig config2 = new OffsetConfig(); config2.setName("end"); config2.setOperator(Operator.PLUS); @@ -58,12 +58,11 @@ public void testParseCondition() { variables.add(config2); String result = DataArchiveConditionUtil.parseCondition(condition, variables, date); - SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); Calendar calendar = Calendar.getInstance(); calendar.setTime(date); - calendar.add(Calendar.MINUTE, 10); + calendar.add(Calendar.MONTH, 10); String expect = condition.replace("${start}", sdf.format(calendar.getTime())); - sdf = new SimpleDateFormat("yyyy-MM-dd"); calendar.setTime(date); calendar.add(Calendar.DAY_OF_MONTH, 1); expect = expect.replace("${end}", sdf.format(calendar.getTime())); @@ -76,7 +75,7 @@ public void testNotFoundVariable() { String condition = "select * from test where gmt_create>'${start}' and gmt_create<'${end}'"; OffsetConfig config = new OffsetConfig(); config.setName("start"); - config.setPattern("yyyy-MM-dd HH:mm:ss|+10m"); + config.setPattern("yyyy-MM-dd HH:mm:ss|+10M"); List variables = new LinkedList<>(); variables.add(config); diff --git a/server/odc-service/src/test/java/com/oceanbase/odc/service/onlineschemachange/OnlineSchemaChangeFlowableTaskTest.java b/server/odc-service/src/test/java/com/oceanbase/odc/service/onlineschemachange/OnlineSchemaChangeFlowableTaskTest.java index 4e2c2c7e03..2531d4ee95 100644 --- a/server/odc-service/src/test/java/com/oceanbase/odc/service/onlineschemachange/OnlineSchemaChangeFlowableTaskTest.java +++ b/server/odc-service/src/test/java/com/oceanbase/odc/service/onlineschemachange/OnlineSchemaChangeFlowableTaskTest.java @@ -131,6 +131,7 @@ public void testScheduleTasksUpdateHint() { Assert.assertTrue( scheduleTasksUpdateHint.hasDiff(new OnlineSchemaChangeFlowableTask.ScheduleTasksUpdateHint(0))); Assert.assertTrue(scheduleTasksUpdateHint.hasDiff(new OnlineSchemaChangeFlowableTask.ScheduleTasksUpdateHint(1, + Collections.singletonMap(1L, OmsStepName.FULL_TRANSFER.name()), taskMap))); Map muted = new HashMap<>(scheduleTasksUpdateHint.getTaskStepsMap()); muted.put(2L, OmsStepName.APP_SWITCH.name()); diff --git a/server/odc-service/src/test/java/com/oceanbase/odc/service/resource/k8s/K8sResourceOperatorTest.java b/server/odc-service/src/test/java/com/oceanbase/odc/service/resource/k8s/K8sResourceOperatorTest.java index cecbedd889..d71e1831b1 100644 --- a/server/odc-service/src/test/java/com/oceanbase/odc/service/resource/k8s/K8sResourceOperatorTest.java +++ b/server/odc-service/src/test/java/com/oceanbase/odc/service/resource/k8s/K8sResourceOperatorTest.java @@ -25,9 +25,15 @@ import com.oceanbase.odc.service.resource.ResourceID; import com.oceanbase.odc.service.resource.ResourceLocation; import com.oceanbase.odc.service.resource.ResourceState; -import com.oceanbase.odc.service.resource.k8s.client.K8sJobClient; -import com.oceanbase.odc.service.resource.k8s.client.K8sJobClientSelector; import com.oceanbase.odc.service.task.exception.JobException; +import com.oceanbase.odc.service.task.resource.DefaultResourceOperatorBuilder; +import com.oceanbase.odc.service.task.resource.K8sPodResource; +import com.oceanbase.odc.service.task.resource.K8sResourceContext; +import com.oceanbase.odc.service.task.resource.K8sResourceOperator; +import com.oceanbase.odc.service.task.resource.K8sResourceOperatorContext; +import com.oceanbase.odc.service.task.resource.PodConfig; +import com.oceanbase.odc.service.task.resource.client.K8sJobClient; +import com.oceanbase.odc.service.task.resource.client.K8sJobClientSelector; /** * test for K8SResourceOperator diff --git a/server/odc-service/src/test/java/com/oceanbase/odc/service/sqlcheck/OracleSqlCheckerTest.java b/server/odc-service/src/test/java/com/oceanbase/odc/service/sqlcheck/OracleSqlCheckerTest.java index 22e3175570..ad5474ea9a 100644 --- a/server/odc-service/src/test/java/com/oceanbase/odc/service/sqlcheck/OracleSqlCheckerTest.java +++ b/server/odc-service/src/test/java/com/oceanbase/odc/service/sqlcheck/OracleSqlCheckerTest.java @@ -1221,7 +1221,7 @@ public void check_truncateTbl_violationGenerated() { null, Collections.singletonList(new TruncateTableExists())); List actual = sqlChecker.check(toOffsetString(sqls), null); - SqlCheckRuleType type = SqlCheckRuleType.TRUNCATE_TBLE_EXISTS; + SqlCheckRuleType type = SqlCheckRuleType.TRUNCATE_TABLE_EXISTS; CheckViolation c1 = new CheckViolation(sqls[1], 1, 0, 0, 17, type, new Object[] {}); List expect = Collections.singletonList(c1); diff --git a/server/odc-service/src/test/java/com/oceanbase/odc/service/task/dummy/LocalMockK8sJobClientTest.java b/server/odc-service/src/test/java/com/oceanbase/odc/service/task/dummy/LocalMockK8sJobClientTest.java new file mode 100644 index 0000000000..cecc9eccd9 --- /dev/null +++ b/server/odc-service/src/test/java/com/oceanbase/odc/service/task/dummy/LocalMockK8sJobClientTest.java @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.task.dummy; + +import org.junit.Assert; +import org.junit.Ignore; +import org.junit.Test; +import org.mockito.Mockito; + +import com.oceanbase.odc.service.common.model.HostProperties; +import com.oceanbase.odc.service.task.config.JobConfiguration; +import com.oceanbase.odc.service.task.config.JobConfigurationHolder; +import com.oceanbase.odc.service.task.config.TaskFrameworkProperties; +import com.oceanbase.odc.service.task.exception.JobException; +import com.oceanbase.odc.service.task.resource.K8sPodResource; +import com.oceanbase.odc.service.task.resource.K8sResourceContext; +import com.oceanbase.odc.service.task.resource.PodConfig; +import com.oceanbase.odc.service.task.resource.client.K8sJobClient; +import com.oceanbase.odc.service.task.schedule.JobCredentialProvider; + +/** + * @author longpeng.zlp + * @date 2024/8/28 14:27 + */ +public class LocalMockK8sJobClientTest { + @Ignore + @Test + public void testLocalProcessStart() throws JobException { + JobConfiguration jobConfiguration = Mockito.mock(JobConfiguration.class); + JobCredentialProvider jobCredentialProvider = Mockito.mock(JobCredentialProvider.class); + TaskFrameworkProperties taskFrameworkProperties = Mockito.mock(TaskFrameworkProperties.class); + Mockito.when(taskFrameworkProperties.getJobProcessMaxMemorySizeInMB()).thenReturn(2048L); + Mockito.when(taskFrameworkProperties.getJobProcessMinMemorySizeInMB()).thenReturn(2048L); + HostProperties hostProperties = Mockito.mock(HostProperties.class); + Mockito.when(jobConfiguration.getJobCredentialProvider()).thenReturn(jobCredentialProvider); + Mockito.when(jobConfiguration.getTaskFrameworkProperties()).thenReturn(taskFrameworkProperties); + Mockito.when(jobConfiguration.getHostProperties()).thenReturn(hostProperties); + JobConfigurationHolder.setJobConfiguration(jobConfiguration); + LocalMockK8sJobClient localMockK8sJobClient = new LocalMockK8sJobClient(); + K8sJobClient k8sJobClient = localMockK8sJobClient.select("any"); + K8sResourceContext k8sResourceContext = + new K8sResourceContext(new PodConfig(), "local", "local", "image", "local", null); + K8sPodResource k8sResource = k8sJobClient.create(k8sResourceContext); + Assert.assertNotNull(k8sResource); + } +} diff --git a/server/odc-service/src/test/java/com/oceanbase/odc/service/task/executor/TaskResultTest.java b/server/odc-service/src/test/java/com/oceanbase/odc/service/task/executor/TaskResultTest.java new file mode 100644 index 0000000000..b0564f5b13 --- /dev/null +++ b/server/odc-service/src/test/java/com/oceanbase/odc/service/task/executor/TaskResultTest.java @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.task.executor; + +import java.util.HashMap; +import java.util.Map; +import java.util.TreeMap; + +import org.junit.Assert; +import org.junit.Test; + +import com.oceanbase.odc.core.shared.constant.TaskStatus; +import com.oceanbase.odc.service.task.schedule.JobIdentity; + +/** + * @author longpeng.zlp + * @date 2024/11/11 15:09 + */ +public class TaskResultTest { + @Test + public void testTaskResult() { + Map logMetadata1 = new HashMap() { + { + put("key1", "value1"); + put("key2", "value2"); + } + }; + + Map logMetadata2 = new TreeMap() { + { + put("key1", null); + } + }; + + TaskResult result1 = createTaskResult(TaskStatus.RUNNING, 0.877, logMetadata1, "res1"); + Assert.assertFalse(result1.isProgressChanged(result1)); + Assert.assertTrue(result1.isProgressChanged(null)); + // log meta changed + TaskResult result2 = createTaskResult(TaskStatus.RUNNING, 0.877, logMetadata2, "res1"); + Assert.assertTrue(result1.isProgressChanged(result2)); + // statue changed + TaskResult result3 = createTaskResult(TaskStatus.DONE, 0.877, logMetadata1, "res1"); + Assert.assertTrue(result1.isProgressChanged(result3)); + // result json changed + TaskResult result4 = createTaskResult(TaskStatus.RUNNING, 0.877, logMetadata1, null); + Assert.assertTrue(result1.isProgressChanged(result4)); + // result progress changed + TaskResult result5 = createTaskResult(TaskStatus.RUNNING, 0.8774, logMetadata1, "res1"); + Assert.assertTrue(result1.isProgressChanged(result5)); + } + + private TaskResult createTaskResult(TaskStatus taskStatus, double progress, Map logMetadata, + String resultJson) { + TaskResult taskResult = new TaskResult(); + taskResult.setJobIdentity(JobIdentity.of(1L)); + taskResult.setResultJson(resultJson); + taskResult.setProgress(progress); + taskResult.setStatus(taskStatus); + taskResult.setLogMetadata(logMetadata); + return taskResult; + } +} diff --git a/server/odc-service/src/test/java/com/oceanbase/odc/service/task/executor/task/BaseTaskTest.java b/server/odc-service/src/test/java/com/oceanbase/odc/service/task/executor/task/BaseTaskTest.java deleted file mode 100644 index 506d00c50c..0000000000 --- a/server/odc-service/src/test/java/com/oceanbase/odc/service/task/executor/task/BaseTaskTest.java +++ /dev/null @@ -1,138 +0,0 @@ -/* - * Copyright (c) 2023 OceanBase. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.oceanbase.odc.service.task.executor.task; - -import java.util.HashMap; - -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.mockito.MockedStatic; -import org.mockito.Mockito; - -import com.oceanbase.odc.common.util.SystemUtils; -import com.oceanbase.odc.service.task.caller.DefaultJobContext; -import com.oceanbase.odc.service.task.caller.JobContext; -import com.oceanbase.odc.service.task.constants.JobEnvKeyConstants; -import com.oceanbase.odc.service.task.executor.server.TaskMonitor; -import com.oceanbase.odc.service.task.schedule.JobIdentity; - -/** - * @author longpeng.zlp - * @date 2024/10/11 14:11 - */ -public class BaseTaskTest { - private DefaultJobContext jobContext; - - @Before - public void init() { - jobContext = new DefaultJobContext(); - jobContext.setJobParameters(new HashMap<>()); - jobContext.setJobProperties(new HashMap<>()); - JobIdentity jobIdentity = new JobIdentity(); - jobIdentity.setId(1L); - jobContext.setJobIdentity(jobIdentity); - jobContext.setJobClass(DummyBaseTask.class.getName()); - } - - - @Test - public void testExceptionListenerNormal() { - try (MockedStatic mockSystemUtil = Mockito.mockStatic(SystemUtils.class)) { - mockSystemUtil.when(() -> { - SystemUtils.getEnvOrProperty(JobEnvKeyConstants.ODC_EXECUTOR_PORT); - }).thenReturn("9099"); - DummyBaseTask dummyBaseTask = new DummyBaseTask(false); - dummyBaseTask.start(new TaskContext() { - @Override - public ExceptionListener getExceptionListener() { - return dummyBaseTask; - } - - @Override - public JobContext getJobContext() { - return jobContext; - } - }); - DefaultTaskResult taskResult = DefaultTaskResultBuilder.build(dummyBaseTask); - DefaultTaskResultBuilder.assignErrorMessage(taskResult, dummyBaseTask); - Assert.assertNull(taskResult.getErrorMessage()); - } - } - - @Test - public void testExceptionListenerWithException() { - try (MockedStatic mockSystemUtil = Mockito.mockStatic(SystemUtils.class)) { - mockSystemUtil.when(() -> { - SystemUtils.getEnvOrProperty(JobEnvKeyConstants.ODC_EXECUTOR_PORT); - }).thenReturn("9099"); - DummyBaseTask dummyBaseTask = new DummyBaseTask(true); - dummyBaseTask.start(new TaskContext() { - @Override - public ExceptionListener getExceptionListener() { - return dummyBaseTask; - } - - @Override - public JobContext getJobContext() { - return jobContext; - } - }); - DefaultTaskResult taskResult = DefaultTaskResultBuilder.build(dummyBaseTask); - DefaultTaskResultBuilder.assignErrorMessage(taskResult, dummyBaseTask); - Assert.assertTrue(taskResult.getErrorMessage().contains("exception should be thrown")); - } - } - - private static final class DummyBaseTask extends BaseTask { - private final boolean shouldThrowException; - - private DummyBaseTask(boolean shouldThrowException) { - this.shouldThrowException = shouldThrowException; - } - - @Override - protected void doInit(JobContext context) throws Exception {} - - @Override - protected boolean doStart(JobContext context, TaskContext taskContext) throws Exception { - if (shouldThrowException) { - throw new IllegalStateException("exception should be thrown"); - } - return true; - } - - protected TaskMonitor createTaskMonitor() { - return Mockito.mock(TaskMonitor.class); - } - - @Override - protected void doStop() throws Exception {} - - @Override - protected void doClose() throws Exception {} - - @Override - public double getProgress() { - return 100; - } - - @Override - public String getTaskResult() { - return "res"; - } - } -} diff --git a/server/odc-service/src/test/java/com/oceanbase/odc/service/task/status/JobStatusFsmTest.java b/server/odc-service/src/test/java/com/oceanbase/odc/service/task/status/JobStatusFsmTest.java new file mode 100644 index 0000000000..195b2f30b7 --- /dev/null +++ b/server/odc-service/src/test/java/com/oceanbase/odc/service/task/status/JobStatusFsmTest.java @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.task.status; + +import org.junit.Assert; +import org.junit.Test; + +import com.oceanbase.odc.core.shared.constant.TaskStatus; +import com.oceanbase.odc.service.task.enums.JobStatus; +import com.oceanbase.odc.service.task.state.JobStatusFsm; + +/** + * @author longpeng.zlp + * @date 2024/10/23 09:54 + */ +public class JobStatusFsmTest { + private final JobStatusFsm jobStatusFsm = new JobStatusFsm(); + + @Test + public void testJobStatusFsm() { + // check remaining + checkStatusRemain(TaskStatus.RUNNING); + checkStatusRemain(TaskStatus.PREPARING); + // check transfer + checkStatusTransfer(TaskStatus.CANCELED, JobStatus.CANCELED); + checkStatusTransfer(TaskStatus.DONE, JobStatus.DONE); + checkStatusTransfer(TaskStatus.FAILED, JobStatus.FAILED); + checkStatusTransfer(TaskStatus.ABNORMAL, JobStatus.FAILED); + } + + private void checkStatusRemain(TaskStatus remainStatus) { + for (JobStatus jobStatus : JobStatus.values()) { + Assert.assertEquals(jobStatus, jobStatusFsm.determinateJobStatus(jobStatus, remainStatus)); + } + } + + private void checkStatusTransfer(TaskStatus remainStatus, JobStatus targetStatus) { + for (JobStatus jobStatus : JobStatus.values()) { + Assert.assertEquals(targetStatus, jobStatusFsm.determinateJobStatus(jobStatus, remainStatus)); + } + } +} diff --git a/server/odc-service/src/test/resources/session/test_db_schema_extractor.yaml b/server/odc-service/src/test/resources/session/test_db_schema_extractor.yaml index 85e0f90885..5e69004838 100644 --- a/server/odc-service/src/test/resources/session/test_db_schema_extractor.yaml +++ b/server/odc-service/src/test/resources/session/test_db_schema_extractor.yaml @@ -183,4 +183,185 @@ default_schema: DEFAULT_SCHEMA sqls: - "SELECT * FROM DB1.TABLE1@FAKE_DBLINK;" - expected: [ ] \ No newline at end of file + expected: [ ] +# drop function ,procedure and trigger with dbname +- id: 23 + dialect_type: OB_MYSQL + default_schema: DEFAULT_SCHEMA + sqls: + - "drop function db1.func1" + expected: + - schema: db1 + table: ~ +- id: 24 + dialect_type: OB_MYSQL + default_schema: DEFAULT_SCHEMA + sqls: + - "drop procedure db1.proc1" + expected: + - schema: db1 + table: ~ +- id: 25 + dialect_type: OB_MYSQL + default_schema: DEFAULT_SCHEMA + sqls: + - "drop trigger db1.trigger1" + expected: + - schema: db1 + table: ~ +- id: 26 + dialect_type: OB_ORACLE + default_schema: DEFAULT_SCHEMA + sqls: + - "DROP FUNCTION DB1.FUNC1" + expected: + - schema: DB1 + table: ~ +- id: 27 + dialect_type: OB_ORACLE + default_schema: DEFAULT_SCHEMA + sqls: + - "DROP PROCEDURE DB1.PROC1" + expected: + - schema: DB1 + table: ~ +- id: 28 + dialect_type: OB_ORACLE + default_schema: DEFAULT_SCHEMA + sqls: + - "DROP TRIGGER DB1.TRIGGER1" + expected: + - schema: DB1 + table: ~ +#drop sequence、type、package、synonym、public synonym with dbname for ob oracle +- id: 29 + dialect_type: OB_ORACLE + default_schema: DEFAULT_SCHEMA + sqls: + - "DROP SEQUENCE DB1.SEQ1" + expected: + - schema: DB1 + table: ~ +- id: 30 + dialect_type: OB_ORACLE + default_schema: DEFAULT_SCHEMA + sqls: + - "DROP TYPE DB1.TYPE1" + expected: + - schema: DB1 + table: ~ +- id: 31 + dialect_type: OB_ORACLE + default_schema: DEFAULT_SCHEMA + sqls: + - "DROP PACKAGE DB1.PACKAGE1" + expected: + - schema: DB1 + table: ~ +- id: 32 + dialect_type: OB_ORACLE + default_schema: DEFAULT_SCHEMA + sqls: + - "DROP SYNONYM DB1.SYNONYM1" + expected: + - schema: DB1 + table: ~ +- id : 33 + dialect_type: OB_ORACLE + default_schema: DEFAULT_SCHEMA + sqls: + - "DROP PUBLIC SYNONYM DB1.SYNONYM1" + expected: + - schema: DB1 + table: ~ +# drop function ,procedure and trigger without dbname +- id: 34 + dialect_type: OB_MYSQL + default_schema: DEFAULT_SCHEMA + sqls: + - "drop function func1" + expected: + - schema: DEFAULT_SCHEMA + table: ~ +- id: 35 + dialect_type: OB_MYSQL + default_schema: DEFAULT_SCHEMA + sqls: + - "drop procedure proc1" + expected: + - schema: DEFAULT_SCHEMA + table: ~ +- id: 36 + dialect_type: OB_MYSQL + default_schema: DEFAULT_SCHEMA + sqls: + - "drop trigger trigger1" + expected: + - schema: DEFAULT_SCHEMA + table: ~ +- id: 37 + dialect_type: OB_ORACLE + default_schema: DEFAULT_SCHEMA + sqls: + - "DROP FUNCTION FUNC1" + expected: + - schema: DEFAULT_SCHEMA + table: ~ +- id: 38 + dialect_type: OB_ORACLE + default_schema: DEFAULT_SCHEMA + sqls: + - "DROP PROCEDURE PROC1" + expected: + - schema: DEFAULT_SCHEMA + table: ~ +- id: 39 + dialect_type: OB_ORACLE + default_schema: DEFAULT_SCHEMA + sqls: + - "DROP TRIGGER TRIGGER1" + expected: + - schema: DEFAULT_SCHEMA + table: ~ +#drop sequence、type、package、synonym、public synonym without dbname for ob oracle +- id: 40 + dialect_type: OB_ORACLE + default_schema: DEFAULT_SCHEMA + sqls: + - "DROP SEQUENCE SEQ1" + expected: + - schema: DEFAULT_SCHEMA + table: ~ +- id: 41 + dialect_type: OB_ORACLE + default_schema: DEFAULT_SCHEMA + sqls: + - "DROP TYPE TYPE1" + expected: + - schema: DEFAULT_SCHEMA + table: ~ +- id: 42 + dialect_type: OB_ORACLE + default_schema: DEFAULT_SCHEMA + sqls: + - "DROP PACKAGE PACKAGE1" + expected: + - schema: DEFAULT_SCHEMA + table: ~ +- id: 43 + dialect_type: OB_ORACLE + default_schema: DEFAULT_SCHEMA + sqls: + - "DROP SYNONYM SYNONYM1" + expected: + - schema: DEFAULT_SCHEMA + table: ~ +- id : 44 + dialect_type: OB_ORACLE + default_schema: DEFAULT_SCHEMA + sqls: + - "DROP PUBLIC SYNONYM SYNONYM1" + expected: + - schema: DEFAULT_SCHEMA + table: ~ + diff --git a/server/odc-test/pom.xml b/server/odc-test/pom.xml index d3bf3ebbd6..b11dd83467 100644 --- a/server/odc-test/pom.xml +++ b/server/odc-test/pom.xml @@ -7,7 +7,7 @@ com.oceanbase odc-parent - 4.3.2-SNAPSHOT + 4.3.3-SNAPSHOT ../../pom.xml odc-test diff --git a/server/odc-test/src/main/java/com/oceanbase/odc/test/database/TestDBConfigurations.java b/server/odc-test/src/main/java/com/oceanbase/odc/test/database/TestDBConfigurations.java index b30b390ee5..f606dc7ad5 100644 --- a/server/odc-test/src/main/java/com/oceanbase/odc/test/database/TestDBConfigurations.java +++ b/server/odc-test/src/main/java/com/oceanbase/odc/test/database/TestDBConfigurations.java @@ -150,6 +150,11 @@ private void createTestDatabasesAndUpdateConfig(TestDBType type, TestDBConfigura sql = new StringBuilder("GRANT SYSDBA, RESOURCE, CREATE SESSION TO ").append(username); stmt.execute(sql.toString()); log.info("grant sysdba to new created user, username: {}", username); + // Although the above code has granted sysdba role to the new user, the connection created with the + // new user does not use the sysdba role, so the following grant statement is required + sql = new StringBuilder("GRANT ALL PRIVILEGES TO ").append(username); + stmt.execute(sql.toString()); + log.info("grant all privileges to new created user, username: {}", username); configuration.setDefaultDBName(username); configuration.setUsername(username); } diff --git a/server/plugins/connect-plugin-api/pom.xml b/server/plugins/connect-plugin-api/pom.xml index f26d4921e8..5623a7dc1a 100644 --- a/server/plugins/connect-plugin-api/pom.xml +++ b/server/plugins/connect-plugin-api/pom.xml @@ -5,7 +5,7 @@ plugin-parent com.oceanbase - 4.3.2-SNAPSHOT + 4.3.3-SNAPSHOT ../pom.xml 4.0.0 diff --git a/server/plugins/connect-plugin-api/src/main/java/com/oceanbase/odc/plugin/connect/api/TestResult.java b/server/plugins/connect-plugin-api/src/main/java/com/oceanbase/odc/plugin/connect/api/TestResult.java index aa20507701..e2230ee08f 100644 --- a/server/plugins/connect-plugin-api/src/main/java/com/oceanbase/odc/plugin/connect/api/TestResult.java +++ b/server/plugins/connect-plugin-api/src/main/java/com/oceanbase/odc/plugin/connect/api/TestResult.java @@ -84,6 +84,22 @@ public static TestResult initScriptFailed(Throwable throwable) { return fail(ErrorCodes.ConnectionInitScriptFailed, new String[] {message}); } + public static TestResult bucketNotExist(String bucketName) { + return fail(ErrorCodes.BucketNotExist, new String[] {bucketName}); + } + + public static TestResult invalidAccessKeyId(String accessKeyId) { + return fail(ErrorCodes.InvalidAccessKeyId, new String[] {accessKeyId}); + } + + public static TestResult akAccessDenied(String accessKeyId) { + return fail(ErrorCodes.AccessDenied, new String[] {accessKeyId}); + } + + public static TestResult signatureDoesNotMatch(String accessKeyId) { + return fail(ErrorCodes.SignatureDoesNotMatch, new String[] {accessKeyId}); + } + public static TestResult success() { TestResult result = new TestResult(); result.setActive(true); diff --git a/server/plugins/connect-plugin-doris/pom.xml b/server/plugins/connect-plugin-doris/pom.xml index 96581d2f12..7fa1cb09b2 100644 --- a/server/plugins/connect-plugin-doris/pom.xml +++ b/server/plugins/connect-plugin-doris/pom.xml @@ -6,7 +6,7 @@ com.oceanbase plugin-parent - 4.3.2-SNAPSHOT + 4.3.3-SNAPSHOT ../pom.xml diff --git a/server/plugins/connect-plugin-mysql/pom.xml b/server/plugins/connect-plugin-mysql/pom.xml index a10ac956f1..626a25c9d6 100644 --- a/server/plugins/connect-plugin-mysql/pom.xml +++ b/server/plugins/connect-plugin-mysql/pom.xml @@ -6,7 +6,7 @@ com.oceanbase plugin-parent - 4.3.2-SNAPSHOT + 4.3.3-SNAPSHOT ../pom.xml diff --git a/server/plugins/connect-plugin-ob-mysql/pom.xml b/server/plugins/connect-plugin-ob-mysql/pom.xml index 0645701904..156a6b6d33 100644 --- a/server/plugins/connect-plugin-ob-mysql/pom.xml +++ b/server/plugins/connect-plugin-ob-mysql/pom.xml @@ -5,7 +5,7 @@ plugin-parent com.oceanbase - 4.3.2-SNAPSHOT + 4.3.3-SNAPSHOT ../pom.xml 4.0.0 diff --git a/server/plugins/connect-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/connect/obmysql/OBMySQLSessionExtension.java b/server/plugins/connect-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/connect/obmysql/OBMySQLSessionExtension.java index fb4b428ced..2c0002ec08 100644 --- a/server/plugins/connect-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/connect/obmysql/OBMySQLSessionExtension.java +++ b/server/plugins/connect-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/connect/obmysql/OBMySQLSessionExtension.java @@ -31,6 +31,7 @@ import com.oceanbase.odc.plugin.connect.model.DBClientInfo; import com.oceanbase.tools.dbbrowser.util.VersionUtils; +import lombok.NonNull; import lombok.extern.slf4j.Slf4j; /** @@ -46,7 +47,17 @@ public class OBMySQLSessionExtension implements SessionExtensionPoint { @Override public void killQuery(Connection connection, String connectionId) { - JdbcOperationsUtil.getJdbcOperations(connection).execute("KILL QUERY " + connectionId); + JdbcOperationsUtil.getJdbcOperations(connection).execute(getKillQuerySql(connectionId)); + } + + @Override + public String getKillQuerySql(@NonNull String connectionId) { + return "KILL QUERY " + connectionId; + } + + @Override + public String getKillSessionSql(@NonNull String connectionId) { + return "KILL " + connectionId; } @Override diff --git a/server/plugins/connect-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/connect/obmysql/diagnose/PlanGraphBuilder.java b/server/plugins/connect-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/connect/obmysql/diagnose/PlanGraphBuilder.java index a05aa030d1..6dae6fb688 100644 --- a/server/plugins/connect-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/connect/obmysql/diagnose/PlanGraphBuilder.java +++ b/server/plugins/connect-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/connect/obmysql/diagnose/PlanGraphBuilder.java @@ -26,6 +26,8 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.apache.commons.lang3.math.NumberUtils; + import com.oceanbase.odc.common.graph.Graph; import com.oceanbase.odc.common.util.StringUtils; import com.oceanbase.odc.core.shared.exception.UnexpectedException; @@ -83,7 +85,8 @@ private static void parsePlanByJsonMap(Map jsonMap, Graph graph, graph.insertVertex(operator); id2Operator.put(operator.getGraphId(), operator); if (!"-1".equals(parentId)) { - graph.insertEdge(id2Operator.get(parentId), operator, (int) jsonMap.get("EST.ROWS")); + int rows = NumberUtils.isDigits(jsonMap.get("EST.ROWS").toString()) ? (int) jsonMap.get("EST.ROWS") : 0; + graph.insertEdge(id2Operator.get(parentId), operator, rows); } operator.setStatus(QueryStatus.PREPARING); String name = (String) jsonMap.get("NAME"); diff --git a/server/plugins/connect-plugin-ob-oracle/pom.xml b/server/plugins/connect-plugin-ob-oracle/pom.xml index c642b58922..d1b6a0b10e 100644 --- a/server/plugins/connect-plugin-ob-oracle/pom.xml +++ b/server/plugins/connect-plugin-ob-oracle/pom.xml @@ -5,7 +5,7 @@ plugin-parent com.oceanbase - 4.3.2-SNAPSHOT + 4.3.3-SNAPSHOT ../pom.xml 4.0.0 diff --git a/server/plugins/connect-plugin-ob-oracle/src/main/java/com/oceanbase/odc/plugin/connect/oboracle/OBOracleSessionExtension.java b/server/plugins/connect-plugin-ob-oracle/src/main/java/com/oceanbase/odc/plugin/connect/oboracle/OBOracleSessionExtension.java index 2696b256a7..e20da0282c 100644 --- a/server/plugins/connect-plugin-ob-oracle/src/main/java/com/oceanbase/odc/plugin/connect/oboracle/OBOracleSessionExtension.java +++ b/server/plugins/connect-plugin-ob-oracle/src/main/java/com/oceanbase/odc/plugin/connect/oboracle/OBOracleSessionExtension.java @@ -32,6 +32,7 @@ import com.oceanbase.odc.plugin.connect.model.DBClientInfo; import com.oceanbase.tools.dbbrowser.util.VersionUtils; +import lombok.NonNull; import lombok.extern.slf4j.Slf4j; /** @@ -47,7 +48,17 @@ public class OBOracleSessionExtension implements SessionExtensionPoint { @Override public void killQuery(Connection connection, String connectionId) { - JdbcOperationsUtil.getJdbcOperations(connection).execute("KILL QUERY " + connectionId); + JdbcOperationsUtil.getJdbcOperations(connection).execute(getKillQuerySql(connectionId)); + } + + @Override + public String getKillQuerySql(@NonNull String connectionId) { + return "KILL QUERY " + connectionId; + } + + @Override + public String getKillSessionSql(@NonNull String connectionId) { + return "KILL " + connectionId; } @Override diff --git a/server/plugins/connect-plugin-oracle/pom.xml b/server/plugins/connect-plugin-oracle/pom.xml index f7ce69d8be..ac3f6294a2 100644 --- a/server/plugins/connect-plugin-oracle/pom.xml +++ b/server/plugins/connect-plugin-oracle/pom.xml @@ -21,7 +21,7 @@ plugin-parent com.oceanbase - 4.3.2-SNAPSHOT + 4.3.3-SNAPSHOT ../pom.xml 4.0.0 diff --git a/server/plugins/connect-plugin-oracle/src/main/java/com/oceanbase/odc/plugin/connect/oracle/OracleSessionExtension.java b/server/plugins/connect-plugin-oracle/src/main/java/com/oceanbase/odc/plugin/connect/oracle/OracleSessionExtension.java index a702def74c..c1dda63316 100644 --- a/server/plugins/connect-plugin-oracle/src/main/java/com/oceanbase/odc/plugin/connect/oracle/OracleSessionExtension.java +++ b/server/plugins/connect-plugin-oracle/src/main/java/com/oceanbase/odc/plugin/connect/oracle/OracleSessionExtension.java @@ -42,7 +42,17 @@ public class OracleSessionExtension extends OBOracleSessionExtension { @Override public void killQuery(Connection connection, String connectionId) { - JdbcOperationsUtil.getJdbcOperations(connection).execute("ALTER SYSTEM KILL SESSION '" + connectionId + "'"); + JdbcOperationsUtil.getJdbcOperations(connection).execute(getKillQuerySql(connectionId)); + } + + @Override + public String getKillQuerySql(@NonNull String connectionId) { + return this.getKillSessionSql(connectionId); + } + + @Override + public String getKillSessionSql(@NonNull String connectionId) { + return "ALTER SYSTEM KILL SESSION '" + connectionId + "'" + " IMMEDIATE"; } @Override @@ -113,6 +123,4 @@ public boolean setClientInfo(Connection connection, DBClientInfo clientInfo) { return false; } } - - } diff --git a/server/plugins/connect-plugin-postgres/pom.xml b/server/plugins/connect-plugin-postgres/pom.xml index 4d3b4ea999..bd55295235 100644 --- a/server/plugins/connect-plugin-postgres/pom.xml +++ b/server/plugins/connect-plugin-postgres/pom.xml @@ -22,7 +22,7 @@ plugin-parent com.oceanbase - 4.3.2-SNAPSHOT + 4.3.3-SNAPSHOT ../pom.xml connect-plugin-postgres diff --git a/server/plugins/pom.xml b/server/plugins/pom.xml index 053993f888..21e661d754 100644 --- a/server/plugins/pom.xml +++ b/server/plugins/pom.xml @@ -5,7 +5,7 @@ com.oceanbase odc-parent - 4.3.2-SNAPSHOT + 4.3.3-SNAPSHOT ../../pom.xml pom diff --git a/server/plugins/sample-plugin-api/pom.xml b/server/plugins/sample-plugin-api/pom.xml index 5a6f36ccec..e5c7add671 100644 --- a/server/plugins/sample-plugin-api/pom.xml +++ b/server/plugins/sample-plugin-api/pom.xml @@ -5,7 +5,7 @@ com.oceanbase plugin-parent - 4.3.2-SNAPSHOT + 4.3.3-SNAPSHOT ../pom.xml sample-plugin-api diff --git a/server/plugins/sample-plugin/pom.xml b/server/plugins/sample-plugin/pom.xml index 1c85f05790..919e72c4af 100644 --- a/server/plugins/sample-plugin/pom.xml +++ b/server/plugins/sample-plugin/pom.xml @@ -5,7 +5,7 @@ com.oceanbase plugin-parent - 4.3.2-SNAPSHOT + 4.3.3-SNAPSHOT ../pom.xml sample-plugin diff --git a/server/plugins/schema-plugin-api/pom.xml b/server/plugins/schema-plugin-api/pom.xml index 77b3c9a764..4847c90823 100644 --- a/server/plugins/schema-plugin-api/pom.xml +++ b/server/plugins/schema-plugin-api/pom.xml @@ -6,7 +6,7 @@ com.oceanbase plugin-parent - 4.3.2-SNAPSHOT + 4.3.3-SNAPSHOT ../pom.xml diff --git a/server/plugins/schema-plugin-api/src/main/java/com/oceanbase/odc/plugin/schema/api/TableExtensionPoint.java b/server/plugins/schema-plugin-api/src/main/java/com/oceanbase/odc/plugin/schema/api/TableExtensionPoint.java index e0658823c1..2adc68b617 100644 --- a/server/plugins/schema-plugin-api/src/main/java/com/oceanbase/odc/plugin/schema/api/TableExtensionPoint.java +++ b/server/plugins/schema-plugin-api/src/main/java/com/oceanbase/odc/plugin/schema/api/TableExtensionPoint.java @@ -21,6 +21,7 @@ import org.pf4j.ExtensionPoint; import com.oceanbase.tools.dbbrowser.model.DBObjectIdentity; +import com.oceanbase.tools.dbbrowser.model.DBObjectType; import com.oceanbase.tools.dbbrowser.model.DBTable; /** @@ -32,7 +33,7 @@ */ public interface TableExtensionPoint extends ExtensionPoint { - List list(Connection connection, String schemaName); + List list(Connection connection, String schemaName, DBObjectType tableType); List showNamesLike(Connection connection, String schemaName, String tableNameLike); @@ -43,4 +44,6 @@ public interface TableExtensionPoint extends ExtensionPoint { String generateCreateDDL(Connection connection, DBTable table); String generateUpdateDDL(Connection connection, DBTable oldTable, DBTable newTable); + + boolean syncExternalTableFiles(Connection connection, String schemaName, String tableName); } diff --git a/server/plugins/schema-plugin-doris/pom.xml b/server/plugins/schema-plugin-doris/pom.xml index dad1ff7783..4cb0fe5e2c 100644 --- a/server/plugins/schema-plugin-doris/pom.xml +++ b/server/plugins/schema-plugin-doris/pom.xml @@ -6,7 +6,7 @@ com.oceanbase plugin-parent - 4.3.2-SNAPSHOT + 4.3.3-SNAPSHOT ../pom.xml diff --git a/server/plugins/schema-plugin-mysql/pom.xml b/server/plugins/schema-plugin-mysql/pom.xml index 463d216374..e605736f68 100644 --- a/server/plugins/schema-plugin-mysql/pom.xml +++ b/server/plugins/schema-plugin-mysql/pom.xml @@ -6,7 +6,7 @@ com.oceanbase plugin-parent - 4.3.2-SNAPSHOT + 4.3.3-SNAPSHOT ../pom.xml diff --git a/server/plugins/schema-plugin-mysql/src/main/java/com/oceanbase/odc/plugin/schema/mysql/MySQLTableExtension.java b/server/plugins/schema-plugin-mysql/src/main/java/com/oceanbase/odc/plugin/schema/mysql/MySQLTableExtension.java index 06bbe4c002..0488dd4467 100644 --- a/server/plugins/schema-plugin-mysql/src/main/java/com/oceanbase/odc/plugin/schema/mysql/MySQLTableExtension.java +++ b/server/plugins/schema-plugin-mysql/src/main/java/com/oceanbase/odc/plugin/schema/mysql/MySQLTableExtension.java @@ -22,6 +22,7 @@ import com.oceanbase.odc.plugin.schema.mysql.utils.DBAccessorUtil; import com.oceanbase.odc.plugin.schema.obmysql.OBMySQLTableExtension; import com.oceanbase.tools.dbbrowser.editor.DBTableEditor; +import com.oceanbase.tools.dbbrowser.model.DBObjectType; import com.oceanbase.tools.dbbrowser.model.DBTable; import com.oceanbase.tools.dbbrowser.schema.DBSchemaAccessor; import com.oceanbase.tools.dbbrowser.stats.DBStatsAccessor; @@ -44,9 +45,14 @@ public DBTable getDetail(@NonNull Connection connection, @NonNull String schemaN table.setOwner(schemaName); table.setName(tableName); table.setColumns(schemaAccessor.listTableColumns(schemaName, tableName)); - table.setConstraints(schemaAccessor.listTableConstraints(schemaName, tableName)); table.setPartition(schemaAccessor.getPartition(schemaName, tableName)); - table.setIndexes(schemaAccessor.listTableIndexes(schemaName, tableName)); + if (!schemaAccessor.isExternalTable(schemaName, tableName)) { + table.setConstraints(schemaAccessor.listTableConstraints(schemaName, tableName)); + table.setIndexes(schemaAccessor.listTableIndexes(schemaName, tableName)); + table.setType(DBObjectType.TABLE); + } else { + table.setType(DBObjectType.EXTERNAL_TABLE); + } table.setDDL(schemaAccessor.getTableDDL(schemaName, tableName)); table.setTableOptions(schemaAccessor.getTableOptions(schemaName, tableName)); table.setStats(getTableStats(connection, schemaName, tableName)); @@ -68,4 +74,9 @@ protected DBStatsAccessor getStatsAccessor(Connection connection) { return DBAccessorUtil.getStatsAccessor(connection); } + @Override + public boolean syncExternalTableFiles(Connection connection, String schemaName, String tableName) { + throw new UnsupportedOperationException("not implemented yet"); + } + } diff --git a/server/plugins/schema-plugin-ob-mysql/pom.xml b/server/plugins/schema-plugin-ob-mysql/pom.xml index 1655445e1b..246fc7d92b 100644 --- a/server/plugins/schema-plugin-ob-mysql/pom.xml +++ b/server/plugins/schema-plugin-ob-mysql/pom.xml @@ -6,7 +6,7 @@ com.oceanbase plugin-parent - 4.3.2-SNAPSHOT + 4.3.3-SNAPSHOT ../pom.xml schema-plugin-ob-mysql diff --git a/server/plugins/schema-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/schema/obmysql/OBMySQLTableExtension.java b/server/plugins/schema-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/schema/obmysql/OBMySQLTableExtension.java index fec24ae055..62801657bd 100644 --- a/server/plugins/schema-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/schema/obmysql/OBMySQLTableExtension.java +++ b/server/plugins/schema-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/schema/obmysql/OBMySQLTableExtension.java @@ -47,10 +47,26 @@ public class OBMySQLTableExtension implements TableExtensionPoint { @Override - public List list(@NonNull Connection connection, @NonNull String schemaName) { - return getSchemaAccessor(connection).showTables(schemaName).stream().map(item -> { + public List list(@NonNull Connection connection, @NonNull String schemaName, + @NonNull DBObjectType tableType) { + List nameList; + switch (tableType) { + case TABLE: + nameList = getSchemaAccessor(connection).showTables(schemaName); + return generateDBObjectIdentityByTableType(schemaName, nameList, DBObjectType.TABLE); + case EXTERNAL_TABLE: + nameList = getSchemaAccessor(connection).showExternalTables(schemaName); + return generateDBObjectIdentityByTableType(schemaName, nameList, DBObjectType.EXTERNAL_TABLE); + default: + throw new IllegalArgumentException("Unsupported table type: " + tableType); + } + } + + private List generateDBObjectIdentityByTableType(String schemaName, List nameList, + DBObjectType tableType) { + return nameList.stream().map(item -> { DBObjectIdentity identity = new DBObjectIdentity(); - identity.setType(DBObjectType.TABLE); + identity.setType(tableType); identity.setSchemaName(schemaName); identity.setName(item); return identity; @@ -74,9 +90,14 @@ public DBTable getDetail(@NonNull Connection connection, @NonNull String schemaN table.setOwner(schemaName); table.setName(tableName); table.setColumns(schemaAccessor.listTableColumns(schemaName, tableName)); - table.setConstraints(schemaAccessor.listTableConstraints(schemaName, tableName)); + if (!schemaAccessor.isExternalTable(schemaName, tableName)) { + table.setConstraints(schemaAccessor.listTableConstraints(schemaName, tableName)); + table.setIndexes(schemaAccessor.listTableIndexes(schemaName, tableName)); + table.setType(DBObjectType.TABLE); + } else { + table.setType(DBObjectType.EXTERNAL_TABLE); + } table.setPartition(parser.getPartition()); - table.setIndexes(schemaAccessor.listTableIndexes(schemaName, tableName)); table.setDDL(ddl); table.setTableOptions(schemaAccessor.getTableOptions(schemaName, tableName)); table.setStats(getTableStats(connection, schemaName, tableName)); @@ -117,6 +138,13 @@ public String generateUpdateDDL(@NonNull Connection connection, @NonNull DBTable return getTableEditor(connection).generateUpdateObjectDDL(oldTable, newTable); } + @Override + public boolean syncExternalTableFiles(Connection connection, String schemaName, String tableName) { + DBSchemaAccessor schemaAccessor = getSchemaAccessor(connection); + schemaAccessor.syncExternalTableFiles(schemaName, tableName); + return true; + } + protected DBTableEditor getTableEditor(Connection connection) { return DBAccessorUtil.getTableEditor(connection); } diff --git a/server/plugins/schema-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/schema/obmysql/parser/BaseOBGetDBTableByParser.java b/server/plugins/schema-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/schema/obmysql/parser/BaseOBGetDBTableByParser.java new file mode 100644 index 0000000000..5eb5814d80 --- /dev/null +++ b/server/plugins/schema-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/schema/obmysql/parser/BaseOBGetDBTableByParser.java @@ -0,0 +1,267 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.plugin.schema.obmysql.parser; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + +import com.oceanbase.odc.common.util.StringUtils; +import com.oceanbase.tools.dbbrowser.model.DBTableColumn; +import com.oceanbase.tools.dbbrowser.model.DBTableIndex; +import com.oceanbase.tools.dbbrowser.model.DBTablePartition; +import com.oceanbase.tools.dbbrowser.model.DBTablePartitionDefinition; +import com.oceanbase.tools.dbbrowser.model.DBTablePartitionOption; +import com.oceanbase.tools.dbbrowser.model.DBTablePartitionType; +import com.oceanbase.tools.sqlparser.statement.Expression; +import com.oceanbase.tools.sqlparser.statement.createtable.CreateTable; +import com.oceanbase.tools.sqlparser.statement.createtable.Partition; +import com.oceanbase.tools.sqlparser.statement.createtable.PartitionElement; +import com.oceanbase.tools.sqlparser.statement.createtable.SubListPartitionElement; +import com.oceanbase.tools.sqlparser.statement.createtable.SubPartitionElement; +import com.oceanbase.tools.sqlparser.statement.createtable.SubPartitionOption; +import com.oceanbase.tools.sqlparser.statement.createtable.SubRangePartitionElement; +import com.oceanbase.tools.sqlparser.statement.expression.CollectionExpression; +import com.oceanbase.tools.sqlparser.statement.expression.ConstExpression; + +import lombok.NonNull; +import lombok.extern.slf4j.Slf4j; + +/** + * @description: + * @author: zijia.cj + * @date: 2024/11/29 14:34 + * @since: 4.3.3 + */ +@Slf4j +public abstract class BaseOBGetDBTableByParser implements GetDBTableByParser { + + public final DBTablePartition getPartition() { + DBTablePartition partition = new DBTablePartition(); + DBTablePartitionOption partitionOption = new DBTablePartitionOption(); + partitionOption.setType(DBTablePartitionType.NOT_PARTITIONED); + partition.setPartitionOption(partitionOption); + List partitionDefinitions = new ArrayList<>(); + partition.setPartitionDefinitions(partitionDefinitions); + + if (Objects.isNull(getCreateTableStmt())) { + partition.setWarning("Failed to parse table ddl"); + return partition; + } + Partition partitionStmt = getCreateTableStmt().getPartition(); + if (Objects.isNull(partitionStmt)) { + return partition; + } + parsePartitionStmt(partition, partitionStmt); + + /** + * In order to adapt to the front-end only the expression field is used for Hash、List and Range + * partition types + */ + if (Objects.nonNull(partition.getPartitionOption().getType()) + && partition.getPartitionOption().getType().supportExpression() + && StringUtils.isBlank(partition.getPartitionOption().getExpression())) { + List columnNames = partition.getPartitionOption().getColumnNames(); + if (!columnNames.isEmpty()) { + partition.getPartitionOption().setExpression(String.join(", ", columnNames)); + } + } + fillSubPartitions(partition, partitionStmt); + return partition; + } + + private void fillSubPartitions(@NonNull DBTablePartition partition, @NonNull Partition partitionStmt) { + if (partitionStmt.getSubPartitionOption() == null) { + return; + } + DBTablePartition subPartition = new DBTablePartition(); + partition.setSubpartition(subPartition); + DBTablePartitionOption subPartitionOption = new DBTablePartitionOption(); + subPartitionOption.setType(DBTablePartitionType.NOT_PARTITIONED); + subPartition.setPartitionOption(subPartitionOption); + List subPartitionDefinitions = new ArrayList<>(); + subPartition.setPartitionDefinitions(subPartitionDefinitions); + partition.setSubpartitionTemplated(partitionStmt.getSubPartitionOption().getTemplates() != null); + String type = partitionStmt.getSubPartitionOption().getType(); + DBTablePartitionType subDBTablePartitionType = DBTablePartitionType.fromValue(type); + if (DBTablePartitionType.NOT_PARTITIONED == subDBTablePartitionType) { + partition.setWarning("not support this subpartition type, type: " + type); + return; + } + fillSubPartitionOption(partitionStmt, subPartitionOption, subDBTablePartitionType); + fillSubPartitionDefinitions(partition, partitionStmt, subPartitionDefinitions, subDBTablePartitionType); + } + + private void fillSubPartitionOption(@NonNull Partition partitionStmt, + @NonNull DBTablePartitionOption subPartitionOption, + @NonNull DBTablePartitionType subDBTablePartitionType) { + subPartitionOption.setType(subDBTablePartitionType); + SubPartitionOption parsedSubPartitionOption = partitionStmt.getSubPartitionOption(); + fillSubPartitionKey(subPartitionOption, subDBTablePartitionType, parsedSubPartitionOption); + /** + * 
+         * subpartitionsNum indicates the number of subpartitions in each partition.
+         * Therefore, subpartitionsNum should be configured only when the subpartitions are templated.
+         * If subpartition is not templated, the subpartitionsNum is not fixed.
+         * such as the following example of non-template subpartition table
+         *
+         * CREATE TABLE ranges_list (col1 INT,col2 INT)
+         *        PARTITION BY RANGE COLUMNS(col1)
+         *        SUBPARTITION BY LIST(col2)
+         *        (PARTITION p0 VALUES LESS THAN(100)
+         *          (SUBPARTITION sp0 VALUES IN(1,3),
+         *           SUBPARTITION sp1 VALUES IN(4,6),
+         *           SUBPARTITION sp2 VALUES IN(7,9)),
+         *         PARTITION p1 VALUES LESS THAN(200)
+         *          (SUBPARTITION sp3 VALUES IN(1,3))
+         *        );
+         *
+         * In this case, the subpartitionsNum is not fixed.
+         *
+ */ + subPartitionOption + .setPartitionsNum( + parsedSubPartitionOption.getTemplates() != null ? parsedSubPartitionOption.getTemplates().size() + : null); + } + + private void fillSubPartitionDefinitions(@NonNull DBTablePartition partition, @NonNull Partition partitionStmt, + @NonNull List subPartitionDefinitions, + @NonNull DBTablePartitionType subDBTablePartitionType) { + List partitionElements = partitionStmt.getPartitionElements(); + if (partitionElements == null || partitionElements.isEmpty()) { + log.warn("no partitions found in partition statement"); + return; + } + for (int i = 0; i < partitionElements.size(); i++) { + DBTablePartitionDefinition partitionDefinition = partition.getPartitionDefinitions().get(i); + PartitionElement partitionElement = partitionElements.get(i); + if (partitionElement == null) { + continue; + } + List subPartitionElements = partitionElement.getSubPartitionElements(); + if (subPartitionElements != null) { + fillNonTemplatedSubPartitionDefinitions(subPartitionDefinitions, subDBTablePartitionType, + partitionElement, partitionDefinition); + } else { + fillTemplatedSubPartitionDefinitions(partitionStmt, subPartitionDefinitions, subDBTablePartitionType, + partitionElement, partitionDefinition); + } + } + } + + private void fillNonTemplatedSubPartitionDefinitions( + @NonNull List subPartitionDefinitions, + @NonNull DBTablePartitionType subDBTablePartitionType, @NonNull PartitionElement partitionElement, + @NonNull DBTablePartitionDefinition partitionDefinition) { + List subPartitionElements = partitionElement.getSubPartitionElements(); + if (subPartitionElements == null || subPartitionElements.isEmpty()) { + log.warn("no non-templated sub-partitions found for partition {}", partitionDefinition.getName()); + return; + } + for (int j = 0; j < subPartitionElements.size(); j++) { + SubPartitionElement subPartitionElement = subPartitionElements.get(j); + if (subPartitionElement == null) { + continue; + } + DBTablePartitionDefinition subPartitionDefinition = new DBTablePartitionDefinition(); + fillSubPartitionValue(subPartitionElement, subPartitionDefinition); + subPartitionDefinition.setParentPartitionDefinition(partitionDefinition); + subPartitionDefinition.setName( + removeIdentifiers(subPartitionElement.getRelation())); + subPartitionDefinition.setOrdinalPosition(j); + subPartitionDefinition.setType(subDBTablePartitionType); + subPartitionDefinitions.add(subPartitionDefinition); + } + } + + private void fillTemplatedSubPartitionDefinitions(@NonNull Partition partitionStmt, + @NonNull List subPartitionDefinitions, + @NonNull DBTablePartitionType subDBTablePartitionType, @NonNull PartitionElement partitionElement, + @NonNull DBTablePartitionDefinition partitionDefinition) { + String parentPartitionName = removeIdentifiers(partitionElement.getRelation()); + List templates = partitionStmt.getSubPartitionOption().getTemplates(); + if (templates == null || templates.isEmpty()) { + log.warn("no templated sub-partitions found for partition {}", parentPartitionName); + return; + } + for (int j = 0; j < templates.size(); j++) { + SubPartitionElement subPartitionElement = templates.get(j); + if (subPartitionElement == null) { + continue; + } + DBTablePartitionDefinition subPartitionDefinition = new DBTablePartitionDefinition(); + fillSubPartitionValue(subPartitionElement, subPartitionDefinition); + subPartitionDefinition.setParentPartitionDefinition(partitionDefinition); + subPartitionDefinition.setName( + generateTemplateSubPartitionName(parentPartitionName, subPartitionElement.getRelation())); + subPartitionDefinition.setOrdinalPosition(j); + subPartitionDefinition.setType(subDBTablePartitionType); + subPartitionDefinitions.add(subPartitionDefinition); + } + } + + private void fillSubPartitionValue(@NonNull SubPartitionElement subPartitionElement, + @NonNull DBTablePartitionDefinition subPartitionDefinition) { + if (subPartitionElement instanceof SubListPartitionElement) { + SubListPartitionElement subListPartitionElement = + (SubListPartitionElement) subPartitionElement; + List> valuesList = new ArrayList<>(); + for (Expression listExpr : subListPartitionElement.getListExprs()) { + if (listExpr instanceof CollectionExpression) { + valuesList.add( + ((CollectionExpression) listExpr).getExpressionList().stream() + .map(Expression::getText) + .collect(Collectors.toList())); + } else if (listExpr instanceof ConstExpression) { + valuesList.add(Collections.singletonList(listExpr.getText())); + } + } + subPartitionDefinition.setValuesList(valuesList); + } else if (subPartitionElement instanceof SubRangePartitionElement) { + SubRangePartitionElement subRangePartitionElement = + (SubRangePartitionElement) subPartitionElement; + subPartitionDefinition.setMaxValues( + subRangePartitionElement.getRangeExprs().stream().map(Expression::getText) + .collect(Collectors.toList())); + } + } + + protected abstract String generateTemplateSubPartitionName(String partitionName, String subPartitionName); + + protected abstract void fillSubPartitionKey(DBTablePartitionOption subPartitionOption, + DBTablePartitionType subDBTablePartitionType, + SubPartitionOption parsedSubPartitionOption); + + protected abstract void parsePartitionStmt(DBTablePartition partition, Partition partitionStmt); + + protected abstract String removeIdentifiers(String str); + + protected abstract CreateTable getCreateTableStmt(); + + @Override + public List listIndexes() { + throw new UnsupportedOperationException("Not supported yet"); + } + + @Override + public List listColumns() { + throw new UnsupportedOperationException("Not supported yet"); + } + +} diff --git a/server/plugins/schema-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/schema/obmysql/parser/OBMySQLGetDBTableByParser.java b/server/plugins/schema-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/schema/obmysql/parser/OBMySQLGetDBTableByParser.java index d5fe07f718..19625a8dbd 100644 --- a/server/plugins/schema-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/schema/obmysql/parser/OBMySQLGetDBTableByParser.java +++ b/server/plugins/schema-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/schema/obmysql/parser/OBMySQLGetDBTableByParser.java @@ -18,16 +18,13 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.Objects; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import com.oceanbase.odc.common.util.StringUtils; import com.oceanbase.tools.dbbrowser.model.DBConstraintType; import com.oceanbase.tools.dbbrowser.model.DBForeignKeyModifyRule; -import com.oceanbase.tools.dbbrowser.model.DBTableColumn; import com.oceanbase.tools.dbbrowser.model.DBTableConstraint; -import com.oceanbase.tools.dbbrowser.model.DBTableIndex; import com.oceanbase.tools.dbbrowser.model.DBTablePartition; import com.oceanbase.tools.dbbrowser.model.DBTablePartitionDefinition; import com.oceanbase.tools.dbbrowser.model.DBTablePartitionOption; @@ -49,7 +46,10 @@ import com.oceanbase.tools.sqlparser.statement.createtable.Partition; import com.oceanbase.tools.sqlparser.statement.createtable.RangePartition; import com.oceanbase.tools.sqlparser.statement.createtable.RangePartitionElement; +import com.oceanbase.tools.sqlparser.statement.createtable.SubListPartitionElement; +import com.oceanbase.tools.sqlparser.statement.createtable.SubPartitionElement; import com.oceanbase.tools.sqlparser.statement.createtable.SubPartitionOption; +import com.oceanbase.tools.sqlparser.statement.createtable.SubRangePartitionElement; import com.oceanbase.tools.sqlparser.statement.createtable.TableElement; import com.oceanbase.tools.sqlparser.statement.expression.CollectionExpression; import com.oceanbase.tools.sqlparser.statement.expression.ColumnReference; @@ -64,7 +64,7 @@ * @since 4.2.0 */ @Slf4j -public class OBMySQLGetDBTableByParser implements GetDBTableByParser { +public class OBMySQLGetDBTableByParser extends BaseOBGetDBTableByParser { private final CreateTable createTableStmt; private static final char MYSQL_IDENTIFIER_WRAP_CHAR = '`'; @@ -87,8 +87,8 @@ private CreateTable parseTableDDL(String ddl) { } @Override - public List listColumns() { - throw new UnsupportedOperationException("Not supported yet"); + public CreateTable getCreateTableStmt() { + return this.createTableStmt; } /** @@ -184,41 +184,37 @@ public List listConstraints() { return constraints; } - private String removeIdentifiers(String str) { + protected String removeIdentifiers(String str) { return StringUtils.unwrap(str, MYSQL_IDENTIFIER_WRAP_CHAR); } @Override - public List listIndexes() { - throw new UnsupportedOperationException("Not supported yet"); + protected String generateTemplateSubPartitionName(String partitionName, String subPartitionName) { + return removeIdentifiers(partitionName) + 's' + removeIdentifiers(subPartitionName); } @Override - public DBTablePartition getPartition() { - DBTablePartition partition = new DBTablePartition(); - DBTablePartition subPartition = new DBTablePartition(); - partition.setSubpartition(subPartition); - - DBTablePartitionOption partitionOption = new DBTablePartitionOption(); - partitionOption.setType(DBTablePartitionType.NOT_PARTITIONED); - partition.setPartitionOption(partitionOption); - DBTablePartitionOption subPartitionOption = new DBTablePartitionOption(); - subPartitionOption.setType(DBTablePartitionType.NOT_PARTITIONED); - subPartition.setPartitionOption(subPartitionOption); - - List partitionDefinitions = new ArrayList<>(); - partition.setPartitionDefinitions(partitionDefinitions); - List subPartitionDefinitions = new ArrayList<>(); - subPartition.setPartitionDefinitions(subPartitionDefinitions); - - if (Objects.isNull(createTableStmt)) { - partition.setWarning("Failed to parse table ddl"); - return partition; - } - Partition partitionStmt = createTableStmt.getPartition(); - if (Objects.isNull(partitionStmt)) { - return partition; + protected void fillSubPartitionKey(DBTablePartitionOption subPartitionOption, + DBTablePartitionType subDBTablePartitionType, SubPartitionOption parsedSubPartitionOption) { + // When expressions are supported, only single partition keys are supported + if (subDBTablePartitionType.supportExpression()) { + Expression expression = parsedSubPartitionOption.getSubPartitionTargets().get(0); + if (expression instanceof ColumnReference) { + subPartitionOption.setColumnNames(Collections.singletonList(removeIdentifiers(expression.getText()))); + } else { + subPartitionOption.setExpression(expression.getText()); + } + } else { + // When expressions are not supported, multiple columns are supported as partition keys + subPartitionOption.setColumnNames(parsedSubPartitionOption.getSubPartitionTargets() == null ? null + : parsedSubPartitionOption.getSubPartitionTargets().stream() + .map(item -> removeIdentifiers(item.getText())) + .collect(Collectors.toList())); } + } + + @Override + protected void parsePartitionStmt(DBTablePartition partition, Partition partitionStmt) { if (partitionStmt instanceof HashPartition) { parseHashPartitionStmt((HashPartition) partitionStmt, partition); } else if (partitionStmt instanceof KeyPartition) { @@ -228,49 +224,32 @@ public DBTablePartition getPartition() { } else if (partitionStmt instanceof ListPartition) { parseListPartitionStmt((ListPartition) partitionStmt, partition); } + } - /** - * In order to adapt to the front-end only the expression field is used for Hash、List and Range - * partition types - */ - if (Objects.nonNull(partition.getPartitionOption().getType()) - && partition.getPartitionOption().getType().supportExpression() - && StringUtils.isBlank(partition.getPartitionOption().getExpression())) { - List columnNames = partition.getPartitionOption().getColumnNames(); - if (!columnNames.isEmpty()) { - partition.getPartitionOption().setExpression(String.join(", ", columnNames)); + private void fillSubPartitionValue(SubPartitionElement subPartitionElement, + DBTablePartitionDefinition subPartitionDefinition) { + if (subPartitionElement instanceof SubListPartitionElement) { + SubListPartitionElement subListPartitionElement = + (SubListPartitionElement) subPartitionElement; + List> valuesList = new ArrayList<>(); + for (Expression listExpr : subListPartitionElement.getListExprs()) { + if (listExpr instanceof CollectionExpression) { + valuesList.add( + ((CollectionExpression) listExpr).getExpressionList().stream() + .map(Expression::getText) + .collect(Collectors.toList())); + } else if (listExpr instanceof ConstExpression) { + valuesList.add(Collections.singletonList(listExpr.getText())); + } } - } - if (partitionStmt.getSubPartitionOption() == null) { - return partition; - } - // TODO 目前 ODC 仅支持 HASH/KEY 二级分区, 其它类型后续需补充 - partition.setSubpartitionTemplated(partitionStmt.getSubPartitionOption().getTemplates() != null); - SubPartitionOption subOption = partitionStmt.getSubPartitionOption(); - String type = partitionStmt.getSubPartitionOption().getType(); - if ("key".equals(type.toLowerCase())) { - subPartitionOption.setType(DBTablePartitionType.KEY); - subPartitionOption.setColumnNames(subOption.getSubPartitionTargets() == null ? null - : subOption.getSubPartitionTargets().stream().map(item -> removeIdentifiers(item.getText())) + subPartitionDefinition.setValuesList(valuesList); + } else if (subPartitionElement instanceof SubRangePartitionElement) { + SubRangePartitionElement subRangePartitionElement = + (SubRangePartitionElement) subPartitionElement; + subPartitionDefinition.setMaxValues( + subRangePartitionElement.getRangeExprs().stream().map(Expression::getText) .collect(Collectors.toList())); - subPartitionOption - .setPartitionsNum(partition.getSubpartitionTemplated() ? subOption.getTemplates().size() - : partitionStmt.getPartitionElements().get(0).getSubPartitionElements().size()); - } else if ("hash".equals(type.toLowerCase())) { - subPartitionOption.setType(DBTablePartitionType.HASH); - Expression expression = subOption.getSubPartitionTargets().get(0); - if (expression instanceof ColumnReference) { - subPartitionOption.setColumnNames(Collections.singletonList(removeIdentifiers(expression.getText()))); - } else { - subPartitionOption.setExpression(expression.getText()); - } - subPartitionOption - .setPartitionsNum(partition.getSubpartitionTemplated() ? subOption.getTemplates().size() - : partitionStmt.getPartitionElements().get(0).getSubPartitionElements().size()); - } else { - partition.setWarning("Only support HASH/KEY subpartition currently"); } - return partition; } private void parseHashPartitionStmt(HashPartition statement, DBTablePartition partition) { diff --git a/server/plugins/schema-plugin-ob-mysql/src/test/java/com/oceanbase/odc/plugin/schema/obmysql/parser/OBMySQLGetDBTableByParserTest.java b/server/plugins/schema-plugin-ob-mysql/src/test/java/com/oceanbase/odc/plugin/schema/obmysql/parser/OBMySQLGetDBTableByParserTest.java index fb460606f2..7701bd0466 100644 --- a/server/plugins/schema-plugin-ob-mysql/src/test/java/com/oceanbase/odc/plugin/schema/obmysql/parser/OBMySQLGetDBTableByParserTest.java +++ b/server/plugins/schema-plugin-ob-mysql/src/test/java/com/oceanbase/odc/plugin/schema/obmysql/parser/OBMySQLGetDBTableByParserTest.java @@ -230,7 +230,7 @@ public void getPartition_secondary_key_no_template_Success() { Assert.assertEquals("col3", partition.getSubpartition().getPartitionOption().getColumnNames().get(1)); Assert.assertEquals(DBTablePartitionType.KEY, partition.getSubpartition().getPartitionOption().getType()); Assert.assertEquals(2, partition.getSubpartition().getPartitionOption().getColumnNames().size()); - Assert.assertTrue(partition.getSubpartition().getPartitionOption().getPartitionsNum() == 3); + Assert.assertNull(partition.getSubpartition().getPartitionOption().getPartitionsNum()); } @Test @@ -278,7 +278,7 @@ public void getPartition_secondary_hash_no_template_Success() { Assert.assertEquals(false, partition.getSubpartitionTemplated()); Assert.assertEquals(DBTablePartitionType.HASH, partition.getSubpartition().getPartitionOption().getType()); Assert.assertEquals("col2", partition.getSubpartition().getPartitionOption().getColumnNames().get(0)); - Assert.assertTrue(partition.getSubpartition().getPartitionOption().getPartitionsNum() == 3); + Assert.assertNull(partition.getSubpartition().getPartitionOption().getPartitionsNum()); } @Test @@ -345,4 +345,660 @@ public void getConstraints_test_out_line_constraints_with_idx_name_Success() { Assert.assertEquals(constraints.get(0).getName(), "idx_name"); Assert.assertEquals(constraints.get(0).getType(), DBConstraintType.UNIQUE_KEY); } + + @Test + public void getSubpartition_RangeColumnsAndRange_ColumnKey_Template_Success() { + String ddl = "CREATE TABLE t_ranges_range (col1 INT NOT NULL,col2 varchar(50),col3 INT NOT NULL) \n" + + "PARTITION BY RANGE COLUMNS(col1)\n" + + "SUBPARTITION BY RANGE(col3)\n" + + "SUBPARTITION TEMPLATE \n" + + "(SUBPARTITION mp0 VALUES LESS THAN(1000),\n" + + " SUBPARTITION mp1 VALUES LESS THAN(2000),\n" + + " SUBPARTITION mp2 VALUES LESS THAN(3000)\n" + + ")\n" + + "(PARTITION p0 VALUES LESS THAN(100),\n" + + " PARTITION p1 VALUES LESS THAN(200),\n" + + " PARTITION p2 VALUES LESS THAN(300)\n" + + "); \n"; + OBMySQLGetDBTableByParser table = new OBMySQLGetDBTableByParser(ddl); + DBTablePartition partition = table.getPartition(); + Assert.assertEquals(DBTablePartitionType.RANGE_COLUMNS, partition.getPartitionOption().getType()); + Assert.assertEquals(1, partition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("col1", partition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals(3L, partition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals("p0", partition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals("100", partition.getPartitionDefinitions().get(0).getMaxValues().get(0)); + DBTablePartition subpartition = partition.getSubpartition(); + Assert.assertEquals(DBTablePartitionType.RANGE, subpartition.getPartitionOption().getType()); + Assert.assertEquals(1, subpartition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("col3", subpartition.getPartitionOption().getColumnNames().get(0)); + Assert.assertNull(subpartition.getPartitionOption().getExpression()); + Assert.assertTrue(partition.getSubpartitionTemplated()); + Assert.assertEquals(3L, subpartition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals("p0smp0", subpartition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals(1, subpartition.getPartitionDefinitions().get(0).getMaxValues().size()); + Assert.assertEquals("1000", subpartition.getPartitionDefinitions().get(0).getMaxValues().get(0)); + Assert.assertEquals("p0", + subpartition.getPartitionDefinitions().get(0).getParentPartitionDefinition().getName()); + } + + @Test + public void getSubpartition_RangeColumnsAndRange_ExpressionKey_Template_Success() { + String ddl = "CREATE TABLE t_ranges_range_expr (col1 INT NOT NULL,col2 varchar(50),col3 INT NOT NULL) \n" + + "PARTITION BY RANGE COLUMNS(col1)\n" + + "SUBPARTITION BY RANGE(ABS(col3))\n" + + "SUBPARTITION TEMPLATE \n" + + "(SUBPARTITION mp0 VALUES LESS THAN(1000),\n" + + " SUBPARTITION mp1 VALUES LESS THAN(2000),\n" + + " SUBPARTITION mp2 VALUES LESS THAN(3000)\n" + + ")\n" + + "(PARTITION p0 VALUES LESS THAN(100),\n" + + " PARTITION p1 VALUES LESS THAN(200),\n" + + " PARTITION p2 VALUES LESS THAN(300)\n" + + "); \n"; + OBMySQLGetDBTableByParser table = new OBMySQLGetDBTableByParser(ddl); + DBTablePartition partition = table.getPartition(); + Assert.assertEquals(DBTablePartitionType.RANGE_COLUMNS, partition.getPartitionOption().getType()); + Assert.assertEquals(1, partition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("col1", partition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals(3L, partition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals("p0", partition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals("100", partition.getPartitionDefinitions().get(0).getMaxValues().get(0)); + DBTablePartition subpartition = partition.getSubpartition(); + Assert.assertEquals(DBTablePartitionType.RANGE, subpartition.getPartitionOption().getType()); + Assert.assertEquals("ABS(col3)", subpartition.getPartitionOption().getExpression()); + Assert.assertTrue(partition.getSubpartitionTemplated()); + Assert.assertEquals(3L, subpartition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals("p0smp0", subpartition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals(1, subpartition.getPartitionDefinitions().get(0).getMaxValues().size()); + Assert.assertEquals("1000", subpartition.getPartitionDefinitions().get(0).getMaxValues().get(0)); + Assert.assertEquals("p0", + subpartition.getPartitionDefinitions().get(0).getParentPartitionDefinition().getName()); + } + + @Test + public void getSubpartition_RangeAndRange_ColumnKey_NoTemplate_Success() { + String ddl = "CREATE TABLE range_range(col1 INT,col2 INT) \n" + + " PARTITION BY RANGE(col1)\n" + + " SUBPARTITION BY RANGE(col2)\n" + + " (PARTITION p0 VALUES LESS THAN(100)\n" + + " (SUBPARTITION sp0 VALUES LESS THAN(100),\n" + + " SUBPARTITION sp1 VALUES LESS THAN(200),\n" + + " SUBPARTITION sp2 VALUES LESS THAN(300),\n" + + " SUBPARTITION sp3 VALUES LESS THAN(400)\n" + + " ),\n" + + " PARTITION p1 VALUES LESS THAN(200)\n" + + " (SUBPARTITION sp4 VALUES LESS THAN(100),\n" + + " SUBPARTITION sp5 VALUES LESS THAN(200),\n" + + " SUBPARTITION sp6 VALUES LESS THAN(300),\n" + + " SUBPARTITION sp7 VALUES LESS THAN(400)\n" + + " )\n" + + " );\n"; + OBMySQLGetDBTableByParser table = new OBMySQLGetDBTableByParser(ddl); + DBTablePartition partition = table.getPartition(); + Assert.assertEquals(DBTablePartitionType.RANGE, partition.getPartitionOption().getType()); + Assert.assertEquals(1, partition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("col1", partition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals(2L, partition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals("p0", partition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals("100", partition.getPartitionDefinitions().get(0).getMaxValues().get(0)); + DBTablePartition subpartition = partition.getSubpartition(); + Assert.assertEquals(DBTablePartitionType.RANGE, subpartition.getPartitionOption().getType()); + Assert.assertEquals(1, subpartition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("col2", subpartition.getPartitionOption().getColumnNames().get(0)); + Assert.assertNull(subpartition.getPartitionOption().getExpression()); + Assert.assertFalse(partition.getSubpartitionTemplated()); + Assert.assertNull(subpartition.getPartitionOption().getPartitionsNum()); + Assert.assertEquals("sp0", subpartition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals(1, subpartition.getPartitionDefinitions().get(0).getMaxValues().size()); + Assert.assertEquals("100", subpartition.getPartitionDefinitions().get(0).getMaxValues().get(0)); + Assert.assertEquals("p0", + subpartition.getPartitionDefinitions().get(0).getParentPartitionDefinition().getName()); + } + + @Test + public void getSubpartition_RangeAndRange_ExpressionKey_NoTemplate_Success() { + String ddl = "CREATE TABLE range_range_expr(col1 INT,col2 TIMESTAMP) \n" + + " PARTITION BY RANGE(col1)\n" + + " SUBPARTITION BY RANGE(UNIX_TIMESTAMP(col2))\n" + + " (PARTITION p0 VALUES LESS THAN(100)\n" + + " (SUBPARTITION sp0 VALUES LESS THAN(UNIX_TIMESTAMP('2021/04/01')),\n" + + " SUBPARTITION sp1 VALUES LESS THAN(UNIX_TIMESTAMP('2021/07/01')),\n" + + " SUBPARTITION sp2 VALUES LESS THAN(UNIX_TIMESTAMP('2021/10/01')),\n" + + " SUBPARTITION sp3 VALUES LESS THAN(UNIX_TIMESTAMP('2022/01/01'))\n" + + " ),\n" + + " PARTITION p1 VALUES LESS THAN(200)\n" + + " (SUBPARTITION sp4 VALUES LESS THAN(UNIX_TIMESTAMP('2021/04/01')),\n" + + " SUBPARTITION sp5 VALUES LESS THAN(UNIX_TIMESTAMP('2021/07/01')),\n" + + " SUBPARTITION sp6 VALUES LESS THAN(UNIX_TIMESTAMP('2021/10/01')),\n" + + " SUBPARTITION sp7 VALUES LESS THAN(UNIX_TIMESTAMP('2022/01/01'))\n" + + " )\n" + + " );\n"; + OBMySQLGetDBTableByParser table = new OBMySQLGetDBTableByParser(ddl); + DBTablePartition partition = table.getPartition(); + Assert.assertEquals(DBTablePartitionType.RANGE, partition.getPartitionOption().getType()); + Assert.assertEquals(1, partition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("col1", partition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals(2L, partition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals("p0", partition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals("100", partition.getPartitionDefinitions().get(0).getMaxValues().get(0)); + DBTablePartition subpartition = partition.getSubpartition(); + Assert.assertEquals(DBTablePartitionType.RANGE, subpartition.getPartitionOption().getType()); + Assert.assertEquals("UNIX_TIMESTAMP(col2)", subpartition.getPartitionOption().getExpression()); + Assert.assertFalse(partition.getSubpartitionTemplated()); + Assert.assertNull(subpartition.getPartitionOption().getPartitionsNum()); + Assert.assertEquals("sp0", subpartition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals(1, subpartition.getPartitionDefinitions().get(0).getMaxValues().size()); + Assert.assertEquals("UNIX_TIMESTAMP('2021/04/01')", + subpartition.getPartitionDefinitions().get(0).getMaxValues().get(0)); + Assert.assertEquals("p0", + subpartition.getPartitionDefinitions().get(0).getParentPartitionDefinition().getName()); + } + + @Test + public void getSubpartition_RangeColumnsAndRangeColumns_ColumnKey_Template_Success() { + String ddl = "CREATE TABLE t_ranges_ranges(col1 INT,col2 INT,col3 INT) \n" + + " PARTITION BY RANGE COLUMNS(col1)\n" + + " SUBPARTITION BY RANGE COLUMNS(col2,col3)\n" + + " SUBPARTITION TEMPLATE \n" + + " (SUBPARTITION mp0 VALUES LESS THAN(1000,1000),\n" + + " SUBPARTITION mp1 VALUES LESS THAN(2000,2000),\n" + + " SUBPARTITION mp2 VALUES LESS THAN(3000,3000)\n" + + " )\n" + + " (PARTITION p0 VALUES LESS THAN(100),\n" + + " PARTITION p1 VALUES LESS THAN(200),\n" + + " PARTITION p2 VALUES LESS THAN(300)\n" + + " ); "; + OBMySQLGetDBTableByParser table = new OBMySQLGetDBTableByParser(ddl); + DBTablePartition partition = table.getPartition(); + Assert.assertEquals(DBTablePartitionType.RANGE_COLUMNS, partition.getPartitionOption().getType()); + Assert.assertEquals(1, partition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("col1", partition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals(3, partition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals("p0", partition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals("100", partition.getPartitionDefinitions().get(0).getMaxValues().get(0)); + DBTablePartition subpartition = partition.getSubpartition(); + Assert.assertEquals(DBTablePartitionType.RANGE_COLUMNS, subpartition.getPartitionOption().getType()); + Assert.assertEquals(2, subpartition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("col2", subpartition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals("col3", subpartition.getPartitionOption().getColumnNames().get(1)); + Assert.assertNull(subpartition.getPartitionOption().getExpression()); + Assert.assertTrue(partition.getSubpartitionTemplated()); + Assert.assertEquals(3L, subpartition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals("p0smp0", subpartition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals(2, subpartition.getPartitionDefinitions().get(0).getMaxValues().size()); + Assert.assertEquals("1000", subpartition.getPartitionDefinitions().get(0).getMaxValues().get(0)); + Assert.assertEquals("1000", subpartition.getPartitionDefinitions().get(0).getMaxValues().get(1)); + Assert.assertEquals("p0", + subpartition.getPartitionDefinitions().get(0).getParentPartitionDefinition().getName()); + } + + @Test + public void getSubpartition_RangeColumnsAndRangeColumns_ColumnKey_NoTemplate_Success() { + String ddl = "CREATE TABLE ranges_ranges (col1 INT NOT NULL,col2 INT NOT NULL,col3 INT NOT NULL) \n" + + "PARTITION BY RANGE COLUMNS(col1)\n" + + "SUBPARTITION BY RANGE COLUMNS(col2,col3)\n" + + "(PARTITION p0 VALUES LESS THAN(100)\n" + + " (SUBPARTITION sp0 VALUES LESS THAN(1000,1000),\n" + + " SUBPARTITION sp1 VALUES LESS THAN(2000,2000),\n" + + " SUBPARTITION sp2 VALUES LESS THAN(3000,3000)),\n" + + " PARTITION p1 VALUES LESS THAN(200)\n" + + " (SUBPARTITION sp3 VALUES LESS THAN(1000,1000),\n" + + " SUBPARTITION sp4 VALUES LESS THAN(2000,2000),\n" + + " SUBPARTITION sp5 VALUES LESS THAN(3000,3000))\n" + + ");\n"; + OBMySQLGetDBTableByParser table = new OBMySQLGetDBTableByParser(ddl); + DBTablePartition partition = table.getPartition(); + Assert.assertEquals(DBTablePartitionType.RANGE_COLUMNS, partition.getPartitionOption().getType()); + Assert.assertEquals(1, partition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("col1", partition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals(2, partition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals("p0", partition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals("100", partition.getPartitionDefinitions().get(0).getMaxValues().get(0)); + DBTablePartition subpartition = partition.getSubpartition(); + Assert.assertEquals(DBTablePartitionType.RANGE_COLUMNS, subpartition.getPartitionOption().getType()); + Assert.assertEquals(2, subpartition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("col2", subpartition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals("col3", subpartition.getPartitionOption().getColumnNames().get(1)); + Assert.assertNull(subpartition.getPartitionOption().getExpression()); + Assert.assertFalse(partition.getSubpartitionTemplated()); + Assert.assertNull(subpartition.getPartitionOption().getPartitionsNum()); + Assert.assertEquals("sp0", subpartition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals(2, subpartition.getPartitionDefinitions().get(0).getMaxValues().size()); + Assert.assertEquals("1000", subpartition.getPartitionDefinitions().get(0).getMaxValues().get(0)); + Assert.assertEquals("1000", subpartition.getPartitionDefinitions().get(0).getMaxValues().get(1)); + Assert.assertEquals("p0", + subpartition.getPartitionDefinitions().get(0).getParentPartitionDefinition().getName()); + } + + @Test + public void getSubpartition_RangeColumnsAndList_ColumnKey_Template_Success() { + String ddl = "CREATE TABLE t_ranges_list(col1 INT,col2 INT) \n" + + " PARTITION BY RANGE COLUMNS(col1)\n" + + " SUBPARTITION BY LIST(col2)\n" + + " SUBPARTITION TEMPLATE \n" + + " (SUBPARTITION mp0 VALUES IN(1,3),\n" + + " SUBPARTITION mp1 VALUES IN(4,6),\n" + + " SUBPARTITION mp2 VALUES IN(7)\n" + + " )\n" + + " (PARTITION p0 VALUES LESS THAN(100),\n" + + " PARTITION p1 VALUES LESS THAN(200),\n" + + " PARTITION p2 VALUES LESS THAN(300)\n" + + " ); \n"; + OBMySQLGetDBTableByParser table = new OBMySQLGetDBTableByParser(ddl); + DBTablePartition partition = table.getPartition(); + Assert.assertEquals(DBTablePartitionType.RANGE_COLUMNS, partition.getPartitionOption().getType()); + Assert.assertEquals(1, partition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("col1", partition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals(3, partition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals("p0", partition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals("100", partition.getPartitionDefinitions().get(0).getMaxValues().get(0)); + DBTablePartition subpartition = partition.getSubpartition(); + Assert.assertEquals(DBTablePartitionType.LIST, subpartition.getPartitionOption().getType()); + Assert.assertEquals(1, subpartition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("col2", subpartition.getPartitionOption().getColumnNames().get(0)); + Assert.assertNull(subpartition.getPartitionOption().getExpression()); + Assert.assertTrue(partition.getSubpartitionTemplated()); + Assert.assertEquals(3L, subpartition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals("p0smp0", subpartition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals(2, subpartition.getPartitionDefinitions().get(0).getValuesList().size()); + Assert.assertEquals("1", subpartition.getPartitionDefinitions().get(0).getValuesList().get(0).get(0)); + Assert.assertEquals("3", subpartition.getPartitionDefinitions().get(0).getValuesList().get(1).get(0)); + Assert.assertEquals("p0", + subpartition.getPartitionDefinitions().get(0).getParentPartitionDefinition().getName()); + } + + @Test + public void getSubpartition_RangeColumnsAndList_ExpressionKey_Template_Success() { + String ddl = "CREATE TABLE t_ranges_list_expr(col1 INT,col2 INT) \n" + + " PARTITION BY RANGE COLUMNS(col1)\n" + + " SUBPARTITION BY LIST(abs(col2))\n" + + " SUBPARTITION TEMPLATE \n" + + " (SUBPARTITION mp0 VALUES IN(1,3),\n" + + " SUBPARTITION mp1 VALUES IN(4,6),\n" + + " SUBPARTITION mp2 VALUES IN(7)\n" + + " )\n" + + " (PARTITION p0 VALUES LESS THAN(100),\n" + + " PARTITION p1 VALUES LESS THAN(200),\n" + + " PARTITION p2 VALUES LESS THAN(300)\n" + + " ); \n"; + OBMySQLGetDBTableByParser table = new OBMySQLGetDBTableByParser(ddl); + DBTablePartition partition = table.getPartition(); + Assert.assertEquals(DBTablePartitionType.RANGE_COLUMNS, partition.getPartitionOption().getType()); + Assert.assertEquals(1, partition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("col1", partition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals(3, partition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals("p0", partition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals("100", partition.getPartitionDefinitions().get(0).getMaxValues().get(0)); + DBTablePartition subpartition = partition.getSubpartition(); + Assert.assertEquals(DBTablePartitionType.LIST, subpartition.getPartitionOption().getType()); + Assert.assertEquals("abs(col2)", subpartition.getPartitionOption().getExpression()); + Assert.assertTrue(partition.getSubpartitionTemplated()); + Assert.assertEquals(3L, subpartition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals("p0smp0", subpartition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals(2, subpartition.getPartitionDefinitions().get(0).getValuesList().size()); + Assert.assertEquals("1", subpartition.getPartitionDefinitions().get(0).getValuesList().get(0).get(0)); + Assert.assertEquals("3", subpartition.getPartitionDefinitions().get(0).getValuesList().get(1).get(0)); + Assert.assertEquals("p0", + subpartition.getPartitionDefinitions().get(0).getParentPartitionDefinition().getName()); + } + + @Test + public void getSubpartition_RangeColumnsAndList_ColumnKey_NoTemplate_Success() { + String ddl = "CREATE TABLE ranges_list (col1 INT,col2 INT) \n" + + " PARTITION BY RANGE COLUMNS(col1)\n" + + " SUBPARTITION BY LIST(col2)\n" + + " (PARTITION p0 VALUES LESS THAN(100)\n" + + " (SUBPARTITION sp0 VALUES IN(1,3),\n" + + " SUBPARTITION sp1 VALUES IN(4,6),\n" + + " SUBPARTITION sp2 VALUES IN(7,9)),\n" + + " PARTITION p1 VALUES LESS THAN(200)\n" + + " (SUBPARTITION sp3 VALUES IN(1,3))\n" + + " ); \n"; + OBMySQLGetDBTableByParser table = new OBMySQLGetDBTableByParser(ddl); + DBTablePartition partition = table.getPartition(); + Assert.assertEquals(DBTablePartitionType.RANGE_COLUMNS, partition.getPartitionOption().getType()); + Assert.assertEquals(1, partition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("col1", partition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals(2, partition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals("p0", partition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals("100", partition.getPartitionDefinitions().get(0).getMaxValues().get(0)); + Assert.assertEquals("col1", partition.getPartitionOption().getColumnNames().get(0)); + DBTablePartition subpartition = partition.getSubpartition(); + Assert.assertEquals(DBTablePartitionType.LIST, subpartition.getPartitionOption().getType()); + Assert.assertEquals(1, subpartition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("col2", subpartition.getPartitionOption().getColumnNames().get(0)); + Assert.assertNull(subpartition.getPartitionOption().getExpression()); + Assert.assertFalse(partition.getSubpartitionTemplated()); + Assert.assertNull(subpartition.getPartitionOption().getPartitionsNum()); + Assert.assertEquals("sp0", subpartition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals(2, subpartition.getPartitionDefinitions().get(0).getValuesList().size()); + Assert.assertEquals("1", subpartition.getPartitionDefinitions().get(0).getValuesList().get(0).get(0)); + Assert.assertEquals("3", subpartition.getPartitionDefinitions().get(0).getValuesList().get(1).get(0)); + Assert.assertEquals("p0", + subpartition.getPartitionDefinitions().get(0).getParentPartitionDefinition().getName()); + } + + @Test + public void getSubpartition_RangeColumnsAndList_ExpressionKey_NoTemplate_Success() { + String ddl = "CREATE TABLE ranges_list_expr (col1 INT,col2 INT) \n" + + " PARTITION BY RANGE COLUMNS(col1)\n" + + " SUBPARTITION BY LIST(abs(col2))\n" + + " (PARTITION p0 VALUES LESS THAN(100)\n" + + " (SUBPARTITION sp0 VALUES IN(1,3),\n" + + " SUBPARTITION sp1 VALUES IN(4,6),\n" + + " SUBPARTITION sp2 VALUES IN(7,9)),\n" + + " PARTITION p1 VALUES LESS THAN(200)\n" + + " (SUBPARTITION sp3 VALUES IN(1,3))\n" + + " ); \n"; + OBMySQLGetDBTableByParser table = new OBMySQLGetDBTableByParser(ddl); + DBTablePartition partition = table.getPartition(); + Assert.assertEquals(DBTablePartitionType.RANGE_COLUMNS, partition.getPartitionOption().getType()); + Assert.assertEquals(1, partition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("col1", partition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals(2, partition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals("p0", partition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals("100", partition.getPartitionDefinitions().get(0).getMaxValues().get(0)); + Assert.assertEquals("col1", partition.getPartitionOption().getColumnNames().get(0)); + DBTablePartition subpartition = partition.getSubpartition(); + Assert.assertEquals(DBTablePartitionType.LIST, subpartition.getPartitionOption().getType()); + Assert.assertEquals("abs(col2)", subpartition.getPartitionOption().getExpression()); + Assert.assertFalse(partition.getSubpartitionTemplated()); + Assert.assertNull(subpartition.getPartitionOption().getPartitionsNum()); + Assert.assertEquals("sp0", subpartition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals(2, subpartition.getPartitionDefinitions().get(0).getValuesList().size()); + Assert.assertEquals("1", subpartition.getPartitionDefinitions().get(0).getValuesList().get(0).get(0)); + Assert.assertEquals("3", subpartition.getPartitionDefinitions().get(0).getValuesList().get(1).get(0)); + Assert.assertEquals("p0", + subpartition.getPartitionDefinitions().get(0).getParentPartitionDefinition().getName()); + } + + @Test + public void getSubpartition_RangeColumnsAndListColumns_ColumnKey_Template_Success() { + String ddl = "CREATE TABLE t_ranges_lists(col1 INT,col2 INT,col3 INT) \n" + + " PARTITION BY RANGE COLUMNS(col1)\n" + + " SUBPARTITION BY LIST COLUMNS(col2,col3)\n" + + " SUBPARTITION TEMPLATE \n" + + " (SUBPARTITION mp0 VALUES IN((1,1),(3,3)),\n" + + " SUBPARTITION mp1 VALUES IN((4,4),(6,6)),\n" + + " SUBPARTITION mp2 VALUES IN((7,7),(8,8))\n" + + " )\n" + + " (PARTITION p0 VALUES LESS THAN(100),\n" + + " PARTITION p1 VALUES LESS THAN(200),\n" + + " PARTITION p2 VALUES LESS THAN(300)\n" + + " ); \n"; + OBMySQLGetDBTableByParser table = new OBMySQLGetDBTableByParser(ddl); + DBTablePartition partition = table.getPartition(); + Assert.assertEquals(DBTablePartitionType.RANGE_COLUMNS, partition.getPartitionOption().getType()); + Assert.assertEquals(1, partition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("col1", partition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals(3, partition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals("p0", partition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals("100", partition.getPartitionDefinitions().get(0).getMaxValues().get(0)); + Assert.assertEquals("col1", partition.getPartitionOption().getColumnNames().get(0)); + DBTablePartition subpartition = partition.getSubpartition(); + Assert.assertEquals(DBTablePartitionType.LIST_COLUMNS, subpartition.getPartitionOption().getType()); + Assert.assertEquals(2, subpartition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("col2", subpartition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals("col3", subpartition.getPartitionOption().getColumnNames().get(1)); + Assert.assertNull(subpartition.getPartitionOption().getExpression()); + Assert.assertTrue(partition.getSubpartitionTemplated()); + Assert.assertEquals(3L, subpartition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals("p0smp0", subpartition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals(2, subpartition.getPartitionDefinitions().get(0).getValuesList().size()); + Assert.assertEquals("1", subpartition.getPartitionDefinitions().get(0).getValuesList().get(0).get(0)); + Assert.assertEquals("1", subpartition.getPartitionDefinitions().get(0).getValuesList().get(0).get(1)); + Assert.assertEquals("3", subpartition.getPartitionDefinitions().get(0).getValuesList().get(1).get(0)); + Assert.assertEquals("3", subpartition.getPartitionDefinitions().get(0).getValuesList().get(1).get(1)); + Assert.assertEquals("p0", + subpartition.getPartitionDefinitions().get(0).getParentPartitionDefinition().getName()); + } + + @Test + public void getSubpartition_RangeColumnsAndListColumns_ColumnKey_NoTemplate_Success() { + String ddl = "CREATE TABLE ranges_lists (col1 INT,col2 INT,col3 INT) \n" + + " PARTITION BY RANGE COLUMNS(col1)\n" + + " SUBPARTITION BY LIST COLUMNS(col2,col3)\n" + + " (PARTITION p0 VALUES LESS THAN(100)\n" + + " (SUBPARTITION sp0 VALUES IN(1,1),\n" + + " SUBPARTITION sp1 VALUES IN(4,4),\n" + + " SUBPARTITION sp2 VALUES IN(7,7)),\n" + + " PARTITION p1 VALUES LESS THAN(200)\n" + + " (SUBPARTITION sp3 VALUES IN(9,9),\n" + + " SUBPARTITION sp4 VALUES IN(10,10),\n" + + " SUBPARTITION sp5 VALUES IN(11,11))\n" + + " ); \n"; + OBMySQLGetDBTableByParser table = new OBMySQLGetDBTableByParser(ddl); + DBTablePartition partition = table.getPartition(); + Assert.assertEquals(DBTablePartitionType.RANGE_COLUMNS, partition.getPartitionOption().getType()); + Assert.assertEquals(1, partition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("col1", partition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals(2, partition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals("p0", partition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals("100", partition.getPartitionDefinitions().get(0).getMaxValues().get(0)); + Assert.assertEquals("col1", partition.getPartitionOption().getColumnNames().get(0)); + DBTablePartition subpartition = partition.getSubpartition(); + Assert.assertEquals(DBTablePartitionType.LIST_COLUMNS, subpartition.getPartitionOption().getType()); + Assert.assertEquals(2, subpartition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("col2", subpartition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals("col3", subpartition.getPartitionOption().getColumnNames().get(1)); + Assert.assertNull(subpartition.getPartitionOption().getExpression()); + Assert.assertFalse(partition.getSubpartitionTemplated()); + Assert.assertNull(subpartition.getPartitionOption().getPartitionsNum()); + Assert.assertEquals("sp0", subpartition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals(2, subpartition.getPartitionDefinitions().get(0).getValuesList().size()); + Assert.assertEquals("1", subpartition.getPartitionDefinitions().get(0).getValuesList().get(0).get(0)); + Assert.assertEquals("1", subpartition.getPartitionDefinitions().get(0).getValuesList().get(1).get(0)); + Assert.assertEquals("p0", + subpartition.getPartitionDefinitions().get(0).getParentPartitionDefinition().getName()); + } + + @Test + public void getSubpartition_RangeColumnsAndHash_ColumnKey_Template_Success() { + String ddl = "CREATE TABLE `t_ranges_hash` (\n" + + " `col1` int(11) DEFAULT NULL,\n" + + " `col2` int(11) DEFAULT NULL\n" + + ") \n" + + " partition by range columns(col1) subpartition by hash(col2) subpartition template (\n" + + "subpartition `p0`,\n" + + "subpartition `p1`,\n" + + "subpartition `p2`,\n" + + "subpartition `p3`,\n" + + "subpartition `p4`)\n" + + "(partition `p0` values less than (100),\n" + + "partition `p1` values less than (200),\n" + + "partition `p2` values less than (300))"; + OBMySQLGetDBTableByParser table = new OBMySQLGetDBTableByParser(ddl); + DBTablePartition partition = table.getPartition(); + Assert.assertEquals(DBTablePartitionType.RANGE_COLUMNS, partition.getPartitionOption().getType()); + Assert.assertEquals(1, partition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("col1", partition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals(3, partition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals("p0", partition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals("100", partition.getPartitionDefinitions().get(0).getMaxValues().get(0)); + Assert.assertEquals("col1", partition.getPartitionOption().getColumnNames().get(0)); + DBTablePartition subpartition = partition.getSubpartition(); + Assert.assertEquals(DBTablePartitionType.HASH, subpartition.getPartitionOption().getType()); + Assert.assertEquals(1, subpartition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("col2", subpartition.getPartitionOption().getColumnNames().get(0)); + Assert.assertNull(subpartition.getPartitionOption().getExpression()); + Assert.assertTrue(partition.getSubpartitionTemplated()); + Assert.assertEquals(5L, subpartition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals("p0sp0", subpartition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals("p0", + subpartition.getPartitionDefinitions().get(0).getParentPartitionDefinition().getName()); + } + + @Test + public void getSubpartition_RangeColumnsAndHash_ExpressionKey_Template_Success() { + String ddl = "CREATE TABLE `t_ranges_hash_expr` (\n" + + " `col1` int(11) DEFAULT NULL,\n" + + " `col2` int(11) DEFAULT NULL\n" + + ") \n" + + " partition by range columns(col1) subpartition by hash(abs(col2)) subpartition template (\n" + + "subpartition `p0`,\n" + + "subpartition `p1`,\n" + + "subpartition `p2`,\n" + + "subpartition `p3`,\n" + + "subpartition `p4`)\n" + + "(partition `p0` values less than (100),\n" + + "partition `p1` values less than (200),\n" + + "partition `p2` values less than (300))"; + OBMySQLGetDBTableByParser table = new OBMySQLGetDBTableByParser(ddl); + DBTablePartition partition = table.getPartition(); + Assert.assertEquals(DBTablePartitionType.RANGE_COLUMNS, partition.getPartitionOption().getType()); + Assert.assertEquals(1, partition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("col1", partition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals(3, partition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals("p0", partition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals("100", partition.getPartitionDefinitions().get(0).getMaxValues().get(0)); + Assert.assertEquals("col1", partition.getPartitionOption().getColumnNames().get(0)); + DBTablePartition subpartition = partition.getSubpartition(); + Assert.assertEquals(DBTablePartitionType.HASH, subpartition.getPartitionOption().getType()); + Assert.assertEquals("abs(col2)", subpartition.getPartitionOption().getExpression()); + Assert.assertTrue(partition.getSubpartitionTemplated()); + Assert.assertEquals(5L, subpartition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals("p0sp0", subpartition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals("p0", + subpartition.getPartitionDefinitions().get(0).getParentPartitionDefinition().getName()); + } + + @Test + public void getSubpartition_RangeColumnsAndHash_ColumnKey_NoTemplate_Success() { + String ddl = "CREATE TABLE ranges_hash (col1 INT,col2 INT) \n" + + " PARTITION BY RANGE COLUMNS(col1)\n" + + " SUBPARTITION BY HASH(col2)\n" + + " (PARTITION p0 VALUES LESS THAN(100)\n" + + " (SUBPARTITION sp0,\n" + + " SUBPARTITION sp1,\n" + + " SUBPARTITION sp2),\n" + + " PARTITION p1 VALUES LESS THAN(200)\n" + + " (SUBPARTITION sp3,\n" + + " SUBPARTITION sp4,\n" + + " SUBPARTITION sp5)\n" + + " ); \n"; + OBMySQLGetDBTableByParser table = new OBMySQLGetDBTableByParser(ddl); + DBTablePartition partition = table.getPartition(); + Assert.assertEquals(DBTablePartitionType.RANGE_COLUMNS, partition.getPartitionOption().getType()); + Assert.assertEquals(1, partition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("col1", partition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals(2, partition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals("p0", partition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals("100", partition.getPartitionDefinitions().get(0).getMaxValues().get(0)); + Assert.assertEquals("col1", partition.getPartitionOption().getColumnNames().get(0)); + DBTablePartition subpartition = partition.getSubpartition(); + Assert.assertEquals(DBTablePartitionType.HASH, subpartition.getPartitionOption().getType()); + Assert.assertEquals(1, subpartition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("col2", subpartition.getPartitionOption().getColumnNames().get(0)); + Assert.assertNull(subpartition.getPartitionOption().getExpression()); + Assert.assertFalse(partition.getSubpartitionTemplated()); + Assert.assertNull(subpartition.getPartitionOption().getPartitionsNum()); + Assert.assertEquals("sp0", subpartition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals("p0", + subpartition.getPartitionDefinitions().get(0).getParentPartitionDefinition().getName()); + } + + @Test + public void getSubpartition_RangeColumnsAndHash_ExpressionKey_NoTemplate_Success() { + String ddl = "CREATE TABLE ranges_hash_expr (col1 INT,col2 INT) \n" + + " PARTITION BY RANGE COLUMNS(col1)\n" + + " SUBPARTITION BY HASH(abs(col2))\n" + + " (PARTITION p0 VALUES LESS THAN(100)\n" + + " (SUBPARTITION sp0,\n" + + " SUBPARTITION sp1,\n" + + " SUBPARTITION sp2),\n" + + " PARTITION p1 VALUES LESS THAN(200)\n" + + " (SUBPARTITION sp3,\n" + + " SUBPARTITION sp4,\n" + + " SUBPARTITION sp5)\n" + + " ); \n"; + OBMySQLGetDBTableByParser table = new OBMySQLGetDBTableByParser(ddl); + DBTablePartition partition = table.getPartition(); + Assert.assertEquals(DBTablePartitionType.RANGE_COLUMNS, partition.getPartitionOption().getType()); + Assert.assertEquals(1, partition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("col1", partition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals(2, partition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals("p0", partition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals("100", partition.getPartitionDefinitions().get(0).getMaxValues().get(0)); + Assert.assertEquals("col1", partition.getPartitionOption().getColumnNames().get(0)); + DBTablePartition subpartition = partition.getSubpartition(); + Assert.assertEquals(DBTablePartitionType.HASH, subpartition.getPartitionOption().getType()); + Assert.assertEquals("abs(col2)", subpartition.getPartitionOption().getExpression()); + Assert.assertFalse(partition.getSubpartitionTemplated()); + Assert.assertNull(subpartition.getPartitionOption().getPartitionsNum()); + Assert.assertEquals("sp0", subpartition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals("p0", + subpartition.getPartitionDefinitions().get(0).getParentPartitionDefinition().getName()); + } + + @Test + public void getSubpartition_RangeColumnsAndKey_ColumnKey_Template_Success() { + String ddl = "CREATE TABLE `t_ranges_key` (\n" + + " `col1` int(11) DEFAULT NULL,\n" + + " `col2` int(11) DEFAULT NULL\n" + + ") \n" + + " partition by range columns(col1) subpartition by key(col2) subpartition template (\n" + + "subpartition `p0`,\n" + + "subpartition `p1`,\n" + + "subpartition `p2`)\n" + + "(partition `p0` values less than (100),\n" + + "partition `p1` values less than (200),\n" + + "partition `p2` values less than (300))"; + OBMySQLGetDBTableByParser table = new OBMySQLGetDBTableByParser(ddl); + DBTablePartition partition = table.getPartition(); + Assert.assertEquals(DBTablePartitionType.RANGE_COLUMNS, partition.getPartitionOption().getType()); + Assert.assertEquals(1, partition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("col1", partition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals(3, partition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals("p0", partition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals("100", partition.getPartitionDefinitions().get(0).getMaxValues().get(0)); + Assert.assertEquals("col1", partition.getPartitionOption().getColumnNames().get(0)); + DBTablePartition subpartition = partition.getSubpartition(); + Assert.assertEquals(DBTablePartitionType.KEY, subpartition.getPartitionOption().getType()); + Assert.assertEquals(1, subpartition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("col2", subpartition.getPartitionOption().getColumnNames().get(0)); + Assert.assertNull(subpartition.getPartitionOption().getExpression()); + Assert.assertTrue(partition.getSubpartitionTemplated()); + Assert.assertEquals(3L, subpartition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals("p0sp0", subpartition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals("p0", + subpartition.getPartitionDefinitions().get(0).getParentPartitionDefinition().getName()); + } + + @Test + public void getSubpartition_RangeColumnsAndKey_ColumnKey_NoTemplate_Success() { + String ddl = "CREATE TABLE `ranges_key` (\n" + + " `col1` int(11) NOT NULL,\n" + + " `col2` varchar(50) DEFAULT NULL,\n" + + " `col3` int(11) NOT NULL\n" + + ") \n" + + " partition by range columns(col1) subpartition by key(col3)\n" + + "(partition `p0` values less than (100) (\n" + + "subpartition `sp0`,\n" + + "subpartition `sp1`,\n" + + "subpartition `sp2`),\n" + + "partition `p1` values less than (200) (\n" + + "subpartition `sp3`,\n" + + "subpartition `sp4`))"; + OBMySQLGetDBTableByParser table = new OBMySQLGetDBTableByParser(ddl); + DBTablePartition partition = table.getPartition(); + Assert.assertEquals(DBTablePartitionType.RANGE_COLUMNS, partition.getPartitionOption().getType()); + Assert.assertEquals(1, partition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("col1", partition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals(2, partition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals("p0", partition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals("100", partition.getPartitionDefinitions().get(0).getMaxValues().get(0)); + Assert.assertEquals("col1", partition.getPartitionOption().getColumnNames().get(0)); + DBTablePartition subpartition = partition.getSubpartition(); + Assert.assertEquals(DBTablePartitionType.KEY, subpartition.getPartitionOption().getType()); + Assert.assertEquals(1, subpartition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("col3", subpartition.getPartitionOption().getColumnNames().get(0)); + Assert.assertNull(subpartition.getPartitionOption().getExpression()); + Assert.assertFalse(partition.getSubpartitionTemplated()); + Assert.assertNull(subpartition.getPartitionOption().getPartitionsNum()); + Assert.assertEquals("sp0", subpartition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals("p0", + subpartition.getPartitionDefinitions().get(0).getParentPartitionDefinition().getName()); + } + } diff --git a/server/plugins/schema-plugin-ob-oracle/pom.xml b/server/plugins/schema-plugin-ob-oracle/pom.xml index b35cbbec2f..a6e89f54ad 100644 --- a/server/plugins/schema-plugin-ob-oracle/pom.xml +++ b/server/plugins/schema-plugin-ob-oracle/pom.xml @@ -6,7 +6,7 @@ com.oceanbase plugin-parent - 4.3.2-SNAPSHOT + 4.3.3-SNAPSHOT ../pom.xml diff --git a/server/plugins/schema-plugin-ob-oracle/src/main/java/com/oceanbase/odc/plugin/schema/oboracle/OBOracleTableExtension.java b/server/plugins/schema-plugin-ob-oracle/src/main/java/com/oceanbase/odc/plugin/schema/oboracle/OBOracleTableExtension.java index bc9e78218a..193fc3c654 100644 --- a/server/plugins/schema-plugin-ob-oracle/src/main/java/com/oceanbase/odc/plugin/schema/oboracle/OBOracleTableExtension.java +++ b/server/plugins/schema-plugin-ob-oracle/src/main/java/com/oceanbase/odc/plugin/schema/oboracle/OBOracleTableExtension.java @@ -30,6 +30,7 @@ import com.oceanbase.odc.plugin.schema.oboracle.utils.DBAccessorUtil; import com.oceanbase.tools.dbbrowser.editor.DBTableEditor; import com.oceanbase.tools.dbbrowser.model.DBConstraintType; +import com.oceanbase.tools.dbbrowser.model.DBObjectType; import com.oceanbase.tools.dbbrowser.model.DBTable; import com.oceanbase.tools.dbbrowser.model.DBTable.DBTableOptions; import com.oceanbase.tools.dbbrowser.model.DBTableColumn; @@ -66,16 +67,21 @@ public DBTable getDetail(@NonNull Connection connection, @NonNull String schemaN table.setOwner(schemaName); table.setName(tableName); table.setColumns(columns); - /** - * If the constraint name cannot be obtained through ddl of the table, then the constraint - * information will still be obtained through DBSchemaAccessor - */ - List constraints = parser.listConstraints(); - table.setConstraints(constraints.stream().anyMatch(c -> Objects.isNull(c.getName())) - ? accessor.listTableConstraints(schemaName, tableName) - : constraints); + if (!accessor.isExternalTable(schemaName, tableName)) { + /** + * If the constraint name cannot be obtained through ddl of the table, then the constraint + * information will still be obtained through DBSchemaAccessor + */ + List constraints = parser.listConstraints(); + table.setConstraints(constraints.stream().anyMatch(c -> Objects.isNull(c.getName())) + ? accessor.listTableConstraints(schemaName, tableName) + : constraints); + table.setIndexes(parser.listIndexes()); + table.setType(DBObjectType.TABLE); + } else { + table.setType(DBObjectType.EXTERNAL_TABLE); + } table.setPartition(parser.getPartition()); - table.setIndexes(parser.listIndexes()); DBTableOptions tableOptions = accessor.getTableOptions(schemaName, tableName); table.setTableOptions(tableOptions); table.setDDL(getTableDDL(connection, schemaName, tableName, parser, columns, tableOptions)); diff --git a/server/plugins/schema-plugin-ob-oracle/src/main/java/com/oceanbase/odc/plugin/schema/oboracle/parser/OBOracleGetDBTableByParser.java b/server/plugins/schema-plugin-ob-oracle/src/main/java/com/oceanbase/odc/plugin/schema/oboracle/parser/OBOracleGetDBTableByParser.java index dab498c8fd..ddb85be12c 100644 --- a/server/plugins/schema-plugin-ob-oracle/src/main/java/com/oceanbase/odc/plugin/schema/oboracle/parser/OBOracleGetDBTableByParser.java +++ b/server/plugins/schema-plugin-ob-oracle/src/main/java/com/oceanbase/odc/plugin/schema/oboracle/parser/OBOracleGetDBTableByParser.java @@ -28,13 +28,12 @@ import com.oceanbase.odc.common.util.JdbcOperationsUtil; import com.oceanbase.odc.common.util.StringUtils; -import com.oceanbase.odc.plugin.schema.obmysql.parser.GetDBTableByParser; +import com.oceanbase.odc.plugin.schema.obmysql.parser.BaseOBGetDBTableByParser; import com.oceanbase.tools.dbbrowser.model.DBConstraintDeferability; import com.oceanbase.tools.dbbrowser.model.DBConstraintType; import com.oceanbase.tools.dbbrowser.model.DBForeignKeyModifyRule; import com.oceanbase.tools.dbbrowser.model.DBIndexAlgorithm; import com.oceanbase.tools.dbbrowser.model.DBIndexType; -import com.oceanbase.tools.dbbrowser.model.DBTableColumn; import com.oceanbase.tools.dbbrowser.model.DBTableConstraint; import com.oceanbase.tools.dbbrowser.model.DBTableIndex; import com.oceanbase.tools.dbbrowser.model.DBTablePartition; @@ -62,7 +61,10 @@ import com.oceanbase.tools.sqlparser.statement.createtable.Partition; import com.oceanbase.tools.sqlparser.statement.createtable.RangePartition; import com.oceanbase.tools.sqlparser.statement.createtable.RangePartitionElement; +import com.oceanbase.tools.sqlparser.statement.createtable.SubPartitionOption; import com.oceanbase.tools.sqlparser.statement.createtable.TableElement; +import com.oceanbase.tools.sqlparser.statement.expression.CollectionExpression; +import com.oceanbase.tools.sqlparser.statement.expression.ConstExpression; import com.oceanbase.tools.sqlparser.statement.expression.RelationReference; import lombok.Getter; @@ -75,7 +77,7 @@ * @since 4.2.0 */ @Slf4j -public class OBOracleGetDBTableByParser implements GetDBTableByParser { +public class OBOracleGetDBTableByParser extends BaseOBGetDBTableByParser { @Getter private final CreateTable createTableStmt; private final Connection connection; @@ -95,6 +97,11 @@ public OBOracleGetDBTableByParser(@NonNull Connection connection, @NonNull Strin this.createTableStmt = parseTableDDL(schemaName, tableName); } + @Override + public CreateTable getCreateTableStmt() { + return this.createTableStmt; + } + private CreateTable parseTableDDL(@NonNull String schemaName, @NonNull String tableName) { CreateTable statement = null; OracleSqlBuilder builder = new OracleSqlBuilder(); @@ -116,11 +123,6 @@ private CreateTable parseTableDDL(@NonNull String schemaName, @NonNull String ta return statement; } - @Override - public List listColumns() { - throw new UnsupportedOperationException("Not supported yet"); - } - @Override public List listConstraints() { if (this.constraints.size() > 0 || this.createTableStmt == null) { @@ -243,7 +245,7 @@ public List listConstraints() { return constraints; } - private String removeIdentifiers(String str) { + protected String removeIdentifiers(String str) { if (Objects.isNull(str)) { return ""; } @@ -372,40 +374,18 @@ private CreateIndex parseIndexDDL(String ddl) { } @Override - public DBTablePartition getPartition() { - DBTablePartition partition = new DBTablePartition(); - partition.setPartitionOption(new DBTablePartitionOption()); - partition.setPartitionDefinitions(new ArrayList<>()); - - if (Objects.isNull(createTableStmt)) { - partition.setWarning("Failed to parse ob oracle table ddl"); - return partition; - } - Partition partitionStmt = createTableStmt.getPartition(); - if (Objects.isNull(partitionStmt)) { - return partition; - } - if (partitionStmt instanceof HashPartition) { - parseHashPartitionStmt((HashPartition) partitionStmt, partition); - } else if (partitionStmt instanceof RangePartition) { - parseRangePartitionStmt((RangePartition) partitionStmt, partition); - } else if (partitionStmt instanceof ListPartition) { - parseListPartitionStmt((ListPartition) partitionStmt, partition); - } + protected String generateTemplateSubPartitionName(String partitionName, String subPartitionName) { + return removeIdentifiers(partitionName) + 'S' + removeIdentifiers(subPartitionName); + } - /** - * In order to adapt to the front-end only the expression field is used for Hash、List and Range - * partition types - */ - if (Objects.nonNull(partition.getPartitionOption().getType()) - && partition.getPartitionOption().getType().supportExpression() - && StringUtils.isBlank(partition.getPartitionOption().getExpression())) { - List columnNames = partition.getPartitionOption().getColumnNames(); - if (!columnNames.isEmpty()) { - partition.getPartitionOption().setExpression(String.join(", ", columnNames)); - } - } - return partition; + @Override + protected void fillSubPartitionKey(DBTablePartitionOption subPartitionOption, + DBTablePartitionType subDBTablePartitionType, SubPartitionOption parsedSubPartitionOption) { + // expressions are not supported, multiple columns are supported as partition keys + subPartitionOption.setColumnNames(parsedSubPartitionOption.getSubPartitionTargets() == null ? null + : parsedSubPartitionOption.getSubPartitionTargets().stream() + .map(item -> removeIdentifiers(item.getText())) + .collect(Collectors.toList())); } private void parseHashPartitionStmt(HashPartition statement, DBTablePartition partition) { @@ -470,7 +450,16 @@ private void parseListPartitionStmt(ListPartition statement, DBTablePartition pa DBTablePartitionDefinition partitionDefinition = new DBTablePartitionDefinition(); partitionDefinition.setOrdinalPosition(i); List> valuesList = new ArrayList<>(); - element.getListExprs().forEach(item -> valuesList.add(Collections.singletonList(item.getText()))); + for (Expression listExpr : element.getListExprs()) { + if (listExpr instanceof CollectionExpression) { + valuesList.add( + ((CollectionExpression) listExpr).getExpressionList().stream() + .map(Expression::getText) + .collect(Collectors.toList())); + } else if (listExpr instanceof ConstExpression) { + valuesList.add(Collections.singletonList(listExpr.getText())); + } + } partitionDefinition.setValuesList(valuesList); partitionDefinition.setName(removeIdentifiers(element.getRelation())); partitionDefinition.setType(DBTablePartitionType.LIST); @@ -478,4 +467,16 @@ private void parseListPartitionStmt(ListPartition statement, DBTablePartition pa } } + + @Override + protected void parsePartitionStmt(DBTablePartition partition, Partition partitionStmt) { + if (partitionStmt instanceof HashPartition) { + parseHashPartitionStmt((HashPartition) partitionStmt, partition); + } else if (partitionStmt instanceof RangePartition) { + parseRangePartitionStmt((RangePartition) partitionStmt, partition); + } else if (partitionStmt instanceof ListPartition) { + parseListPartitionStmt((ListPartition) partitionStmt, partition); + } + } + } diff --git a/server/plugins/schema-plugin-ob-oracle/src/test/java/com/oceanbase/odc/plugin/schema/oboracle/parser/OBOracleGetDBTableByParserTest.java b/server/plugins/schema-plugin-ob-oracle/src/test/java/com/oceanbase/odc/plugin/schema/oboracle/parser/OBOracleGetDBTableByParserTest.java index e87a902f5d..3cc07ae575 100644 --- a/server/plugins/schema-plugin-ob-oracle/src/test/java/com/oceanbase/odc/plugin/schema/oboracle/parser/OBOracleGetDBTableByParserTest.java +++ b/server/plugins/schema-plugin-ob-oracle/src/test/java/com/oceanbase/odc/plugin/schema/oboracle/parser/OBOracleGetDBTableByParserTest.java @@ -69,33 +69,396 @@ private static void batchExcuteSql(String str) { } @Test - public void getPartition_Hash_Success() { + public void getPartition_Hash_SingleKey_Success() { OBOracleGetDBTableByParser table = - new OBOracleGetDBTableByParser(connection, TEST_DATABASE_NAME, "HASH_PART_BY_PARSER"); + new OBOracleGetDBTableByParser(connection, TEST_DATABASE_NAME, "HASH_PART_SINGLE_KEY_BY_PARSER"); DBTablePartition partition = table.getPartition(); Assert.assertEquals(6L, partition.getPartitionOption().getPartitionsNum().longValue()); Assert.assertEquals(DBTablePartitionType.HASH, partition.getPartitionOption().getType()); Assert.assertEquals("COL1", partition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals("P0", partition.getPartitionDefinitions().get(0).getName()); + Assert.assertNull(partition.getSubpartition()); + Assert.assertNull(partition.getSubpartitionTemplated()); } @Test - public void getPartition_list_Success() { + public void getPartition_Hash_MultipleKey_Success() { OBOracleGetDBTableByParser table = - new OBOracleGetDBTableByParser(connection, TEST_DATABASE_NAME, "LIST_PART_BY_PARSER"); + new OBOracleGetDBTableByParser(connection, TEST_DATABASE_NAME, "HASH_PART_MULTIPLE_KEY_BY_PARSER"); + DBTablePartition partition = table.getPartition(); + Assert.assertEquals(10L, partition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals(DBTablePartitionType.HASH, partition.getPartitionOption().getType()); + Assert.assertEquals(2, partition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("COL1", partition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals("COL2", partition.getPartitionOption().getColumnNames().get(1)); + Assert.assertEquals("P0", partition.getPartitionDefinitions().get(0).getName()); + Assert.assertNull(partition.getSubpartition()); + Assert.assertNull(partition.getSubpartitionTemplated()); + } + + @Test + public void getPartition_List_SingleKey_Success() { + OBOracleGetDBTableByParser table = + new OBOracleGetDBTableByParser(connection, TEST_DATABASE_NAME, "LIST_PART_SINGLE_KEY_BY_PARSER"); DBTablePartition partition = table.getPartition(); Assert.assertEquals(4L, partition.getPartitionOption().getPartitionsNum().longValue()); Assert.assertEquals(DBTablePartitionType.LIST, partition.getPartitionOption().getType()); Assert.assertEquals("LOG_VALUE", partition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals("P01", partition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals(3, partition.getPartitionDefinitions().get(0).getValuesList().size()); + Assert.assertEquals("'A'", partition.getPartitionDefinitions().get(0).getValuesList().get(0).get(0)); + Assert.assertNull(partition.getSubpartition()); + Assert.assertNull(partition.getSubpartitionTemplated()); } @Test - public void getPartition_range_Success() { + public void getPartition_List_MultipleKey_Success() { OBOracleGetDBTableByParser table = - new OBOracleGetDBTableByParser(connection, TEST_DATABASE_NAME, "RANGE_PART_BY_PARSER"); + new OBOracleGetDBTableByParser(connection, TEST_DATABASE_NAME, "LIST_PART_MULTIPLE_KEY_BY_PARSER"); + DBTablePartition partition = table.getPartition(); + Assert.assertEquals(3L, partition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals(DBTablePartitionType.LIST, partition.getPartitionOption().getType()); + Assert.assertEquals(2, partition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("COL2", partition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals("COL2", partition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals("P0", partition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals(2, partition.getPartitionDefinitions().get(0).getValuesList().size()); + Assert.assertEquals(2, partition.getPartitionDefinitions().get(0).getValuesList().get(0).size()); + Assert.assertEquals("1", partition.getPartitionDefinitions().get(0).getValuesList().get(0).get(0)); + Assert.assertEquals("1", partition.getPartitionDefinitions().get(0).getValuesList().get(0).get(1)); + Assert.assertNull(partition.getSubpartition()); + Assert.assertNull(partition.getSubpartitionTemplated()); + } + + @Test + public void getPartition_Range_SingleKey_Success() { + OBOracleGetDBTableByParser table = + new OBOracleGetDBTableByParser(connection, TEST_DATABASE_NAME, "RANGE_PART_SINGLE_KEY_BY_PARSER"); DBTablePartition partition = table.getPartition(); Assert.assertEquals(4L, partition.getPartitionOption().getPartitionsNum().longValue()); Assert.assertEquals(DBTablePartitionType.RANGE, partition.getPartitionOption().getType()); Assert.assertEquals("LOG_DATE", partition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals("M202001", partition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals(1, partition.getPartitionDefinitions().get(0).getMaxValues().size()); + Assert.assertEquals("TO_DATE(' 2020-02-01 00:00:00', 'SYYYY-MM-DD HH24:MI:SS', 'NLS_CALENDAR=GREGORIAN')", + partition.getPartitionDefinitions().get(0).getMaxValues().get(0)); + Assert.assertNull(partition.getSubpartition()); + Assert.assertNull(partition.getSubpartitionTemplated()); + } + + @Test + public void getPartition_Range_MultipleKey_Success() { + OBOracleGetDBTableByParser table = + new OBOracleGetDBTableByParser(connection, TEST_DATABASE_NAME, "RANGE_PART_MULTIPLE_KEY_BY_PARSER"); + DBTablePartition partition = table.getPartition(); + Assert.assertEquals(4L, partition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals(DBTablePartitionType.RANGE, partition.getPartitionOption().getType()); + Assert.assertEquals(2, partition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("COL1", partition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals("COL2", partition.getPartitionOption().getColumnNames().get(1)); + Assert.assertEquals("M202001", partition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals(2, partition.getPartitionDefinitions().get(0).getMaxValues().size()); + Assert.assertEquals("1", partition.getPartitionDefinitions().get(0).getMaxValues().get(0)); + Assert.assertEquals("1", partition.getPartitionDefinitions().get(0).getMaxValues().get(1)); + Assert.assertNull(partition.getSubpartition()); + Assert.assertNull(partition.getSubpartitionTemplated()); + } + + @Test + public void getSubPartition_TemplateSingleRangeMultipleRange_Success() { + OBOracleGetDBTableByParser table = + new OBOracleGetDBTableByParser(connection, TEST_DATABASE_NAME, "T_SINGLE_RANGE_MULTIPLE_RANGE"); + DBTablePartition partition = table.getPartition(); + Assert.assertEquals(2, partition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals(DBTablePartitionType.RANGE, partition.getPartitionOption().getType()); + Assert.assertEquals(1, partition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("COL1", partition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals("P0", partition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals(1, partition.getPartitionDefinitions().get(0).getMaxValues().size()); + Assert.assertEquals("100", partition.getPartitionDefinitions().get(0).getMaxValues().get(0)); + DBTablePartition subpartition = partition.getSubpartition(); + Assert.assertTrue(partition.getSubpartitionTemplated()); + Assert.assertEquals(DBTablePartitionType.RANGE, subpartition.getPartitionOption().getType()); + Assert.assertEquals(2, subpartition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("COL2", subpartition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals("COL3", subpartition.getPartitionOption().getColumnNames().get(1)); + Assert.assertNull(subpartition.getPartitionOption().getExpression()); + Assert.assertEquals(3L, subpartition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals("P0SMP0", subpartition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals(2, subpartition.getPartitionDefinitions().get(0).getMaxValues().size()); + Assert.assertEquals("2020", subpartition.getPartitionDefinitions().get(0).getMaxValues().get(0)); + Assert.assertEquals("2020", subpartition.getPartitionDefinitions().get(0).getMaxValues().get(1)); + Assert.assertEquals("P0", + subpartition.getPartitionDefinitions().get(0).getParentPartitionDefinition().getName()); + } + + @Test + public void getSubPartition_TemplateSingleRangeSingleRange_Success() { + OBOracleGetDBTableByParser table = + new OBOracleGetDBTableByParser(connection, TEST_DATABASE_NAME, "T_SINGLE_RANGE_SINGLE_RANGE"); + DBTablePartition partition = table.getPartition(); + Assert.assertEquals(2, partition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals(DBTablePartitionType.RANGE, partition.getPartitionOption().getType()); + Assert.assertEquals(1, partition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("COL1", partition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals("P0", partition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals(1, partition.getPartitionDefinitions().get(0).getMaxValues().size()); + Assert.assertEquals("100", partition.getPartitionDefinitions().get(0).getMaxValues().get(0)); + DBTablePartition subpartition = partition.getSubpartition(); + Assert.assertTrue(partition.getSubpartitionTemplated()); + Assert.assertEquals(DBTablePartitionType.RANGE, subpartition.getPartitionOption().getType()); + Assert.assertEquals(1, subpartition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("COL2", subpartition.getPartitionOption().getColumnNames().get(0)); + Assert.assertNull(subpartition.getPartitionOption().getExpression()); + Assert.assertEquals(3L, subpartition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals("P0SMP0", subpartition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals(1, subpartition.getPartitionDefinitions().get(0).getMaxValues().size()); + Assert.assertEquals("2020", subpartition.getPartitionDefinitions().get(0).getMaxValues().get(0)); + Assert.assertEquals("P0", + subpartition.getPartitionDefinitions().get(0).getParentPartitionDefinition().getName()); + } + + @Test + public void getSubPartition_TemplateSingleHashMultipleList_Success() { + OBOracleGetDBTableByParser table = + new OBOracleGetDBTableByParser(connection, TEST_DATABASE_NAME, "T_SINGLE_HASH_MULTIPLE_LIST"); + DBTablePartition partition = table.getPartition(); + Assert.assertEquals(5, partition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals(DBTablePartitionType.HASH, partition.getPartitionOption().getType()); + Assert.assertEquals(1, partition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("COL1", partition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals("P0", partition.getPartitionDefinitions().get(0).getName()); + DBTablePartition subpartition = partition.getSubpartition(); + Assert.assertTrue(partition.getSubpartitionTemplated()); + Assert.assertEquals(DBTablePartitionType.LIST, subpartition.getPartitionOption().getType()); + Assert.assertEquals(2, subpartition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("COL2", subpartition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals("COL3", subpartition.getPartitionOption().getColumnNames().get(1)); + Assert.assertNull(subpartition.getPartitionOption().getExpression()); + Assert.assertEquals(3L, subpartition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals("P0SSP0", subpartition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals(1, subpartition.getPartitionDefinitions().get(0).getValuesList().size()); + Assert.assertEquals("100", subpartition.getPartitionDefinitions().get(0).getValuesList().get(0).get(0)); + Assert.assertEquals("100", subpartition.getPartitionDefinitions().get(0).getValuesList().get(0).get(1)); + Assert.assertEquals("P0", + subpartition.getPartitionDefinitions().get(0).getParentPartitionDefinition().getName()); + } + + @Test + public void getSubPartition_TemplateSingleHashSingleList_Success() { + OBOracleGetDBTableByParser table = + new OBOracleGetDBTableByParser(connection, TEST_DATABASE_NAME, "T_SINGLE_HASH_SINGLE_LIST"); + DBTablePartition partition = table.getPartition(); + Assert.assertEquals(5, partition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals(DBTablePartitionType.HASH, partition.getPartitionOption().getType()); + Assert.assertEquals(1, partition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("COL1", partition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals("P0", partition.getPartitionDefinitions().get(0).getName()); + DBTablePartition subpartition = partition.getSubpartition(); + Assert.assertTrue(partition.getSubpartitionTemplated()); + Assert.assertEquals(DBTablePartitionType.LIST, subpartition.getPartitionOption().getType()); + Assert.assertEquals(1, subpartition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("COL2", subpartition.getPartitionOption().getColumnNames().get(0)); + Assert.assertNull(subpartition.getPartitionOption().getExpression()); + Assert.assertEquals(3L, subpartition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals("P0SSP0", subpartition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals(1, subpartition.getPartitionDefinitions().get(0).getValuesList().size()); + Assert.assertEquals("100", subpartition.getPartitionDefinitions().get(0).getValuesList().get(0).get(0)); + Assert.assertEquals("P0", + subpartition.getPartitionDefinitions().get(0).getParentPartitionDefinition().getName()); + } + + @Test + public void getSubPartition_TemplateSingleHashMultipleHash_Success() { + OBOracleGetDBTableByParser table = + new OBOracleGetDBTableByParser(connection, TEST_DATABASE_NAME, "T_SINGLE_HASH_MULTIPLE_HASH"); + DBTablePartition partition = table.getPartition(); + Assert.assertEquals(5, partition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals(DBTablePartitionType.HASH, partition.getPartitionOption().getType()); + Assert.assertEquals(1, partition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("COL1", partition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals("P0", partition.getPartitionDefinitions().get(0).getName()); + DBTablePartition subpartition = partition.getSubpartition(); + Assert.assertTrue(partition.getSubpartitionTemplated()); + Assert.assertEquals(DBTablePartitionType.HASH, subpartition.getPartitionOption().getType()); + Assert.assertEquals(2, subpartition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("COL2", subpartition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals("COL3", subpartition.getPartitionOption().getColumnNames().get(1)); + Assert.assertNull(subpartition.getPartitionOption().getExpression()); + Assert.assertEquals(3L, subpartition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals("P0SP0", subpartition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals("P0", + subpartition.getPartitionDefinitions().get(0).getParentPartitionDefinition().getName()); + } + + @Test + public void getSubPartition_TemplateSingleHashSingleHash_Success() { + OBOracleGetDBTableByParser table = + new OBOracleGetDBTableByParser(connection, TEST_DATABASE_NAME, "T_SINGLE_HASH_SINGLE_HASH"); + DBTablePartition partition = table.getPartition(); + Assert.assertEquals(5, partition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals(DBTablePartitionType.HASH, partition.getPartitionOption().getType()); + Assert.assertEquals(1, partition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("COL1", partition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals("P0", partition.getPartitionDefinitions().get(0).getName()); + DBTablePartition subpartition = partition.getSubpartition(); + Assert.assertTrue(partition.getSubpartitionTemplated()); + Assert.assertEquals(DBTablePartitionType.HASH, subpartition.getPartitionOption().getType()); + Assert.assertEquals(1, subpartition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("COL2", subpartition.getPartitionOption().getColumnNames().get(0)); + Assert.assertNull(subpartition.getPartitionOption().getExpression()); + Assert.assertEquals(3L, subpartition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals("P0SP0", subpartition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals("P0", + subpartition.getPartitionDefinitions().get(0).getParentPartitionDefinition().getName()); + } + + @Test + public void getSubPartition_SingleRangeMultipleRange_Success() { + OBOracleGetDBTableByParser table = + new OBOracleGetDBTableByParser(connection, TEST_DATABASE_NAME, "SINGLE_RANGE_MULTIPLE_RANGE"); + DBTablePartition partition = table.getPartition(); + Assert.assertEquals(2, partition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals(DBTablePartitionType.RANGE, partition.getPartitionOption().getType()); + Assert.assertEquals(1, partition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("COL1", partition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals("P0", partition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals(1, partition.getPartitionDefinitions().get(0).getMaxValues().size()); + Assert.assertEquals("100", partition.getPartitionDefinitions().get(0).getMaxValues().get(0)); + DBTablePartition subpartition = partition.getSubpartition(); + Assert.assertFalse(partition.getSubpartitionTemplated()); + Assert.assertEquals(DBTablePartitionType.RANGE, subpartition.getPartitionOption().getType()); + Assert.assertEquals(2, subpartition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("COL2", subpartition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals("COL3", subpartition.getPartitionOption().getColumnNames().get(1)); + Assert.assertNull(subpartition.getPartitionOption().getExpression()); + Assert.assertNull(subpartition.getPartitionOption().getPartitionsNum()); + Assert.assertEquals("SP0", subpartition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals(2, subpartition.getPartitionDefinitions().get(0).getMaxValues().size()); + Assert.assertEquals("2020", subpartition.getPartitionDefinitions().get(0).getMaxValues().get(0)); + Assert.assertEquals("2020", subpartition.getPartitionDefinitions().get(0).getMaxValues().get(1)); + Assert.assertEquals("P0", + subpartition.getPartitionDefinitions().get(0).getParentPartitionDefinition().getName()); + } + + @Test + public void getSubPartition_SingleRangeSingleRange_Success() { + OBOracleGetDBTableByParser table = + new OBOracleGetDBTableByParser(connection, TEST_DATABASE_NAME, "SINGLE_RANGE_SINGLE_RANGE"); + DBTablePartition partition = table.getPartition(); + Assert.assertEquals(2, partition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals(DBTablePartitionType.RANGE, partition.getPartitionOption().getType()); + Assert.assertEquals(1, partition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("COL1", partition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals("P0", partition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals(1, partition.getPartitionDefinitions().get(0).getMaxValues().size()); + Assert.assertEquals("100", partition.getPartitionDefinitions().get(0).getMaxValues().get(0)); + DBTablePartition subpartition = partition.getSubpartition(); + Assert.assertFalse(partition.getSubpartitionTemplated()); + Assert.assertEquals(DBTablePartitionType.RANGE, subpartition.getPartitionOption().getType()); + Assert.assertEquals(1, subpartition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("COL2", subpartition.getPartitionOption().getColumnNames().get(0)); + Assert.assertNull(subpartition.getPartitionOption().getExpression()); + Assert.assertNull(subpartition.getPartitionOption().getPartitionsNum()); + Assert.assertEquals("SP0", subpartition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals(1, subpartition.getPartitionDefinitions().get(0).getMaxValues().size()); + Assert.assertEquals("2020", subpartition.getPartitionDefinitions().get(0).getMaxValues().get(0)); + Assert.assertEquals("P0", + subpartition.getPartitionDefinitions().get(0).getParentPartitionDefinition().getName()); + } + + @Test + public void getSubPartition_SingleHashMultipleList_Success() { + OBOracleGetDBTableByParser table = + new OBOracleGetDBTableByParser(connection, TEST_DATABASE_NAME, "SINGLE_HASH_MULTIPLE_LIST"); + DBTablePartition partition = table.getPartition(); + Assert.assertEquals(2, partition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals(DBTablePartitionType.HASH, partition.getPartitionOption().getType()); + Assert.assertEquals(1, partition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("COL1", partition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals("P0", partition.getPartitionDefinitions().get(0).getName()); + DBTablePartition subpartition = partition.getSubpartition(); + Assert.assertFalse(partition.getSubpartitionTemplated()); + Assert.assertEquals(DBTablePartitionType.LIST, subpartition.getPartitionOption().getType()); + Assert.assertEquals(2, subpartition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("COL2", subpartition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals("COL3", subpartition.getPartitionOption().getColumnNames().get(1)); + Assert.assertNull(subpartition.getPartitionOption().getExpression()); + Assert.assertNull(subpartition.getPartitionOption().getPartitionsNum()); + Assert.assertEquals("SP0", subpartition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals(2, subpartition.getPartitionDefinitions().get(0).getValuesList().size()); + Assert.assertEquals("1", subpartition.getPartitionDefinitions().get(0).getValuesList().get(0).get(0)); + Assert.assertEquals("1", subpartition.getPartitionDefinitions().get(0).getValuesList().get(0).get(1)); + Assert.assertEquals("P0", + subpartition.getPartitionDefinitions().get(0).getParentPartitionDefinition().getName()); + } + + @Test + public void getSubPartition_SingleHashSingleList_Success() { + OBOracleGetDBTableByParser table = + new OBOracleGetDBTableByParser(connection, TEST_DATABASE_NAME, "SINGLE_HASH_SINGLE_LIST"); + DBTablePartition partition = table.getPartition(); + Assert.assertEquals(2, partition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals(DBTablePartitionType.HASH, partition.getPartitionOption().getType()); + Assert.assertEquals(1, partition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("COL1", partition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals("P0", partition.getPartitionDefinitions().get(0).getName()); + DBTablePartition subpartition = partition.getSubpartition(); + Assert.assertFalse(partition.getSubpartitionTemplated()); + Assert.assertEquals(DBTablePartitionType.LIST, subpartition.getPartitionOption().getType()); + Assert.assertEquals(1, subpartition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("COL2", subpartition.getPartitionOption().getColumnNames().get(0)); + Assert.assertNull(subpartition.getPartitionOption().getExpression()); + Assert.assertNull(subpartition.getPartitionOption().getPartitionsNum()); + Assert.assertEquals("SP0", subpartition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals(2, subpartition.getPartitionDefinitions().get(0).getValuesList().size()); + Assert.assertEquals("1", subpartition.getPartitionDefinitions().get(0).getValuesList().get(0).get(0)); + Assert.assertEquals("P0", + subpartition.getPartitionDefinitions().get(0).getParentPartitionDefinition().getName()); + } + + @Test + public void getSubPartition_SingleListMultipleHash_Success() { + OBOracleGetDBTableByParser table = + new OBOracleGetDBTableByParser(connection, TEST_DATABASE_NAME, "SINGLE_LIST_SINGLE_HASH"); + DBTablePartition partition = table.getPartition(); + Assert.assertEquals(2, partition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals(DBTablePartitionType.LIST, partition.getPartitionOption().getType()); + Assert.assertEquals(1, partition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("COL1", partition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals("P0", partition.getPartitionDefinitions().get(0).getName()); + DBTablePartition subpartition = partition.getSubpartition(); + Assert.assertFalse(partition.getSubpartitionTemplated()); + Assert.assertEquals(DBTablePartitionType.HASH, subpartition.getPartitionOption().getType()); + Assert.assertEquals(1, subpartition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("COL2", subpartition.getPartitionOption().getColumnNames().get(0)); + Assert.assertNull(subpartition.getPartitionOption().getExpression()); + Assert.assertNull(subpartition.getPartitionOption().getPartitionsNum()); + Assert.assertEquals("SP0", subpartition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals("P0", + subpartition.getPartitionDefinitions().get(0).getParentPartitionDefinition().getName()); + } + + @Test + public void getSubPartition_SingleListSingleHash_Success() { + OBOracleGetDBTableByParser table = + new OBOracleGetDBTableByParser(connection, TEST_DATABASE_NAME, "SINGLE_LIST_SINGLE_HASH"); + DBTablePartition partition = table.getPartition(); + Assert.assertEquals(2, partition.getPartitionOption().getPartitionsNum().longValue()); + Assert.assertEquals(DBTablePartitionType.LIST, partition.getPartitionOption().getType()); + Assert.assertEquals(1, partition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("COL1", partition.getPartitionOption().getColumnNames().get(0)); + Assert.assertEquals("P0", partition.getPartitionDefinitions().get(0).getName()); + DBTablePartition subpartition = partition.getSubpartition(); + Assert.assertFalse(partition.getSubpartitionTemplated()); + Assert.assertEquals(DBTablePartitionType.HASH, subpartition.getPartitionOption().getType()); + Assert.assertEquals(1, subpartition.getPartitionOption().getColumnNames().size()); + Assert.assertEquals("COL2", subpartition.getPartitionOption().getColumnNames().get(0)); + Assert.assertNull(subpartition.getPartitionOption().getExpression()); + Assert.assertNull(subpartition.getPartitionOption().getPartitionsNum()); + Assert.assertEquals("SP0", subpartition.getPartitionDefinitions().get(0).getName()); + Assert.assertEquals("P0", + subpartition.getPartitionDefinitions().get(0).getParentPartitionDefinition().getName()); } @Test diff --git a/server/plugins/schema-plugin-ob-oracle/src/test/resources/parser/drop.sql b/server/plugins/schema-plugin-ob-oracle/src/test/resources/parser/drop.sql index be9424c10b..d2771f7f0f 100644 --- a/server/plugins/schema-plugin-ob-oracle/src/test/resources/parser/drop.sql +++ b/server/plugins/schema-plugin-ob-oracle/src/test/resources/parser/drop.sql @@ -13,9 +13,24 @@ BEGIN END; / -call DROPIFEXISTS_TABLE('HASH_PART_BY_PARSER'); -call DROPIFEXISTS_TABLE('LIST_PART_BY_PARSER'); -call DROPIFEXISTS_TABLE('RANGE_PART_BY_PARSER'); +call DROPIFEXISTS_TABLE('HASH_PART_SINGLE_KEY_BY_PARSER'); +call DROPIFEXISTS_TABLE('HASH_PART_MULTIPLE_KEY_BY_PARSER'); +call DROPIFEXISTS_TABLE('LIST_PART_SINGLE_KEY_BY_PARSER'); +call DROPIFEXISTS_TABLE('LIST_PART_MULTIPLE_KEY_BY_PARSER'); +call DROPIFEXISTS_TABLE('RANGE_PART_SINGLE_KEY_BY_PARSER'); +call DROPIFEXISTS_TABLE('RANGE_PART_MULTIPLE_KEY_BY_PARSER'); call DROPIFEXISTS_TABLE('CONSTRAINT_MULTY_BY_PARSER'); call DROPIFEXISTS_TABLE('CONSTRAINT_PRIMARY_BY_PARSER'); -call DROPIFEXISTS_TABLE('TEST_INDEX_BY_PARSER'); \ No newline at end of file +call DROPIFEXISTS_TABLE('TEST_INDEX_BY_PARSER'); +call DROPIFEXISTS_TABLE('T_SINGLE_RANGE_MULTIPLE_RANGE'); +call DROPIFEXISTS_TABLE('T_SINGLE_HASH_MULTIPLE_LIST'); +call DROPIFEXISTS_TABLE('T_SINGLE_HASH_MULTIPLE_HASH'); +call DROPIFEXISTS_TABLE('SINGLE_RANGE_MULTIPLE_RANGE'); +call DROPIFEXISTS_TABLE('SINGLE_HASH_MULTIPLE_LIST'); +call DROPIFEXISTS_TABLE('SINGLE_LIST_MULTIPLE_HASH'); +call DROPIFEXISTS_TABLE('T_SINGLE_RANGE_SINGLE_RANGE'); +call DROPIFEXISTS_TABLE('T_SINGLE_HASH_SINGLE_LIST'); +call DROPIFEXISTS_TABLE('T_SINGLE_HASH_SINGLE_HASH'); +call DROPIFEXISTS_TABLE('SINGLE_RANGE_SINGLE_RANGE'); +call DROPIFEXISTS_TABLE('SINGLE_HASH_SINGLE_LIST'); +call DROPIFEXISTS_TABLE('SINGLE_LIST_SINGLE_HASH'); \ No newline at end of file diff --git a/server/plugins/schema-plugin-ob-oracle/src/test/resources/parser/testGetTableByParser.sql b/server/plugins/schema-plugin-ob-oracle/src/test/resources/parser/testGetTableByParser.sql index eb4ad0cae5..2a39073e71 100644 --- a/server/plugins/schema-plugin-ob-oracle/src/test/resources/parser/testGetTableByParser.sql +++ b/server/plugins/schema-plugin-ob-oracle/src/test/resources/parser/testGetTableByParser.sql @@ -1,4 +1,4 @@ -CREATE TABLE HASH_PART_BY_PARSER (COL1 NUMBER, COL2 VARCHAR2(50)) +CREATE TABLE HASH_PART_SINGLE_KEY_BY_PARSER (COL1 NUMBER, COL2 VARCHAR2(50)) partition by hash(col1) (partition P0, partition P1, @@ -7,14 +7,24 @@ CREATE TABLE HASH_PART_BY_PARSER (COL1 NUMBER, COL2 VARCHAR2(50)) partition P4, partition P5); -CREATE TABLE LIST_PART_BY_PARSER (LOG_ID NUMBER(38), LOG_VALUE VARCHAR2(20)) +CREATE TABLE HASH_PART_MULTIPLE_KEY_BY_PARSER(COL1 INT,COL2 VARCHAR(50)) + PARTITION BY HASH(COL1,COL2) PARTITIONS 10; + +CREATE TABLE LIST_PART_SINGLE_KEY_BY_PARSER (LOG_ID NUMBER(38), LOG_VALUE VARCHAR2(20)) partition by list(log_value) (partition P01 values ('A','B','C'), partition P02 values ('D','E','F'), partition P03 values ('G','H','I'), partition P04 values (DEFAULT)); -CREATE TABLE RANGE_PART_BY_PARSER ( +CREATE TABLE LIST_PART_MULTIPLE_KEY_BY_PARSER(LOG_ID INT,COL2 INT,COL3 INT) + PARTITION BY LIST(COL2,COL3) + (PARTITION "P0" VALUES ((1,1),(3,3)), + PARTITION "P1" VALUES ((4,4),(7,7)), + PARTITION "P2" VALUES ((8,8),(9,9)) + ); + +CREATE TABLE RANGE_PART_SINGLE_KEY_BY_PARSER ( LOG_ID NUMBER(38), LOG_DATE DATE DEFAULT sysdate CONSTRAINT "RANGE_PART_OBNOTNULL_1688526018016703" NOT NULL ENABLE) partition by range(log_date) @@ -23,6 +33,14 @@ CREATE TABLE RANGE_PART_BY_PARSER ( partition M202003 values less than (TO_DATE('2020-04-01 00:00:00', 'SYYYY-MM-DD HH24:MI:SS','NLS_CALENDAR=GREGORIAN')), partition MMAX values less than (MAXVALUE)); +CREATE TABLE RANGE_PART_MULTIPLE_KEY_BY_PARSER(LOG_ID INT,COL1 INT,COL2 INT) + PARTITION BY RANGE(COL1,COL2) + (PARTITION M202001 VALUES LESS THAN(1,1) + , PARTITION M202002 VALUES LESS THAN(3,3) + , PARTITION M202003 VALUES LESS THAN(5,5) + , PARTITION MMAX VALUES LESS THAN (MAXVALUE,MAXVALUE) + ); + CREATE TABLE CONSTRAINT_PRIMARY_BY_PARSER ( COL1 NUMBER(38), COL2 NUMBER(38), @@ -65,4 +83,143 @@ CREATE INDEX IND_FUNCTION_BASED on TEST_INDEX_BY_PARSER (UPPER("COL1")) GLOBAL; CREATE UNIQUE INDEX UNIQUE_IDX_TEST_INDEX_BY_PARSER on TEST_INDEX_BY_PARSER (COL3, COL4) LOCAL; CREATE INDEX NORMAL_IDX_TEST_INDEX_BY_PARSER on TEST_INDEX_BY_PARSER (COL7) LOCAL; -CREATE INDEX IND_FUNCTION_BASED2 ON TEST_INDEX_BY_PARSER(COL8+1); \ No newline at end of file +CREATE INDEX IND_FUNCTION_BASED2 ON TEST_INDEX_BY_PARSER(COL8+1); + +CREATE TABLE T_SINGLE_RANGE_MULTIPLE_RANGE(COL1 INT,COL2 INT,COL3 INT) + PARTITION BY RANGE(COL1) + SUBPARTITION BY RANGE(COL2,COL3) + SUBPARTITION TEMPLATE + (SUBPARTITION MP0 VALUES LESS THAN(2020,2020), + SUBPARTITION MP1 VALUES LESS THAN(2021,2021), + SUBPARTITION MP2 VALUES LESS THAN(2022,2022) + ) + (PARTITION P0 VALUES LESS THAN(100), + PARTITION P1 VALUES LESS THAN(200) + ); + +CREATE TABLE T_SINGLE_HASH_MULTIPLE_LIST(COL1 INT,COL2 INT,COL3 INT) + PARTITION BY HASH(COL1) + SUBPARTITION BY LIST(COL2,COL3) + SUBPARTITION TEMPLATE + (SUBPARTITION SP0 VALUES(100,100), + SUBPARTITION SP1 VALUES(200,200), + SUBPARTITION SP2 VALUES(300,300) + ) + PARTITIONS 5; + +CREATE TABLE T_SINGLE_HASH_MULTIPLE_HASH(COL1 INT,COL2 INT,COL3 INT) + PARTITION BY HASH(COL1) + SUBPARTITION BY HASH(COL2,COL3) + SUBPARTITIONS 3 + PARTITIONS 5; + +CREATE TABLE SINGLE_RANGE_MULTIPLE_RANGE(COL1 INT,COL2 INT,COL3 INT) + PARTITION BY RANGE(COL1) + SUBPARTITION BY RANGE(COL2,COL3) + (PARTITION P0 VALUES LESS THAN(100) + (SUBPARTITION SP0 VALUES LESS THAN(2020,2020), + SUBPARTITION SP1 VALUES LESS THAN(2021,2021) + ), + PARTITION P1 VALUES LESS THAN(200) + (SUBPARTITION SP2 VALUES LESS THAN(2020,2020), + SUBPARTITION SP3 VALUES LESS THAN(2021,2021), + SUBPARTITION SP4 VALUES LESS THAN(2022,2022) + ) + ); + +CREATE TABLE SINGLE_HASH_MULTIPLE_LIST(COL1 INT,COL2 INT,COL3 INT) + PARTITION BY HASH(COL1) + SUBPARTITION BY LIST(COL2,COL3) + (PARTITION P0 + (SUBPARTITION SP0 VALUES((1,1),(3,3)), + SUBPARTITION SP1 VALUES((4,4),(7,7)) + ), + PARTITION P1 + (SUBPARTITION SP2 VALUES((1,1),(3,3)), + SUBPARTITION SP3 VALUES((4,4),(7,7)), + SUBPARTITION SP4 VALUES((8,8),(9,9)) + ) + ); + +CREATE TABLE SINGLE_LIST_MULTIPLE_HASH(COL1 INT,COL2 VARCHAR2(50),COL3 VARCHAR2(50)) + PARTITION BY LIST(COL1) + SUBPARTITION BY HASH(COL2,COL3) + (PARTITION P0 VALUES('01') + (SUBPARTITION SP0, + SUBPARTITION SP1 + ), + PARTITION P1 VALUES('02') + (SUBPARTITION SP2, + SUBPARTITION SP3, + SUBPARTITION SP4 + ) + ); + +CREATE TABLE T_SINGLE_RANGE_SINGLE_RANGE(COL1 INT,COL2 INT) + PARTITION BY RANGE(COL1) + SUBPARTITION BY RANGE(COL2) + SUBPARTITION TEMPLATE + (SUBPARTITION MP0 VALUES LESS THAN(2020), + SUBPARTITION MP1 VALUES LESS THAN(2021), + SUBPARTITION MP2 VALUES LESS THAN(2022) + ) + (PARTITION P0 VALUES LESS THAN(100), + PARTITION P1 VALUES LESS THAN(200) + ); + +CREATE TABLE T_SINGLE_HASH_SINGLE_LIST(COL1 INT,COL2 INT,COL3 INT) + PARTITION BY HASH(COL1) + SUBPARTITION BY LIST(COL2) + SUBPARTITION TEMPLATE + (SUBPARTITION SP0 VALUES(100), + SUBPARTITION SP1 VALUES(200), + SUBPARTITION SP2 VALUES(300) + ) + PARTITIONS 5; + +CREATE TABLE T_SINGLE_HASH_SINGLE_HASH(COL1 INT,COL2 INT,COL3 INT) + PARTITION BY HASH(COL1) + SUBPARTITION BY HASH(COL2) + SUBPARTITIONS 3 + PARTITIONS 5; + +CREATE TABLE SINGLE_RANGE_SINGLE_RANGE(COL1 INT,COL2 INT) + PARTITION BY RANGE(COL1) + SUBPARTITION BY RANGE(COL2) + (PARTITION P0 VALUES LESS THAN(100) + (SUBPARTITION SP0 VALUES LESS THAN(2020), + SUBPARTITION SP1 VALUES LESS THAN(2021) + ), + PARTITION P1 VALUES LESS THAN(200) + (SUBPARTITION SP2 VALUES LESS THAN(2020), + SUBPARTITION SP3 VALUES LESS THAN(2021), + SUBPARTITION SP4 VALUES LESS THAN(2022) + ) + ); + +CREATE TABLE SINGLE_HASH_SINGLE_LIST(COL1 INT,COL2 INT,COL3 INT) + PARTITION BY HASH(COL1) + SUBPARTITION BY LIST(COL2) + (PARTITION P0 + (SUBPARTITION SP0 VALUES(1,3), + SUBPARTITION SP1 VALUES(4,7) + ), + PARTITION P1 + (SUBPARTITION SP2 VALUES(1,3), + SUBPARTITION SP3 VALUES(4,7) + ) + ); + +CREATE TABLE SINGLE_LIST_SINGLE_HASH(COL1 INT,COL2 VARCHAR2(50)) + PARTITION BY LIST(COL1) + SUBPARTITION BY HASH(COL2) + (PARTITION P0 VALUES('01') + (SUBPARTITION SP0, + SUBPARTITION SP1 + ), + PARTITION P1 VALUES('02') + (SUBPARTITION SP2, + SUBPARTITION SP3, + SUBPARTITION SP4 + ) + ); diff --git a/server/plugins/schema-plugin-odp-sharding-ob-mysql/pom.xml b/server/plugins/schema-plugin-odp-sharding-ob-mysql/pom.xml index 49a1427d78..4d0456baa2 100644 --- a/server/plugins/schema-plugin-odp-sharding-ob-mysql/pom.xml +++ b/server/plugins/schema-plugin-odp-sharding-ob-mysql/pom.xml @@ -6,7 +6,7 @@ com.oceanbase plugin-parent - 4.3.2-SNAPSHOT + 4.3.3-SNAPSHOT ../pom.xml diff --git a/server/plugins/schema-plugin-odp-sharding-ob-mysql/src/main/java/com/oceanbase/odc/plugin/schema/odpsharding/obmysql/ODPShardingOBMySQLTableExtension.java b/server/plugins/schema-plugin-odp-sharding-ob-mysql/src/main/java/com/oceanbase/odc/plugin/schema/odpsharding/obmysql/ODPShardingOBMySQLTableExtension.java index 911b0aede4..0a4bb492f5 100644 --- a/server/plugins/schema-plugin-odp-sharding-ob-mysql/src/main/java/com/oceanbase/odc/plugin/schema/odpsharding/obmysql/ODPShardingOBMySQLTableExtension.java +++ b/server/plugins/schema-plugin-odp-sharding-ob-mysql/src/main/java/com/oceanbase/odc/plugin/schema/odpsharding/obmysql/ODPShardingOBMySQLTableExtension.java @@ -63,4 +63,9 @@ protected DBTableEditor getTableEditor(Connection connection) { .setType(DialectType.ODP_SHARDING_OB_MYSQL.getDBBrowserDialectTypeName()).create(); } + @Override + public boolean syncExternalTableFiles(Connection connection, String schemaName, String tableName) { + throw new UnsupportedOperationException("not implemented yet"); + } + } diff --git a/server/plugins/schema-plugin-oracle/pom.xml b/server/plugins/schema-plugin-oracle/pom.xml index bd234f03c3..1085881197 100644 --- a/server/plugins/schema-plugin-oracle/pom.xml +++ b/server/plugins/schema-plugin-oracle/pom.xml @@ -23,7 +23,7 @@ com.oceanbase plugin-parent - 4.3.2-SNAPSHOT + 4.3.3-SNAPSHOT ../pom.xml diff --git a/server/plugins/schema-plugin-oracle/src/main/java/com/oceanbase/odc/plugin/schema/oracle/OracleTableExtension.java b/server/plugins/schema-plugin-oracle/src/main/java/com/oceanbase/odc/plugin/schema/oracle/OracleTableExtension.java index d0ee22b295..e3eccc2747 100644 --- a/server/plugins/schema-plugin-oracle/src/main/java/com/oceanbase/odc/plugin/schema/oracle/OracleTableExtension.java +++ b/server/plugins/schema-plugin-oracle/src/main/java/com/oceanbase/odc/plugin/schema/oracle/OracleTableExtension.java @@ -24,6 +24,7 @@ import com.oceanbase.odc.plugin.schema.oboracle.OBOracleTableExtension; import com.oceanbase.odc.plugin.schema.oracle.utils.DBAccessorUtil; import com.oceanbase.tools.dbbrowser.editor.DBTableEditor; +import com.oceanbase.tools.dbbrowser.model.DBObjectType; import com.oceanbase.tools.dbbrowser.model.DBTable; import com.oceanbase.tools.dbbrowser.model.DBTableStats; import com.oceanbase.tools.dbbrowser.schema.DBSchemaAccessor; @@ -47,15 +48,25 @@ public DBTable getDetail(@NonNull Connection connection, @NonNull String schemaN table.setOwner(schemaName); table.setName(tableName); table.setColumns(schemaAccessor.listTableColumns(schemaName, tableName)); - table.setConstraints(schemaAccessor.listTableConstraints(schemaName, tableName)); table.setPartition(schemaAccessor.getPartition(schemaName, tableName)); - table.setIndexes(schemaAccessor.listTableIndexes(schemaName, tableName)); + if (!schemaAccessor.isExternalTable(schemaName, tableName)) { + table.setConstraints(schemaAccessor.listTableConstraints(schemaName, tableName)); + table.setIndexes(schemaAccessor.listTableIndexes(schemaName, tableName)); + table.setType(DBObjectType.TABLE); + } else { + table.setType(DBObjectType.EXTERNAL_TABLE); + } table.setDDL(schemaAccessor.getTableDDL(schemaName, tableName)); table.setTableOptions(schemaAccessor.getTableOptions(schemaName, tableName)); table.setStats(getTableStats(connection, schemaName, tableName)); return table; } + @Override + public boolean syncExternalTableFiles(Connection connection, String schemaName, String tableName) { + throw new UnsupportedOperationException("not implemented yet"); + } + @Override protected DBTableStats getTableStats(@NonNull Connection connection, @NonNull String schemaName, @NonNull String tableName) { diff --git a/server/plugins/schema-plugin-postgres/pom.xml b/server/plugins/schema-plugin-postgres/pom.xml index c91c5440b3..cd517d9b6c 100644 --- a/server/plugins/schema-plugin-postgres/pom.xml +++ b/server/plugins/schema-plugin-postgres/pom.xml @@ -22,7 +22,7 @@ plugin-parent com.oceanbase - 4.3.2-SNAPSHOT + 4.3.3-SNAPSHOT ../pom.xml diff --git a/server/plugins/schema-plugin-postgres/src/main/java/com/oceanbase/odc/plugin/schema/postgres/PostgresTableExtension.java b/server/plugins/schema-plugin-postgres/src/main/java/com/oceanbase/odc/plugin/schema/postgres/PostgresTableExtension.java index 68d31c186f..d2ee8bf4cf 100644 --- a/server/plugins/schema-plugin-postgres/src/main/java/com/oceanbase/odc/plugin/schema/postgres/PostgresTableExtension.java +++ b/server/plugins/schema-plugin-postgres/src/main/java/com/oceanbase/odc/plugin/schema/postgres/PostgresTableExtension.java @@ -31,5 +31,8 @@ protected DBSchemaAccessor getSchemaAccessor(Connection connection) { return DBAccessorUtil.getSchemaAccessor(connection); } - + @Override + public boolean syncExternalTableFiles(Connection connection, String schemaName, String tableName) { + throw new UnsupportedOperationException("not implemented yet"); + } } diff --git a/server/plugins/task-plugin-api/pom.xml b/server/plugins/task-plugin-api/pom.xml index c417da8a22..2eed9f3b42 100644 --- a/server/plugins/task-plugin-api/pom.xml +++ b/server/plugins/task-plugin-api/pom.xml @@ -5,7 +5,7 @@ plugin-parent com.oceanbase - 4.3.2-SNAPSHOT + 4.3.3-SNAPSHOT ../pom.xml 4.0.0 diff --git a/server/plugins/task-plugin-doris/pom.xml b/server/plugins/task-plugin-doris/pom.xml index f12956bed7..2f127b508c 100644 --- a/server/plugins/task-plugin-doris/pom.xml +++ b/server/plugins/task-plugin-doris/pom.xml @@ -5,7 +5,7 @@ plugin-parent com.oceanbase - 4.3.2-SNAPSHOT + 4.3.3-SNAPSHOT 4.0.0 diff --git a/server/plugins/task-plugin-doris/src/main/java/com/oceanbase/odc/plugin/task/doris/datatransfer/job/DorisScriptImportJob.java b/server/plugins/task-plugin-doris/src/main/java/com/oceanbase/odc/plugin/task/doris/datatransfer/job/DorisScriptImportJob.java index 95644e354d..2af1779ce0 100644 --- a/server/plugins/task-plugin-doris/src/main/java/com/oceanbase/odc/plugin/task/doris/datatransfer/job/DorisScriptImportJob.java +++ b/server/plugins/task-plugin-doris/src/main/java/com/oceanbase/odc/plugin/task/doris/datatransfer/job/DorisScriptImportJob.java @@ -63,7 +63,8 @@ protected boolean isObjectExists() throws SQLException { DBObjectIdentity target = DBObjectIdentity.of(object.getSchema(), DBObjectType.getEnumByName(object.getType()), object.getName()); try (Connection conn = dataSource.getConnection()) { - List tables = new DorisTableExtension().list(conn, object.getSchema()); + List tables = + new DorisTableExtension().list(conn, object.getSchema(), DBObjectType.TABLE); return CollectionUtils.containsAny(tables, target); } } diff --git a/server/plugins/task-plugin-doris/src/main/java/com/oceanbase/odc/plugin/task/doris/datatransfer/job/DorisTransferJobFactory.java b/server/plugins/task-plugin-doris/src/main/java/com/oceanbase/odc/plugin/task/doris/datatransfer/job/DorisTransferJobFactory.java index f1a447e10e..4c69f220f8 100644 --- a/server/plugins/task-plugin-doris/src/main/java/com/oceanbase/odc/plugin/task/doris/datatransfer/job/DorisTransferJobFactory.java +++ b/server/plugins/task-plugin-doris/src/main/java/com/oceanbase/odc/plugin/task/doris/datatransfer/job/DorisTransferJobFactory.java @@ -33,6 +33,7 @@ import com.oceanbase.odc.plugin.task.mysql.datatransfer.job.datax.model.JobContent.Parameter; import com.oceanbase.odc.plugin.task.mysql.datatransfer.job.datax.model.parameter.MySQLWriterPluginParameter; import com.oceanbase.odc.plugin.task.mysql.datatransfer.job.factory.BaseTransferJobFactory; +import com.oceanbase.tools.dbbrowser.model.DBObjectType; import com.oceanbase.tools.dbbrowser.model.DBTableColumn; import com.oceanbase.tools.loaddump.common.enums.ObjectType; @@ -55,7 +56,7 @@ protected List queryTableColumns(Connection connection, ObjectRes @Override protected List queryTransferObjects(Connection connection, boolean transferDDL) { - return new DorisTableExtension().list(connection, transferConfig.getSchemaName()).stream() + return new DorisTableExtension().list(connection, transferConfig.getSchemaName(), DBObjectType.TABLE).stream() .map(table -> new DataTransferObject(ObjectType.TABLE, table.getName())) .collect(Collectors.toList()); } diff --git a/server/plugins/task-plugin-mysql/pom.xml b/server/plugins/task-plugin-mysql/pom.xml index e553272924..1441170c0d 100644 --- a/server/plugins/task-plugin-mysql/pom.xml +++ b/server/plugins/task-plugin-mysql/pom.xml @@ -5,7 +5,7 @@ plugin-parent com.oceanbase - 4.3.2-SNAPSHOT + 4.3.3-SNAPSHOT 4.0.0 diff --git a/server/plugins/task-plugin-mysql/src/main/java/com/oceanbase/odc/plugin/task/mysql/datatransfer/job/MySQLSqlScriptImportJob.java b/server/plugins/task-plugin-mysql/src/main/java/com/oceanbase/odc/plugin/task/mysql/datatransfer/job/MySQLSqlScriptImportJob.java index d340ad3d88..0227e87138 100644 --- a/server/plugins/task-plugin-mysql/src/main/java/com/oceanbase/odc/plugin/task/mysql/datatransfer/job/MySQLSqlScriptImportJob.java +++ b/server/plugins/task-plugin-mysql/src/main/java/com/oceanbase/odc/plugin/task/mysql/datatransfer/job/MySQLSqlScriptImportJob.java @@ -63,7 +63,7 @@ protected boolean isObjectExists() throws SQLException { try (Connection conn = dataSource.getConnection()) { switch (object.getType()) { case "TABLE": - objects = new MySQLTableExtension().list(conn, object.getSchema()); + objects = new MySQLTableExtension().list(conn, object.getSchema(), DBObjectType.TABLE); break; case "VIEW": objects = new MySQLViewExtension().list(conn, object.getSchema()); diff --git a/server/plugins/task-plugin-mysql/src/main/java/com/oceanbase/odc/plugin/task/mysql/datatransfer/job/factory/MySQLTransferJobFactory.java b/server/plugins/task-plugin-mysql/src/main/java/com/oceanbase/odc/plugin/task/mysql/datatransfer/job/factory/MySQLTransferJobFactory.java index 4ba2e3c955..6625e34ed1 100644 --- a/server/plugins/task-plugin-mysql/src/main/java/com/oceanbase/odc/plugin/task/mysql/datatransfer/job/factory/MySQLTransferJobFactory.java +++ b/server/plugins/task-plugin-mysql/src/main/java/com/oceanbase/odc/plugin/task/mysql/datatransfer/job/factory/MySQLTransferJobFactory.java @@ -35,6 +35,7 @@ import com.oceanbase.odc.plugin.task.mysql.datatransfer.job.MySQLSqlScriptImportJob; import com.oceanbase.odc.plugin.task.mysql.datatransfer.job.datax.DataXTransferJob; import com.oceanbase.odc.plugin.task.mysql.datatransfer.job.datax.model.JobConfiguration; +import com.oceanbase.tools.dbbrowser.model.DBObjectType; import com.oceanbase.tools.dbbrowser.model.DBTableColumn; import com.oceanbase.tools.loaddump.common.enums.ObjectType; @@ -58,7 +59,7 @@ protected List queryTableColumns(Connection connection, ObjectRes @Override protected List queryTransferObjects(Connection connection, boolean transferDDL) { List objects = new ArrayList<>(); - new MySQLTableExtension().list(connection, transferConfig.getSchemaName()) + new MySQLTableExtension().list(connection, transferConfig.getSchemaName(), DBObjectType.TABLE) .forEach(table -> objects.add(new DataTransferObject(ObjectType.TABLE, table.getName()))); if (transferDDL) { new MySQLViewExtension().list(connection, transferConfig.getSchemaName()) diff --git a/server/plugins/task-plugin-ob-mysql/pom.xml b/server/plugins/task-plugin-ob-mysql/pom.xml index df29e3d63d..916cacdd0e 100644 --- a/server/plugins/task-plugin-ob-mysql/pom.xml +++ b/server/plugins/task-plugin-ob-mysql/pom.xml @@ -5,7 +5,7 @@ plugin-parent com.oceanbase - 4.3.2-SNAPSHOT + 4.3.3-SNAPSHOT 4.0.0 diff --git a/server/plugins/task-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/task/obmysql/datatransfer/factory/LoadParameterFactory.java b/server/plugins/task-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/task/obmysql/datatransfer/factory/LoadParameterFactory.java index 2dd52d6f54..65ad35dd50 100644 --- a/server/plugins/task-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/task/obmysql/datatransfer/factory/LoadParameterFactory.java +++ b/server/plugins/task-plugin-ob-mysql/src/main/java/com/oceanbase/odc/plugin/task/obmysql/datatransfer/factory/LoadParameterFactory.java @@ -29,6 +29,7 @@ import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.Validate; +import com.google.common.base.MoreObjects; import com.oceanbase.odc.common.util.StringUtils; import com.oceanbase.odc.plugin.task.api.datatransfer.model.CsvColumnMapping; import com.oceanbase.odc.plugin.task.api.datatransfer.model.DataTransferConfig; @@ -52,6 +53,8 @@ */ @Slf4j public class LoadParameterFactory extends BaseParameterFactory { + private static final Integer DEFAULT_INSERT_BATCH_SIZE = 100; + public LoadParameterFactory(DataTransferConfig config, File workingDir, File logDir) { super(config, workingDir, logDir); } @@ -83,9 +86,8 @@ protected LoadParameter doGenerate(File workingDir) throws IOException { } if (transferConfig.isTransferData()) { parameter.setTruncatable(transferConfig.isTruncateTableBeforeImport()); - if (transferConfig.getBatchCommitNum() != null) { - parameter.setBatchSize(transferConfig.getBatchCommitNum()); - } + parameter.setBatchSize( + MoreObjects.firstNonNull(transferConfig.getBatchCommitNum(), DEFAULT_INSERT_BATCH_SIZE)); if (transferConfig.getSkippedDataType() != null) { parameter.getExcludeDataTypes().addAll(transferConfig.getSkippedDataType()); } @@ -95,6 +97,11 @@ protected LoadParameter doGenerate(File workingDir) throws IOException { parameter.setTruncatable(transferConfig.isTruncateTableBeforeImport()); setCsvMappings(parameter, transferConfig); setWhiteListForExternalCsv(parameter, transferConfig, workingDir); + parameter.setBatchSize( + MoreObjects.firstNonNull(transferConfig.getBatchCommitNum(), DEFAULT_INSERT_BATCH_SIZE)); + if (transferConfig.getSkippedDataType() != null) { + parameter.getExcludeDataTypes().addAll(transferConfig.getSkippedDataType()); + } } else if (transferConfig.getDataTransferFormat() == DataTransferFormat.SQL) { parameter.setReplaceObjectIfExists(true); } diff --git a/server/plugins/task-plugin-ob-oracle/pom.xml b/server/plugins/task-plugin-ob-oracle/pom.xml index 838967f209..29ff2da494 100644 --- a/server/plugins/task-plugin-ob-oracle/pom.xml +++ b/server/plugins/task-plugin-ob-oracle/pom.xml @@ -5,7 +5,7 @@ plugin-parent com.oceanbase - 4.3.2-SNAPSHOT + 4.3.3-SNAPSHOT 4.0.0 diff --git a/server/plugins/task-plugin-oracle/pom.xml b/server/plugins/task-plugin-oracle/pom.xml index 923fd2a552..cc9c9124e1 100644 --- a/server/plugins/task-plugin-oracle/pom.xml +++ b/server/plugins/task-plugin-oracle/pom.xml @@ -21,7 +21,7 @@ plugin-parent com.oceanbase - 4.3.2-SNAPSHOT + 4.3.3-SNAPSHOT 4.0.0 diff --git a/server/plugins/task-plugin-oracle/src/main/java/com/oceanbase/odc/plugin/task/oracle/datatransfer/job/OracleSqlScriptImportJob.java b/server/plugins/task-plugin-oracle/src/main/java/com/oceanbase/odc/plugin/task/oracle/datatransfer/job/OracleSqlScriptImportJob.java index 1fda4f8c18..43216ebea0 100644 --- a/server/plugins/task-plugin-oracle/src/main/java/com/oceanbase/odc/plugin/task/oracle/datatransfer/job/OracleSqlScriptImportJob.java +++ b/server/plugins/task-plugin-oracle/src/main/java/com/oceanbase/odc/plugin/task/oracle/datatransfer/job/OracleSqlScriptImportJob.java @@ -72,7 +72,7 @@ protected boolean isObjectExists() throws SQLException { ObjectType type = ObjectType.valueOfName(object.getType()); switch (type) { case TABLE: - objects = new OracleTableExtension().list(conn, object.getSchema()); + objects = new OracleTableExtension().list(conn, object.getSchema(), DBObjectType.TABLE); break; case VIEW: objects = new OracleViewExtension().list(conn, object.getSchema()); diff --git a/server/plugins/task-plugin-oracle/src/main/java/com/oceanbase/odc/plugin/task/oracle/datatransfer/job/factory/OracleTransferJobFactory.java b/server/plugins/task-plugin-oracle/src/main/java/com/oceanbase/odc/plugin/task/oracle/datatransfer/job/factory/OracleTransferJobFactory.java index 513586026f..2c77ef1a43 100644 --- a/server/plugins/task-plugin-oracle/src/main/java/com/oceanbase/odc/plugin/task/oracle/datatransfer/job/factory/OracleTransferJobFactory.java +++ b/server/plugins/task-plugin-oracle/src/main/java/com/oceanbase/odc/plugin/task/oracle/datatransfer/job/factory/OracleTransferJobFactory.java @@ -46,6 +46,7 @@ import com.oceanbase.odc.plugin.task.mysql.datatransfer.job.factory.BaseTransferJobFactory; import com.oceanbase.odc.plugin.task.oracle.datatransfer.job.OracleSchemaExportJob; import com.oceanbase.odc.plugin.task.oracle.datatransfer.job.OracleSqlScriptImportJob; +import com.oceanbase.tools.dbbrowser.model.DBObjectType; import com.oceanbase.tools.dbbrowser.model.DBSynonymType; import com.oceanbase.tools.dbbrowser.model.DBTableColumn; import com.oceanbase.tools.dbbrowser.util.OracleSqlBuilder; @@ -70,7 +71,7 @@ protected List queryTableColumns(Connection connection, ObjectRes @Override protected List queryTransferObjects(Connection connection, boolean transferDDL) { List objects = new ArrayList<>(); - new OracleTableExtension().list(connection, transferConfig.getSchemaName()) + new OracleTableExtension().list(connection, transferConfig.getSchemaName(), DBObjectType.TABLE) .forEach(table -> objects.add(new DataTransferObject(ObjectType.TABLE, table.getName()))); if (transferDDL) { new OracleViewExtension().list(connection, transferConfig.getSchemaName()) diff --git a/server/starters/desktop-starter/pom.xml b/server/starters/desktop-starter/pom.xml index d21ff1931c..9dcdd92000 100644 --- a/server/starters/desktop-starter/pom.xml +++ b/server/starters/desktop-starter/pom.xml @@ -5,7 +5,7 @@ starter-parent com.oceanbase - 4.3.2-SNAPSHOT + 4.3.3-SNAPSHOT ../pom.xml desktop-starter diff --git a/server/starters/pom.xml b/server/starters/pom.xml index ce350454ec..ed3dcbb7b0 100644 --- a/server/starters/pom.xml +++ b/server/starters/pom.xml @@ -5,7 +5,7 @@ com.oceanbase odc-parent - 4.3.2-SNAPSHOT + 4.3.3-SNAPSHOT ../../pom.xml pom diff --git a/server/starters/web-starter/pom.xml b/server/starters/web-starter/pom.xml index 75c369f7ea..a0300c5e11 100644 --- a/server/starters/web-starter/pom.xml +++ b/server/starters/web-starter/pom.xml @@ -5,7 +5,7 @@ starter-parent com.oceanbase - 4.3.2-SNAPSHOT + 4.3.3-SNAPSHOT ../pom.xml web-starter diff --git a/server/starters/web-starter/src/main/java/com/oceanbase/odc/config/WebSecurityConfiguration.java b/server/starters/web-starter/src/main/java/com/oceanbase/odc/config/WebSecurityConfiguration.java index 8360ef658f..47d272a096 100644 --- a/server/starters/web-starter/src/main/java/com/oceanbase/odc/config/WebSecurityConfiguration.java +++ b/server/starters/web-starter/src/main/java/com/oceanbase/odc/config/WebSecurityConfiguration.java @@ -18,6 +18,8 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.boot.web.servlet.FilterRegistrationBean; +import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Profile; import org.springframework.security.authentication.AuthenticationManager; @@ -26,6 +28,11 @@ import org.springframework.security.config.annotation.web.builders.WebSecurity; import org.springframework.security.config.annotation.web.configuration.WebSecurityConfigurerAdapter; import org.springframework.security.config.http.SessionCreationPolicy; +import org.springframework.security.saml2.provider.service.metadata.OpenSamlMetadataResolver; +import org.springframework.security.saml2.provider.service.registration.RelyingPartyRegistrationRepository; +import org.springframework.security.saml2.provider.service.web.DefaultRelyingPartyRegistrationResolver; +import org.springframework.security.saml2.provider.service.web.RelyingPartyRegistrationResolver; +import org.springframework.security.saml2.provider.service.web.Saml2MetadataFilter; import org.springframework.security.web.authentication.UsernamePasswordAuthenticationFilter; import org.springframework.security.web.authentication.logout.LogoutSuccessHandler; import org.springframework.security.web.context.SecurityContextRepository; @@ -50,6 +57,9 @@ import com.oceanbase.odc.service.iam.auth.ldap.ODCLdapAuthenticator; import com.oceanbase.odc.service.iam.auth.local.LocalDaoAuthenticationProvider; import com.oceanbase.odc.service.iam.auth.oauth2.OAuth2SecurityConfigureHelper; +import com.oceanbase.odc.service.iam.auth.saml.CustomSamlProvider; +import com.oceanbase.odc.service.iam.auth.saml.DefaultSamlUserService; +import com.oceanbase.odc.service.iam.auth.saml.SamlSecurityConfigureHelper; import com.oceanbase.odc.service.iam.util.FailedLoginAttemptLimiter; import com.oceanbase.odc.service.integration.ldap.LdapConfigRegistrationManager; @@ -118,6 +128,14 @@ public class WebSecurityConfiguration extends WebSecurityConfigurerAdapter { @Autowired private LdapConfigRegistrationManager ldapConfigRegistrationManager; + @Autowired + private RelyingPartyRegistrationRepository registrations; + + @Autowired + private DefaultSamlUserService defaultSamlUserService; + + @Autowired + private SamlSecurityConfigureHelper samlSecurityConfigureHelper; private BastionAuthenticationProvider bastionAuthenticationProvider() { return new BastionAuthenticationProvider(bastionUserDetailService); @@ -127,6 +145,7 @@ private BastionAuthenticationProvider bastionAuthenticationProvider() { public void configure(AuthenticationManagerBuilder auth) { auth.authenticationProvider(localDaoAuthenticationProvider) .authenticationProvider(bastionAuthenticationProvider()) + .authenticationProvider(new CustomSamlProvider(defaultSamlUserService)) .authenticationProvider( new ODCLdapAuthenticationProvider(new ODCLdapAuthenticator(ldapConfigRegistrationManager), ldapUserDetailsContextMapper)); @@ -139,11 +158,28 @@ public void configure(WebSecurity web) { .and().ignoring().antMatchers(commonSecurityProperties.getAuthWhitelist()); } + @Bean + RelyingPartyRegistrationResolver relyingPartyRegistrationResolver( + RelyingPartyRegistrationRepository registrations) { + return new DefaultRelyingPartyRegistrationResolver(registrations); + } + + @Bean + FilterRegistrationBean metadata(RelyingPartyRegistrationResolver registrations) { + Saml2MetadataFilter metadata = new Saml2MetadataFilter(registrations, new OpenSamlMetadataResolver()); + FilterRegistrationBean filter = new FilterRegistrationBean<>(metadata); + filter.setOrder(-101); + return filter; + } + + @Override protected void configure(HttpSecurity http) throws Exception { corsConfigureHelper.configure(http); usernamePasswordConfigureHelper.configure(http, authenticationManager()); oauth2SecurityConfigureHelper.configure(http); + samlSecurityConfigureHelper.configure(http, authenticationManager()); + ldapSecurityConfigureHelper.configure(http, authenticationManager()); SecurityContextRepository securityContextRepository = securityContextRepository(); diff --git a/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/iam/auth/MappingRuleConvert.java b/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/iam/auth/MappingRuleConvert.java index fd812b904f..21a28c838a 100644 --- a/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/iam/auth/MappingRuleConvert.java +++ b/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/iam/auth/MappingRuleConvert.java @@ -23,6 +23,7 @@ import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.stream.Collectors; import javax.annotation.Nullable; import javax.naming.NamingEnumeration; @@ -35,6 +36,8 @@ import org.springframework.ldap.core.DirContextOperations; import org.springframework.security.authentication.InternalAuthenticationServiceException; import org.springframework.security.oauth2.client.userinfo.OAuth2UserRequest; +import org.springframework.security.saml2.provider.service.authentication.DefaultSaml2AuthenticatedPrincipal; +import org.springframework.security.saml2.provider.service.authentication.Saml2Authentication; import org.springframework.stereotype.Component; import com.google.common.base.MoreObjects; @@ -51,6 +54,7 @@ import com.oceanbase.odc.service.integration.model.SSOIntegrationConfig.MappingRule; import com.oceanbase.odc.service.integration.oauth2.AddableClientRegistrationManager; import com.oceanbase.odc.service.integration.oauth2.TestLoginManager; +import com.oceanbase.odc.service.integration.saml.AddableRelyingPartyRegistrationRepository; import lombok.NonNull; import lombok.SneakyThrows; @@ -69,11 +73,14 @@ public class MappingRuleConvert { @Autowired private TestLoginManager testLoginManager; + @Autowired + private AddableRelyingPartyRegistrationRepository addableRelyingPartyRegistrationRepository; + public MappingResult resolveOAuthMappingResult(OAuth2UserRequest userRequest, Map userInfoMap) { MappingRule mappingRule = resolveMappingRule(userRequest); Verify.notNull(mappingRule, "mappingRule"); userInfoMap = getUserInfoMapFromResponse(userInfoMap, mappingRule); - testLoginManager.saveOauth2TestIdIfNeed(JsonUtils.toJson(userInfoMap)); + testLoginManager.saveOauth2InfoIfNeed(JsonUtils.toJson(userInfoMap)); testLoginManager.abortIfOAuthTestLoginInfo(); Long organizationId = parseOrganizationId(userRequest.getClientRegistration().getRegistrationId()); String userAccountName = String.valueOf(userInfoMap.get(mappingRule.getUserAccountNameField())); @@ -110,6 +117,40 @@ public MappingResult resolveLdapMappingResult(DirContextOperations ctx, String u .build(); } + public MappingResult resolveSamlMappingResult(Saml2Authentication saml2Authentication) { + DefaultSaml2AuthenticatedPrincipal principal = + (DefaultSaml2AuthenticatedPrincipal) saml2Authentication.getPrincipal(); + Map userInfoMap = getUserInfoMap(principal); + testLoginManager.saveSamlInfoIfNeed(JsonUtils.toJson(userInfoMap)); + testLoginManager.abortIfSamlTestLogin(); + + String relyingPartyRegistrationId = principal.getRelyingPartyRegistrationId(); + + SSOIntegrationConfig ssoIntegrationConfig = + addableRelyingPartyRegistrationRepository.findConfigByRegistrationId(relyingPartyRegistrationId); + com.google.common.base.Verify.verifyNotNull(ssoIntegrationConfig, "ssoIntegrationConfig"); + MappingRule mappingRule = ssoIntegrationConfig.getMappingRule(); + String name = principal.getName(); + String parseExtraInfo = parseExtraInfo(userInfoMap, mappingRule); + + return MappingResult.builder() + .organizationId(SSOIntegrationConfig.parseOrganizationId(relyingPartyRegistrationId)) + .userAccountName(name) + .userNickName(getName(userInfoMap, mappingRule)) + .isAdmin(false) + .extraInfo(parseExtraInfo) + .sourceUserInfoMap(userInfoMap) + .build(); + } + + private Map getUserInfoMap(DefaultSaml2AuthenticatedPrincipal principal) { + return principal.getAttributes().entrySet().stream() + .collect(Collectors.toMap( + Map.Entry::getKey, + entry -> entry.getValue().size() == 1 ? entry.getValue().get(0) : entry.getValue())); + } + + @SneakyThrows private Map getUserInfoMap(DirContextOperations ctx) { NamingEnumeration all = ctx.getAttributes().getAll(); @@ -149,9 +190,9 @@ private MappingRule resolveMappingRule(OAuth2UserRequest userRequest) { @Nullable - private String getName(Map oAuth2User, MappingRule mappingRule) { + private String getName(Map userInfoMap, MappingRule mappingRule) { Set nameFields = MoreObjects.firstNonNull(mappingRule.getUserNickNameField(), new HashSet<>()); - Object o = nameFields.stream().map(oAuth2User::get).filter(Objects::nonNull).findFirst().orElse(null); + Object o = nameFields.stream().map(userInfoMap::get).filter(Objects::nonNull).findFirst().orElse(null); return o == null ? null : String.valueOf(o); } diff --git a/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/iam/auth/oauth2/CustomOAuth2AuthorizationRequestResolver.java b/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/iam/auth/oauth2/CustomOAuth2AuthorizationRequestResolver.java index 661d759e93..588ba09747 100644 --- a/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/iam/auth/oauth2/CustomOAuth2AuthorizationRequestResolver.java +++ b/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/iam/auth/oauth2/CustomOAuth2AuthorizationRequestResolver.java @@ -33,7 +33,7 @@ import com.oceanbase.odc.service.integration.model.Oauth2Parameter; import com.oceanbase.odc.service.integration.model.SSOIntegrationConfig; import com.oceanbase.odc.service.integration.oauth2.AddableClientRegistrationManager; -import com.oceanbase.odc.service.integration.oauth2.Oauth2StateManager; +import com.oceanbase.odc.service.integration.oauth2.SSOStateManager; import com.oceanbase.odc.service.state.StatefulUuidStateIdGenerator; import lombok.SneakyThrows; @@ -44,14 +44,14 @@ public class CustomOAuth2AuthorizationRequestResolver implements OAuth2Authoriza private final AddableClientRegistrationManager addableClientRegistrationManager; private StatefulUuidStateIdGenerator statefulUuidStateIdGenerator; - private Oauth2StateManager oauth2StateManager; + private SSOStateManager sSOStateManager; private AntPathRequestMatcher authorizationRequestMatcher; public CustomOAuth2AuthorizationRequestResolver(AddableClientRegistrationManager addableClientRegistrationManager, - StatefulUuidStateIdGenerator statefulUuidStateIdGenerator, Oauth2StateManager oauth2StateManager) { + StatefulUuidStateIdGenerator statefulUuidStateIdGenerator, SSOStateManager sSOStateManager) { this.addableClientRegistrationManager = addableClientRegistrationManager; this.statefulUuidStateIdGenerator = statefulUuidStateIdGenerator; - this.oauth2StateManager = oauth2StateManager; + this.sSOStateManager = sSOStateManager; DefaultOAuth2AuthorizationRequestResolver defaultOAuth2AuthorizationRequestResolver = new DefaultOAuth2AuthorizationRequestResolver( addableClientRegistrationManager, "/oauth2/authorization"); @@ -95,10 +95,10 @@ private OAuth2AuthorizationRequest doResolve(HttpServletRequest request, String String state = authorizationRequest.getState(); String originRedirectUrl = authorizationRequest.getRedirectUri(); UriComponentsBuilder.fromUriString(originRedirectUrl).build().getQueryParams() - .forEach((key, value) -> oauth2StateManager.setStateParameter(state, key, urlDecode(value.get(0)))); + .forEach((key, value) -> sSOStateManager.setStateParameter(state, key, urlDecode(value.get(0)))); URL url = new URL(originRedirectUrl); String urlWithoutQuery = new URL(url.getProtocol(), url.getHost(), url.getPort(), url.getPath()).toString(); - oauth2StateManager.setOdcParameters(state, request); + sSOStateManager.setOdcParameters(state, request); return OAuth2AuthorizationRequest.from(authorizationRequest) .redirectUri(urlWithoutQuery) diff --git a/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/iam/auth/oauth2/OAuth2SecurityConfigureHelper.java b/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/iam/auth/oauth2/OAuth2SecurityConfigureHelper.java index ea0fa20787..831ceec0cc 100644 --- a/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/iam/auth/oauth2/OAuth2SecurityConfigureHelper.java +++ b/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/iam/auth/oauth2/OAuth2SecurityConfigureHelper.java @@ -29,7 +29,7 @@ import com.oceanbase.odc.service.iam.auth.CustomAuthenticationFailureHandler; import com.oceanbase.odc.service.iam.auth.CustomAuthenticationSuccessHandler; import com.oceanbase.odc.service.integration.oauth2.AddableClientRegistrationManager; -import com.oceanbase.odc.service.integration.oauth2.Oauth2StateManager; +import com.oceanbase.odc.service.integration.oauth2.SSOStateManager; import com.oceanbase.odc.service.state.StatefulUuidStateIdGenerator; @Component @@ -54,7 +54,7 @@ public class OAuth2SecurityConfigureHelper { @Autowired private StatefulUuidStateIdGenerator statefulUuidStateIdGenerator; @Autowired - private Oauth2StateManager oauth2StateManager; + private SSOStateManager SSOStateManager; public void configure(HttpSecurity http) @@ -65,7 +65,7 @@ public void configure(HttpSecurity http) .authorizationEndpoint() .authorizationRequestResolver( new CustomOAuth2AuthorizationRequestResolver(this.addableClientRegistrationManager, - statefulUuidStateIdGenerator, oauth2StateManager)) + statefulUuidStateIdGenerator, SSOStateManager)) .and() // token 端点配置, 根据 code 获取 token .tokenEndpoint() @@ -78,7 +78,7 @@ public void configure(HttpSecurity http) .oidcUserService(oidcUserService); http.addFilterBefore( - new OAuth2AbstractTestLoginAuthenticationFilter(), + new OAuth2TestLoginAuthenticationFilter(), OAuth2LoginAuthenticationFilter.class); } } diff --git a/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/iam/auth/oauth2/OAuth2AbstractTestLoginAuthenticationFilter.java b/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/iam/auth/oauth2/OAuth2TestLoginAuthenticationFilter.java similarity index 90% rename from server/starters/web-starter/src/main/java/com/oceanbase/odc/service/iam/auth/oauth2/OAuth2AbstractTestLoginAuthenticationFilter.java rename to server/starters/web-starter/src/main/java/com/oceanbase/odc/service/iam/auth/oauth2/OAuth2TestLoginAuthenticationFilter.java index a4f10f0d0d..1eae4b730b 100644 --- a/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/iam/auth/oauth2/OAuth2AbstractTestLoginAuthenticationFilter.java +++ b/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/iam/auth/oauth2/OAuth2TestLoginAuthenticationFilter.java @@ -20,7 +20,7 @@ import com.oceanbase.odc.service.iam.auth.local.AbstractTestLoginAuthenticationFilter; import com.oceanbase.odc.service.integration.oauth2.TestLoginManager; -public class OAuth2AbstractTestLoginAuthenticationFilter extends AbstractTestLoginAuthenticationFilter { +public class OAuth2TestLoginAuthenticationFilter extends AbstractTestLoginAuthenticationFilter { @Override protected Boolean isTestRequest(HttpServletRequest request) { diff --git a/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/iam/auth/oauth2/OAuth2UserServiceImpl.java b/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/iam/auth/oauth2/OAuth2UserServiceImpl.java index a384181317..cb8aae90ed 100644 --- a/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/iam/auth/oauth2/OAuth2UserServiceImpl.java +++ b/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/iam/auth/oauth2/OAuth2UserServiceImpl.java @@ -36,6 +36,7 @@ import org.springframework.security.oauth2.core.OAuth2AuthenticationException; import org.springframework.security.oauth2.core.OAuth2AuthorizationException; import org.springframework.security.oauth2.core.OAuth2Error; +import org.springframework.security.oauth2.core.endpoint.OAuth2ParameterNames; import org.springframework.security.oauth2.core.user.OAuth2User; import org.springframework.stereotype.Service; import org.springframework.util.StringUtils; @@ -49,7 +50,7 @@ import com.oceanbase.odc.service.iam.auth.MappingRuleConvert; import com.oceanbase.odc.service.iam.auth.SsoUserDetailService; import com.oceanbase.odc.service.iam.model.User; -import com.oceanbase.odc.service.integration.oauth2.Oauth2StateManager; +import com.oceanbase.odc.service.integration.oauth2.SSOStateManager; import com.oceanbase.odc.service.integration.oauth2.TestLoginManager; /** @@ -80,7 +81,7 @@ public class OAuth2UserServiceImpl implements OAuth2UserService> response = getResponse(userRequest, request); PreConditions.notNull(response, "oAuth2User"); - oauth2StateManager.addStateToCurrentRequestParam(); + ssoStateManager.addStateToCurrentRequestParam(OAuth2ParameterNames.STATE); MappingResult mappingResult = mappingRuleConvert.resolveOAuthMappingResult(userRequest, response.getBody()); testLoginManager.abortIfOAuthTestLoginTest(); diff --git a/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/iam/auth/oauth2/OidcUserServiceImpl.java b/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/iam/auth/oauth2/OidcUserServiceImpl.java index fa865c22f8..7eb3618124 100644 --- a/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/iam/auth/oauth2/OidcUserServiceImpl.java +++ b/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/iam/auth/oauth2/OidcUserServiceImpl.java @@ -40,6 +40,7 @@ import org.springframework.security.oauth2.core.OAuth2Error; import org.springframework.security.oauth2.core.converter.ClaimConversionService; import org.springframework.security.oauth2.core.converter.ClaimTypeConverter; +import org.springframework.security.oauth2.core.endpoint.OAuth2ParameterNames; import org.springframework.security.oauth2.core.oidc.OidcIdToken; import org.springframework.security.oauth2.core.oidc.OidcScopes; import org.springframework.security.oauth2.core.oidc.OidcUserInfo; @@ -56,7 +57,7 @@ import com.oceanbase.odc.service.iam.auth.MappingRuleConvert; import com.oceanbase.odc.service.iam.auth.SsoUserDetailService; import com.oceanbase.odc.service.iam.model.User; -import com.oceanbase.odc.service.integration.oauth2.Oauth2StateManager; +import com.oceanbase.odc.service.integration.oauth2.SSOStateManager; import com.oceanbase.odc.service.integration.oauth2.TestLoginManager; /** @@ -88,7 +89,7 @@ public class OidcUserServiceImpl implements OAuth2UserService> createDefaultClaimTypeConverters() { Converter booleanConverter = getConverter(TypeDescriptor.valueOf(Boolean.class)); @@ -135,7 +136,7 @@ public User loadUser(OidcUserRequest userRequest) throws OAuth2AuthenticationExc } } Map userInfoMap = collectClaims(userRequest.getIdToken(), userInfo); - oauth2StateManager.addStateToCurrentRequestParam(); + SSOStateManager.addStateToCurrentRequestParam(OAuth2ParameterNames.STATE); MappingResult mappingResult = mappingRuleConvert.resolveOAuthMappingResult(userRequest, userInfoMap); testLoginManager.abortIfOAuthTestLoginTest(); diff --git a/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/iam/auth/saml/CustomSamlProvider.java b/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/iam/auth/saml/CustomSamlProvider.java new file mode 100644 index 0000000000..ca826a4e3d --- /dev/null +++ b/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/iam/auth/saml/CustomSamlProvider.java @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.iam.auth.saml; + +import org.springframework.security.authentication.AuthenticationProvider; +import org.springframework.security.core.Authentication; +import org.springframework.security.core.AuthenticationException; +import org.springframework.security.saml2.provider.service.authentication.OpenSamlAuthenticationProvider; +import org.springframework.security.saml2.provider.service.authentication.Saml2Authentication; + +import com.oceanbase.odc.core.shared.Verify; +import com.oceanbase.odc.service.iam.model.User; + +/** + * why ues OpenSamlAuthenticationProvider instead of OpenSaml4AuthenticationProvider ? according to + * {@link "https://github.com/spring-projects/spring-security/issues/11434"}, + * OpenSaml4AuthenticationProvider depends on opensaml 4.1+, which requires jdk11. In the current + * version is loaded by default implementation OpenSaml4AuthenticationProvider, although it has been + * deprecated. + */ +public class CustomSamlProvider implements AuthenticationProvider { + + private final AuthenticationProvider defaultAuthenticationProvider; + + private final DefaultSamlUserService defaultSamlUserService; + + public CustomSamlProvider(DefaultSamlUserService defaultSamlUserService) { + this.defaultSamlUserService = defaultSamlUserService; + defaultAuthenticationProvider = new OpenSamlAuthenticationProvider(); + } + + @Override + public Authentication authenticate(Authentication authentication) throws AuthenticationException { + Authentication authenticate = defaultAuthenticationProvider.authenticate(authentication); + Verify.verify(authenticate instanceof Saml2Authentication, + "invalid type of authentication, class: " + authentication.getClass()); + Saml2Authentication saml2Authentication = (Saml2Authentication) authenticate; + User user = defaultSamlUserService.loadUser(saml2Authentication); + return new Saml2Authentication(user, ((Saml2Authentication) authenticate).getSaml2Response(), + authenticate.getAuthorities()); + } + + @Override + public boolean supports(Class authentication) { + return defaultAuthenticationProvider.supports(authentication); + } +} diff --git a/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/iam/auth/saml/DefaultSamlUserService.java b/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/iam/auth/saml/DefaultSamlUserService.java new file mode 100644 index 0000000000..7ae43048b8 --- /dev/null +++ b/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/iam/auth/saml/DefaultSamlUserService.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.iam.auth.saml; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.security.saml2.provider.service.authentication.Saml2Authentication; +import org.springframework.stereotype.Service; + +import com.oceanbase.odc.core.authority.util.SkipAuthorize; +import com.oceanbase.odc.service.iam.auth.MappingRuleConvert; +import com.oceanbase.odc.service.iam.auth.SsoUserDetailService; +import com.oceanbase.odc.service.iam.auth.oauth2.MappingResult; +import com.oceanbase.odc.service.iam.model.User; + +@Service +public class DefaultSamlUserService { + + @Autowired + private SsoUserDetailService ssoUserDetailService; + + @Autowired + private MappingRuleConvert mappingRuleConvert; + + @SkipAuthorize + public User loadUser(Saml2Authentication saml2Authentication) { + MappingResult mappingResult = mappingRuleConvert.resolveSamlMappingResult(saml2Authentication); + return ssoUserDetailService.getOrCreateUser(mappingResult); + } + + +} diff --git a/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/iam/auth/saml/SamlSecurityConfigureHelper.java b/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/iam/auth/saml/SamlSecurityConfigureHelper.java new file mode 100644 index 0000000000..2690de072c --- /dev/null +++ b/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/iam/auth/saml/SamlSecurityConfigureHelper.java @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.iam.auth.saml; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.annotation.Profile; +import org.springframework.security.authentication.AuthenticationManager; +import org.springframework.security.config.annotation.web.builders.HttpSecurity; +import org.springframework.security.saml2.provider.service.registration.RelyingPartyRegistrationRepository; +import org.springframework.security.saml2.provider.service.servlet.filter.Saml2WebSsoAuthenticationFilter; +import org.springframework.stereotype.Component; + +import com.oceanbase.odc.service.iam.auth.CustomAuthenticationFailureHandler; +import com.oceanbase.odc.service.iam.auth.CustomAuthenticationSuccessHandler; + +@Component +@Profile("alipay") +@ConditionalOnProperty(value = {"odc.iam.auth.type"}, havingValue = "local") +public class SamlSecurityConfigureHelper { + + + @Autowired + private RelyingPartyRegistrationRepository registrations; + + @Autowired + private CustomAuthenticationSuccessHandler customAuthenticationSuccessHandler; + + @Autowired + private CustomAuthenticationFailureHandler customAuthenticationFailureHandler; + + public void configure(HttpSecurity http, AuthenticationManager authenticationManager) + throws Exception { + http.saml2Login(); + Saml2WebSsoAuthenticationFilter saml2WebSsoAuthenticationFilter = + new Saml2WebSsoAuthenticationFilter(registrations); + saml2WebSsoAuthenticationFilter.setAuthenticationSuccessHandler(customAuthenticationSuccessHandler); + saml2WebSsoAuthenticationFilter.setAuthenticationFailureHandler(customAuthenticationFailureHandler); + saml2WebSsoAuthenticationFilter.setAuthenticationManager(authenticationManager); + http.addFilterBefore( + saml2WebSsoAuthenticationFilter, + Saml2WebSsoAuthenticationFilter.class); + http.addFilterBefore( + new SamlTestLoginAuthenticationFilter(), + Saml2WebSsoAuthenticationFilter.class); + } +} diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/DefaultK8sJobClientSelector.java b/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/iam/auth/saml/SamlTestLoginAuthenticationFilter.java similarity index 56% rename from server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/DefaultK8sJobClientSelector.java rename to server/starters/web-starter/src/main/java/com/oceanbase/odc/service/iam/auth/saml/SamlTestLoginAuthenticationFilter.java index ed460b9814..e82de6f241 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/task/caller/DefaultK8sJobClientSelector.java +++ b/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/iam/auth/saml/SamlTestLoginAuthenticationFilter.java @@ -13,18 +13,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.oceanbase.odc.service.task.caller; +package com.oceanbase.odc.service.iam.auth.saml; -public class DefaultK8sJobClientSelector implements K8sJobClientSelector { +import javax.servlet.http.HttpServletRequest; - private final K8sJobClient k8sJobClient; - - public DefaultK8sJobClientSelector(K8sJobClient k8sJobClient) { - this.k8sJobClient = k8sJobClient; - } +import com.oceanbase.odc.service.iam.auth.local.AbstractTestLoginAuthenticationFilter; +import com.oceanbase.odc.service.integration.oauth2.TestLoginManager; +public class SamlTestLoginAuthenticationFilter extends AbstractTestLoginAuthenticationFilter { @Override - public K8sJobClient select(JobContext jobContext) { - return k8sJobClient; + protected Boolean isTestRequest(HttpServletRequest request) { + return TestLoginManager.isSamlTestLoginRequest(request); } } diff --git a/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/info/WebInfoAdapter.java b/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/info/WebInfoAdapter.java index 2473cb79f7..e044107b6a 100644 --- a/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/info/WebInfoAdapter.java +++ b/server/starters/web-starter/src/main/java/com/oceanbase/odc/service/info/WebInfoAdapter.java @@ -39,17 +39,17 @@ @SkipAuthorize("odc internal usage") public class WebInfoAdapter implements InfoAdapter { - @Value("${odc.iam.password-login-enabled:true}") - private Boolean passwordLoginEnabled; @Value("${odc.iam.auth.type}") protected Set authType; + @Autowired + protected IntegrationService integrationService; + @Value("${odc.iam.password-login-enabled:true}") + private Boolean passwordLoginEnabled; @Value("${odc.help.supportGroupQRCodeUrl:#{null}}") private String supportGroupQRCodeUrl; @Autowired private PlaysiteOpenApiProperties alipayOpenApiProperties; @Autowired - protected IntegrationService integrationService; - @Autowired private BuildProperties buildProperties; @Override @@ -66,7 +66,7 @@ public String getLoginUrl(HttpServletRequest request) { return alipayOpenApiProperties.getObOfficialLoginUrl(); } else if (authType.contains("local")) { SSOIntegrationConfig sSoClientRegistration = integrationService.getSSoIntegrationConfig(); - if (sSoClientRegistration == null || !sSoClientRegistration.isOauth2OrOidc()) { + if (sSoClientRegistration == null) { return null; } return sSoClientRegistration.resolveLoginRedirectUrl();