feat(vermeer): support task priority based scheduling (#336)

* feat(Scheduler): Support priority/elder/depends based scheduling.

* chore: some tiny error

* Update vermeer/.gitignore

* Update CodeQL workflow to use latest actions

---------

Co-authored-by: imbajin <[email protected]>

Author: Ethereal-O

.github/workflows/codeql-analysis.yml

@@ -32,10 +32,10 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
 
       - name: Setup Java JDK
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           distribution: 'zulu'
           java-version: '11'
@@ -48,7 +48,7 @@ jobs:
 
       # Initializes the CodeQL tools for scanning.
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@v2
+        uses: github/codeql-action/init@v3
         with:
           languages: ${{ matrix.language }}
           # If you wish to specify custom queries, you can do so here or in a config file.
@@ -59,7 +59,7 @@ jobs:
       # Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
       # If this step fails, then you should remove it and run the build manually (see below)
       - name: Autobuild
-        uses: github/codeql-action/autobuild@v2
+        uses: github/codeql-action/autobuild@v3
 
       # ℹ️ Command-line programs to run using the OS shell.
       # 📚 https://git.io/JvXDl
@@ -73,13 +73,13 @@ jobs:
       #   make release
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v2
+        uses: github/codeql-action/analyze@v3
 
   dependency-review:
     runs-on: ubuntu-latest
     steps:
       - name: 'Checkout Repository'
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
       - name: 'Dependency Review'
-        uses: actions/dependency-review-action@v3
+        uses: actions/dependency-review-action@v4
 

vermeer/.gitignore

@@ -83,3 +83,7 @@ node_modules/
 /output/
 /bin/*
 !/bin/*.sh
+
+# Others #
+######################
+test/case/

vermeer/README.md

@@ -3,6 +3,28 @@
 ## Introduction
 Vermeer is a high-performance distributed graph computing platform based on memory, supporting more than 15 graph algorithms, custom algorithm extensions, and custom data source access.
 
+## Run with Docker
+
+Pull the image:
+```
+docker pull hugegraph/vermeer:latest
+```
+
+Create local configuration files, for example, `~/master.ini` and `~/worker.ini`.
+
+Run with Docker. The `--env` flag specifies the file name.
+
+```
+master: docker run -v ~/:/go/bin/config hugegraph/vermeer --env=master
+worker: docker run -v ~/:/go/bin/config hugegraph/vermeer --env=worker
+```
+
+We've also provided a `docker-compose` file. Once you've created `~/master.ini` and `~/worker.ini`, and updated the `master_peer` in `worker.ini` to `172.20.0.10:6689`, you can run it using the following command:
+
+```
+docker-compose up -d
+```
+
 ## Start
 
 ```

vermeer/README.zh-CN.md

@@ -3,6 +3,26 @@
 ## 简介
 Vermeer是一个基于内存的高性能分布式图计算平台，支持15+图算法。支持自定义算法扩展，支持自定义数据源接入。
 
+## 基于 Docker 运行
+
+拉取镜像
+```
+docker pull hugegraph/vermeer:latest
+```
+
+创建好本地配置文件，例如`~/master.ini`与`~/worker.ini`
+
+基于docker运行，其中`--env`指定的是文件名称。
+```
+master: docker run -v ~/:/go/bin/config hugegraph/vermeer --env=master
+worker: docker run -v ~/:/go/bin/config hugegraph/vermeer --env=worker
+```
+
+我们也提供了`docker-compose`文件，当创建好`~/master.ini`与`~/worker.ini`，将`worker.ini`中的`master_peer`修改为`172.20.0.10:6689`后，即可通过以下命令运行：
+```
+docker-compose up -d
+```
+
 ## 运行
 
 ```

vermeer/apps/graphio/local_file.go

@@ -82,6 +82,7 @@ func (a *LocalMaker) MakeTasks(params map[string]string, taskID int32) ([]LoadPa
 			logrus.Errorf(s)
 			return nil, errors.New(s)
 		}
+		logrus.Debugf("MakeTask LoadTypeLocal parse file: %s, s:%d, e:%d", files, s, e)
 		for i := s; i <= e; i++ {
 			part := LoadPartition{}
 			part.Init(partID, taskID, LoadPartTypeVertex)

vermeer/apps/master/bl/compute_task.go

@@ -142,6 +142,8 @@ func (ctb *ComputeTaskBl) ComputeTaskStatus(
 				}
 			}
 			taskMgr.ForceState(computeTask.Task, structure.TaskStateComplete)
+			// for scheduler, mark task complete
+			Scheduler.taskManager.MarkTaskComplete(taskId)
 			graph.SubUsingNum()
 			computeTask.FreeMemory()
 			needQuery := options.GetInt(computeTask.Task.Params, "output.need_query") == 1

vermeer/apps/master/bl/grpc_handlers.go

@@ -26,6 +26,7 @@ import (
 	"time"
 	"vermeer/apps/compute"
 	"vermeer/apps/graphio"
+	"vermeer/apps/master/schedules"
 	"vermeer/apps/master/threshold"
 	"vermeer/apps/master/workers"
 	pb "vermeer/apps/protos"
@@ -103,6 +104,11 @@ func (h *ServerHandler) SayHelloMaster(ctx context.Context, req *pb.HelloMasterR
 		logrus.Errorf("failed to add a WorkerClient to the WorkerManager, error: %s", err)
 		return &pb.HelloMasterResp{}, err
 	}
+	_, err = Scheduler.ChangeWorkerStatus(reqWorker.Name, schedules.WorkerOngoingStatusIdle)
+	if err != nil {
+		logrus.Errorf("failed to change worker status to idle, error: %s", err)
+		return &pb.HelloMasterResp{}, err
+	}
 
 	logrus.Infof("worker say hello name: %s and set to workgroup: %s, client: %s", reqWorker.Name, reqWorker.Group, p.Addr.String())
 

vermeer/apps/master/bl/load_task.go

@@ -204,6 +204,9 @@ func (lb *LoadTaskBl) LoadTaskStatus(taskId int32, state string, workerName stri
 			loadTask.Task.SetState(structure.TaskStateLoaded)
 			//TaskMgr.ForceState(loadTask.Task, structure.TaskStateLoaded)
 
+			// for scheduler, mark task complete
+			Scheduler.taskManager.MarkTaskComplete(taskId)
+
 			logrus.Infof("graph: %s, vertex: %d, edge: %d", graph.Name, graph.VertexCount, graph.EdgeCount)
 			for _, w := range graph.Workers {
 				logrus.Infof(