add gitlab_push

Author: alrocar

README.md

@@ -12,6 +12,7 @@ Deploy this project to your Tinybird account and configure the webhooks for the
 
 - [Auth0 Log Streams](https://www.tinybird.co/docs/get-data-in/guides/ingest-auth0-logs)
 - [Clerk](https://www.tinybird.co/docs/get-data-in/guides/ingest-from-clerk)
+- [GitLab](https://www.tinybird.co/docs/get-data-in/guides/ingest-from-gitlab)
 - [Orb](https://www.tinybird.co/docs/get-data-in/guides/ingest-from-orb)
 - [Resend](https://www.tinybird.co/docs/get-data-in/guides/ingest-from-resend)
 - [Rudderstack](https://www.tinybird.co/docs/get-data-in/guides/ingest-from-rudderstack)

tinybird/datasources/fixtures/gitlab_merge_request.ndjson

@@ -0,0 +1 @@
+{"object_kind":"merge_request","event_type":"merge_request","user":{"id":1,"name":"Administrator","username":"root","avatar_url":"http://www.gravatar.com/avatar/e64c7d89f26bd1972efa854d13d7dd61?s=40&d=identicon","email":"[email protected]"},"project":{"id":1,"name":"Gitlab Test","description":"Aut reprehenderit ut est.","web_url":"http://example.com/gitlabhq/gitlab-test","avatar_url":null,"git_ssh_url":"[email protected]:gitlabhq/gitlab-test.git","git_http_url":"http://example.com/gitlabhq/gitlab-test.git","namespace":"GitlabHQ","visibility_level":20,"path_with_namespace":"gitlabhq/gitlab-test","default_branch":"master","ci_config_path":"","homepage":"http://example.com/gitlabhq/gitlab-test","url":"http://example.com/gitlabhq/gitlab-test.git","ssh_url":"[email protected]:gitlabhq/gitlab-test.git","http_url":"http://example.com/gitlabhq/gitlab-test.git"},"repository":{"name":"Gitlab Test","url":"http://example.com/gitlabhq/gitlab-test.git","description":"Aut reprehenderit ut est.","homepage":"http://example.com/gitlabhq/gitlab-test"},"object_attributes":{"id":99,"iid":1,"target_branch":"master","source_branch":"ms-viewport","source_project_id":14,"author_id":51,"assignee_ids":[6],"assignee_id":6,"reviewer_ids":[6],"title":"MS-Viewport","created_at":"2013-12-03T17:23:34Z","updated_at":"2013-12-03T17:23:34Z","last_edited_at":"2013-12-03T17:23:34Z","last_edited_by_id":1,"milestone_id":null,"state_id":1,"state":"opened","blocking_discussions_resolved":true,"work_in_progress":false,"draft":false,"first_contribution":true,"merge_status":"unchecked","target_project_id":14,"description":"","prepared_at":"2013-12-03T19:23:34Z","total_time_spent":1800,"time_change":30,"human_total_time_spent":"30m","human_time_change":"30s","human_time_estimate":"30m","url":"http://example.com/diaspora/merge_requests/1","source":{"name":"Awesome Project","description":"Aut reprehenderit ut est.","web_url":"http://example.com/awesome_space/awesome_project","avatar_url":null,"git_ssh_url":"[email protected]:awesome_space/awesome_project.git","git_http_url":"http://example.com/awesome_space/awesome_project.git","namespace":"Awesome Space","visibility_level":20,"path_with_namespace":"awesome_space/awesome_project","default_branch":"master","homepage":"http://example.com/awesome_space/awesome_project","url":"http://example.com/awesome_space/awesome_project.git","ssh_url":"[email protected]:awesome_space/awesome_project.git","http_url":"http://example.com/awesome_space/awesome_project.git"},"target":{"name":"Awesome Project","description":"Aut reprehenderit ut est.","web_url":"http://example.com/awesome_space/awesome_project","avatar_url":null,"git_ssh_url":"[email protected]:awesome_space/awesome_project.git","git_http_url":"http://example.com/awesome_space/awesome_project.git","namespace":"Awesome Space","visibility_level":20,"path_with_namespace":"awesome_space/awesome_project","default_branch":"master","homepage":"http://example.com/awesome_space/awesome_project","url":"http://example.com/awesome_space/awesome_project.git","ssh_url":"[email protected]:awesome_space/awesome_project.git","http_url":"http://example.com/awesome_space/awesome_project.git"},"last_commit":{"id":"da1560886d4f094c3e6c9ef40349f7d38b5d27d7","message":"fixed readme","title":"Update file README.md","timestamp":"2012-01-03T23:36:29+02:00","url":"http://example.com/awesome_space/awesome_project/commits/da1560886d4f094c3e6c9ef40349f7d38b5d27d7","author":{"name":"GitLab dev user","email":"gitlabdev@dv6700.(none)"}},"labels":[{"id":206,"title":"API","color":"#ffffff","project_id":14,"created_at":"2013-12-03T17:15:43Z","updated_at":"2013-12-03T17:15:43Z","template":false,"description":"API related issues","type":"ProjectLabel","group_id":41}],"action":"open","detailed_merge_status":"mergeable"},"labels":[{"id":206,"title":"API","color":"#ffffff","project_id":14,"created_at":"2013-12-03T17:15:43Z","updated_at":"2013-12-03T17:15:43Z","template":false,"description":"API related issues","type":"ProjectLabel","group_id":41}],"changes":{"updated_by_id":{"previous":null,"current":1},"draft":{"previous":true,"current":false},"updated_at":{"previous":"2017-09-15 16:50:55 UTC","current":"2017-09-15 16:52:00 UTC"},"labels":{"previous":[{"id":206,"title":"API","color":"#ffffff","project_id":14,"created_at":"2013-12-03T17:15:43Z","updated_at":"2013-12-03T17:15:43Z","template":false,"description":"API related issues","type":"ProjectLabel","group_id":41}],"current":[{"id":205,"title":"Platform","color":"#123123","project_id":14,"created_at":"2013-12-03T17:15:43Z","updated_at":"2013-12-03T17:15:43Z","template":false,"description":"Platform related issues","type":"ProjectLabel","group_id":41}]},"last_edited_at":{"previous":null,"current":"2023-03-15 00:00:10 UTC"},"last_edited_by_id":{"previous":null,"current":3278533}},"assignees":[{"id":6,"name":"User1","username":"user1","avatar_url":"http://www.gravatar.com/avatar/e64c7d89f26bd1972efa854d13d7dd61?s=40&d=identicon"}],"reviewers":[{"id":6,"name":"User1","username":"user1","avatar_url":"http://www.gravatar.com/avatar/e64c7d89f26bd1972efa854d13d7dd61?s=40&d=identicon"}]}

tinybird/datasources/fixtures/gitlab_pipeline.ndjson

@@ -0,0 +1 @@
+{"object_kind":"pipeline","object_attributes":{"id":31,"iid":3,"name":"Pipeline for branch: master","ref":"master","tag":false,"sha":"bcbb5ec396a2c0f828686f14fac9b80b780504f2","before_sha":"bcbb5ec396a2c0f828686f14fac9b80b780504f2","source":"merge_request_event","status":"success","stages":["build","test","deploy"],"created_at":"2016-08-12 15:23:28 UTC","finished_at":"2016-08-12 15:26:29 UTC","duration":63,"variables":[{"key":"NESTOR_PROD_ENVIRONMENT","value":"us-west-1"}],"url":"http://example.com/gitlab-org/gitlab-test/-/pipelines/31"},"merge_request":{"id":1,"iid":1,"title":"Test","source_branch":"test","source_project_id":1,"target_branch":"master","target_project_id":1,"state":"opened","merge_status":"can_be_merged","detailed_merge_status":"mergeable","url":"http://192.168.64.1:3005/gitlab-org/gitlab-test/merge_requests/1"},"user":{"id":1,"name":"Administrator","username":"root","avatar_url":"http://www.gravatar.com/avatar/e32bd13e2add097461cb96824b7a829c?s=80&d=identicon","email":"[email protected]"},"project":{"id":1,"name":"Gitlab Test","description":"Atque in sunt eos similique dolores voluptatem.","web_url":"http://192.168.64.1:3005/gitlab-org/gitlab-test","avatar_url":null,"git_ssh_url":"[email protected]:gitlab-org/gitlab-test.git","git_http_url":"http://192.168.64.1:3005/gitlab-org/gitlab-test.git","namespace":"Gitlab Org","visibility_level":20,"path_with_namespace":"gitlab-org/gitlab-test","default_branch":"master"},"commit":{"id":"bcbb5ec396a2c0f828686f14fac9b80b780504f2","message":"test\n","timestamp":"2016-08-12T17:23:21+02:00","url":"http://example.com/gitlab-org/gitlab-test/commit/bcbb5ec396a2c0f828686f14fac9b80b780504f2","author":{"name":"User","email":"[email protected]"}},"source_pipeline":{"project":{"id":41,"web_url":"https://gitlab.example.com/gitlab-org/upstream-project","path_with_namespace":"gitlab-org/upstream-project"},"pipeline_id":30,"job_id":3401},"builds":[{"id":380,"stage":"deploy","name":"production","status":"skipped","created_at":"2016-08-12 15:23:28 UTC","started_at":null,"finished_at":null,"duration":null,"queued_duration":null,"failure_reason":null,"when":"manual","manual":true,"allow_failure":false,"user":{"id":1,"name":"Administrator","username":"root","avatar_url":"http://www.gravatar.com/avatar/e32bd13e2add097461cb96824b7a829c?s=80&d=identicon","email":"[email protected]"},"runner":null,"artifacts_file":{"filename":null,"size":null},"environment":{"name":"production","action":"start","deployment_tier":"production"}},{"id":377,"stage":"test","name":"test-image","status":"success","created_at":"2016-08-12 15:23:28 UTC","started_at":"2016-08-12 15:26:12 UTC","finished_at":"2016-08-12 15:26:29 UTC","duration":17.0,"queued_duration":196.0,"failure_reason":null,"when":"on_success","manual":false,"allow_failure":false,"user":{"id":1,"name":"Administrator","username":"root","avatar_url":"http://www.gravatar.com/avatar/e32bd13e2add097461cb96824b7a829c?s=80&d=identicon","email":"[email protected]"},"runner":{"id":380987,"description":"shared-runners-manager-6.gitlab.com","active":true,"runner_type":"instance_type","is_shared":true,"tags":["linux","docker","shared-runner"]},"artifacts_file":{"filename":null,"size":null},"environment":null},{"id":378,"stage":"test","name":"test-build","status":"failed","created_at":"2016-08-12 15:23:28 UTC","started_at":"2016-08-12 15:26:12 UTC","finished_at":"2016-08-12 15:26:29 UTC","duration":17.0,"queued_duration":196.0,"failure_reason":"script_failure","when":"on_success","manual":false,"allow_failure":false,"user":{"id":1,"name":"Administrator","username":"root","avatar_url":"http://www.gravatar.com/avatar/e32bd13e2add097461cb96824b7a829c?s=80&d=identicon","email":"[email protected]"},"runner":{"id":380987,"description":"shared-runners-manager-6.gitlab.com","active":true,"runner_type":"instance_type","is_shared":true,"tags":["linux","docker"]},"artifacts_file":{"filename":null,"size":null},"environment":null},{"id":376,"stage":"build","name":"build-image","status":"success","created_at":"2016-08-12 15:23:28 UTC","started_at":"2016-08-12 15:24:56 UTC","finished_at":"2016-08-12 15:25:26 UTC","duration":17.0,"queued_duration":196.0,"failure_reason":null,"when":"on_success","manual":false,"allow_failure":false,"user":{"id":1,"name":"Administrator","username":"root","avatar_url":"http://www.gravatar.com/avatar/e32bd13e2add097461cb96824b7a829c?s=80&d=identicon","email":"[email protected]"},"runner":{"id":380987,"description":"shared-runners-manager-6.gitlab.com","active":true,"runner_type":"instance_type","is_shared":true,"tags":["linux","docker"]},"artifacts_file":{"filename":null,"size":null},"environment":null},{"id":379,"stage":"deploy","name":"staging","status":"created","created_at":"2016-08-12 15:23:28 UTC","started_at":null,"finished_at":null,"duration":null,"queued_duration":null,"failure_reason":null,"when":"on_success","manual":false,"allow_failure":false,"user":{"id":1,"name":"Administrator","username":"root","avatar_url":"http://www.gravatar.com/avatar/e32bd13e2add097461cb96824b7a829c?s=80&d=identicon","email":"[email protected]"},"runner":null,"artifacts_file":{"filename":null,"size":null},"environment":{"name":"staging","action":"start","deployment_tier":"staging"}}]}

tinybird/datasources/fixtures/gitlab_push.ndjson

@@ -0,0 +1 @@
+{"object_kind":"push","event_name":"push","before":"95790bf891e76fee5e1747ab589903a6a1f80f22","after":"da1560886d4f094c3e6c9ef40349f7d38b5d27d7","ref":"refs/heads/master","ref_protected":true,"checkout_sha":"da1560886d4f094c3e6c9ef40349f7d38b5d27d7","user_id":4,"user_name":"John Smith","user_username":"jsmith","user_email":"[email protected]","user_avatar":"https://s.gravatar.com/avatar/d4c74594d841139328695756648b6bd6?s=8://s.gravatar.com/avatar/d4c74594d841139328695756648b6bd6?s=80","project_id":15,"project":{"id":15,"name":"Diaspora","description":"","web_url":"http://example.com/mike/diaspora","avatar_url":null,"git_ssh_url":"[email protected]:mike/diaspora.git","git_http_url":"http://example.com/mike/diaspora.git","namespace":"Mike","visibility_level":0,"path_with_namespace":"mike/diaspora","default_branch":"master","homepage":"http://example.com/mike/diaspora","url":"[email protected]:mike/diaspora.git","ssh_url":"[email protected]:mike/diaspora.git","http_url":"http://example.com/mike/diaspora.git"},"repository":{"name":"Diaspora","url":"[email protected]:mike/diaspora.git","description":"","homepage":"http://example.com/mike/diaspora","git_http_url":"http://example.com/mike/diaspora.git","git_ssh_url":"[email protected]:mike/diaspora.git","visibility_level":0},"commits":[{"id":"b6568db1bc1dcd7f8b4d5a946b0b91f9dacd7327","message":"Update Catalan translation to e38cb41.\n\nSee https://gitlab.com/gitlab-org/gitlab for more information","title":"Update Catalan translation to e38cb41.","timestamp":"2011-12-12T14:27:31+02:00","url":"http://example.com/mike/diaspora/commit/b6568db1bc1dcd7f8b4d5a946b0b91f9dacd7327","author":{"name":"Jordi Mallach","email":"[email protected]"},"added":["CHANGELOG"],"modified":["app/controller/application.rb"],"removed":[]},{"id":"da1560886d4f094c3e6c9ef40349f7d38b5d27d7","message":"fixed readme","title":"fixed readme","timestamp":"2012-01-03T23:36:29+02:00","url":"http://example.com/mike/diaspora/commit/da1560886d4f094c3e6c9ef40349f7d38b5d27d7","author":{"name":"GitLab dev user","email":"gitlabdev@dv6700.(none)"},"added":["CHANGELOG"],"modified":["app/controller/application.rb"],"removed":[]}],"total_commits_count":4}

tinybird/datasources/fixtures/gitlab_work_item.ndjson

@@ -0,0 +1 @@
+{"object_kind":"issue","event_type":"issue","user":{"id":1,"name":"Administrator","username":"root","avatar_url":"http://www.gravatar.com/avatar/e64c7d89f26bd1972efa854d13d7dd61?s=40&d=identicon","email":"[email protected]"},"project":{"id":1,"name":"Gitlab Test","description":"Aut reprehenderit ut est.","web_url":"http://example.com/gitlabhq/gitlab-test","avatar_url":null,"git_ssh_url":"[email protected]:gitlabhq/gitlab-test.git","git_http_url":"http://example.com/gitlabhq/gitlab-test.git","namespace":"GitlabHQ","visibility_level":20,"path_with_namespace":"gitlabhq/gitlab-test","default_branch":"master","ci_config_path":null,"homepage":"http://example.com/gitlabhq/gitlab-test","url":"http://example.com/gitlabhq/gitlab-test.git","ssh_url":"[email protected]:gitlabhq/gitlab-test.git","http_url":"http://example.com/gitlabhq/gitlab-test.git"},"object_attributes":{"id":301,"title":"New API: create/update/delete file","assignee_ids":[51],"assignee_id":51,"author_id":51,"project_id":14,"created_at":"2013-12-03T17:15:43Z","updated_at":"2013-12-03T17:15:43Z","updated_by_id":1,"last_edited_at":null,"last_edited_by_id":null,"relative_position":0,"description":"Create new API for manipulations with repository","milestone_id":null,"state_id":1,"confidential":false,"discussion_locked":true,"due_date":null,"moved_to_id":null,"duplicated_to_id":null,"time_estimate":0,"total_time_spent":0,"time_change":0,"human_total_time_spent":null,"human_time_estimate":null,"human_time_change":null,"weight":null,"health_status":"at_risk","type":"Issue","iid":23,"url":"http://example.com/diaspora/issues/23","state":"opened","action":"open","severity":"high","escalation_status":"triggered","escalation_policy":{"id":18,"name":"Engineering On-call"},"labels":[{"id":206,"title":"API","color":"#ffffff","project_id":14,"created_at":"2013-12-03T17:15:43Z","updated_at":"2013-12-03T17:15:43Z","template":false,"description":"API related issues","type":"ProjectLabel","group_id":41}]},"repository":{"name":"Gitlab Test","url":"http://example.com/gitlabhq/gitlab-test.git","description":"Aut reprehenderit ut est.","homepage":"http://example.com/gitlabhq/gitlab-test"},"assignees":[{"name":"User1","username":"user1","avatar_url":"http://www.gravatar.com/avatar/e64c7d89f26bd1972efa854d13d7dd61?s=40&d=identicon"}],"assignee":{"name":"User1","username":"user1","avatar_url":"http://www.gravatar.com/avatar/e64c7d89f26bd1972efa854d13d7dd61?s=40&d=identicon"},"labels":[{"id":206,"title":"API","color":"#ffffff","project_id":14,"created_at":"2013-12-03T17:15:43Z","updated_at":"2013-12-03T17:15:43Z","template":false,"description":"API related issues","type":"ProjectLabel","group_id":41}],"changes":{"updated_by_id":{"previous":null,"current":1},"updated_at":{"previous":"2017-09-15 16:50:55 UTC","current":"2017-09-15 16:52:00 UTC"},"labels":{"previous":[{"id":206,"title":"API","color":"#ffffff","project_id":14,"created_at":"2013-12-03T17:15:43Z","updated_at":"2013-12-03T17:15:43Z","template":false,"description":"API related issues","type":"ProjectLabel","group_id":41}],"current":[{"id":205,"title":"Platform","color":"#123123","project_id":14,"created_at":"2013-12-03T17:15:43Z","updated_at":"2013-12-03T17:15:43Z","template":false,"description":"Platform related issues","type":"ProjectLabel","group_id":41}]}}}

tinybird/datasources/gitlab_job.datasource

@@ -0,0 +1,40 @@
+TOKEN "append" APPEND
+
+# Refer to https://docs.gitlab.com/ee/user/project/integrations/webhook_events.html#job-events
+
+TAGS "gitlab"
+
+SCHEMA >
+  `object_kind` String `json:$.object_kind`,
+  `ref` String `json:$.ref`,
+  `tag` Boolean `json:$.tag` DEFAULT 'false',
+  `before_sha` String `json:$.before_sha` DEFAULT '',
+  `sha` String `json:$.sha` DEFAULT '',
+  `build_id` String `json:$.build_id` DEFAULT '',
+  `build_name` String `json:$.build_name` DEFAULT '',
+  `build_stage` String `json:$.build_stage` DEFAULT '',
+  `build_status` String `json:$.build_status` DEFAULT '',
+  `build_created_at` DateTime `json:$.build_created_at` DEFAULT now(),
+  `build_started_at` DateTime `json:$.build_started_at` DEFAULT now(),
+  `build_finished_at` DateTime `json:$.build_finished_at` DEFAULT now(),
+  `build_duration` Float32 `json:$.build_duration` DEFAULT 0,
+  `build_queued_duration` Float32 `json:$.build_queued_duration` DEFAULT 0,
+  `build_allow_failure` Boolean `json:$.build_allow_failure` DEFAULT false,
+  `build_failure_reason` String `json:$.build_failure_reason` DEFAULT '',
+  `retries_count` Int32 `json:$.retries_count` DEFAULT 0,
+  `pipeline_id` UInt32 `json:$.pipeline_id` DEFAULT 0,
+  `project_id` UInt32 `json:$.project_id`,
+  `project_name` String `json:$.project_name`,
+  `user_id` UInt32 `json:$.user.id`,
+  `user` JSON `json:$.user`,
+  `commit` JSON `json:$.commit` DEFAULT '{}',
+  `repository_name` String `json:$.repository.name`,
+  `repository` JSON `json:$.repository` DEFAULT '{}',
+  `project` JSON `json:$.project` DEFAULT '{}',
+  `runner` JSON `json:$.runner` DEFAULT '{}',
+  `environment` JSON `json:$.environment` DEFAULT '{}',
+  `source_pipeline` JSON `json:$.source_pipeline` DEFAULT '{}'
+
+
+ENGINE "MergeTree"
+ENGINE_SORTING_KEY "project_id, repository_name, pipeline_id"

tinybird/datasources/gitlab_merge_request.datasource

@@ -0,0 +1,25 @@
+TOKEN "append" APPEND
+
+# Refer to https://docs.gitlab.com/ee/user/project/integrations/webhook_events.html#work-item-events
+
+TAGS "gitlab"
+
+SCHEMA >
+  `object_kind` String `json:$.object_kind`,
+  `event_type` String `json:$.event_type`,
+  `user_id` UInt32 `json:$.user.id`,
+  `user` JSON `json:$.user`,
+  `project_id` UInt32 `json:$.project.id`,
+  `project` JSON `json:$.project`,
+  `object_attributes` JSON `json:$.object_attributes`,
+  `repository_name` String `json:$.repository.name`,  
+  `repository` JSON `json:$.repository`,
+  `assignees` Array(JSON) `json:$.assignees[:]` DEFAULT '[]',
+  `assignee` String `json:$.assignee` DEFAULT '',
+  `labels` Array(JSON) `json:$.labels[:]` DEFAULT '[]',
+  `changes` JSON `json:$.changes` DEFAULT '{}',
+  `reviewers` Array(JSON) `json:$.reviewers[:]` DEFAULT '[]'
+
+
+ENGINE "MergeTree"
+ENGINE_SORTING_KEY "project_id, repository_name, user_id"

tinybird/datasources/gitlab_pipeline.datasource

@@ -0,0 +1,21 @@
+TOKEN "append" APPEND
+
+# Refer to https://docs.gitlab.com/ee/user/project/integrations/webhook_events.html#pipeline-events
+
+TAGS "gitlab"
+
+SCHEMA >
+  `object_kind` String `json:$.object_kind`,
+  `object_attributes` JSON `json:$.object_attributes`,
+  `merge_request` JSON `json:$.merge_request` DEFAULT '{}',
+  `user_id` UInt32 `json:$.user.id`,
+  `user` JSON `json:$.user`,
+  `project_id` UInt32 `json:$.project.id`,
+  `project` JSON `json:$.project`,
+  `commit` JSON `json:$.commit` DEFAULT '{}',
+  `pipeline_id` UInt32 `json:$.source_pipeline.pipeline_id`,
+  `source_pipeline` JSON `json:$.source_pipeline` DEFAULT '{}',
+  `builds` Array(JSON) `json:$.builds[:]` DEFAULT '[]'
+
+ENGINE "MergeTree"
+ENGINE_SORTING_KEY "project_id, pipeline_id, user_id"