Skip to content

Commit a15e630

Browse files
davetroianodavetroiano
committed
GitHub datagen
1 parent a4e775c commit a15e630

File tree

12 files changed

+5166
-15
lines changed

12 files changed

+5166
-15
lines changed

‎config/connector_github_assignees.config‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
{
2+
"name": "datagen-github-assignees",
3+
"config": {
4+
"connector.class": "io.confluent.kafka.connect.datagen.DatagenConnector",
5+
"kafka.topic": "github_assignees",
6+
"quickstart": "github_assignees",
7+
"key.converter": "org.apache.kafka.connect.storage.StringConverter",
8+
"value.converter": "org.apache.kafka.connect.json.JsonConverter",
9+
"value.converter.schemas.enable": "false",
10+
"max.interval": 100,
11+
"iterations": 100,
12+
"tasks.max": "1"
13+
}
14+
}

‎config/connector_github_collaborators.config‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
{
2+
"name": "datagen-github-collaborators",
3+
"config": {
4+
"connector.class": "io.confluent.kafka.connect.datagen.DatagenConnector",
5+
"kafka.topic": "github_collaborators",
6+
"quickstart": "github_collaborators",
7+
"key.converter": "org.apache.kafka.connect.storage.StringConverter",
8+
"value.converter": "org.apache.kafka.connect.json.JsonConverter",
9+
"value.converter.schemas.enable": "false",
10+
"max.interval": 100,
11+
"iterations": 100,
12+
"tasks.max": "1"
13+
}
14+
}

‎config/connector_github_commits.config‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
{
2+
"name": "datagen-github-commits",
3+
"config": {
4+
"connector.class": "io.confluent.kafka.connect.datagen.DatagenConnector",
5+
"kafka.topic": "github_commits",
6+
"quickstart": "github_commits",
7+
"key.converter": "org.apache.kafka.connect.storage.StringConverter",
8+
"value.converter": "org.apache.kafka.connect.json.JsonConverter",
9+
"value.converter.schemas.enable": "false",
10+
"max.interval": 100,
11+
"iterations": 100,
12+
"tasks.max": "1"
13+
}
14+
}

‎config/connector_github_pull_requests.config‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
{
2+
"name": "datagen-github-pull-requests",
3+
"config": {
4+
"connector.class": "io.confluent.kafka.connect.datagen.DatagenConnector",
5+
"kafka.topic": "github_pull_requests",
6+
"quickstart": "github_pull_requests",
7+
"key.converter": "org.apache.kafka.connect.storage.StringConverter",
8+
"value.converter": "org.apache.kafka.connect.json.JsonConverter",
9+
"value.converter.schemas.enable": "false",
10+
"max.interval": 100,
11+
"iterations": 100,
12+
"tasks.max": "1"
13+
}
14+
}

‎config/connector_github_stargazers.config‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
{
2+
"name": "datagen-github-stargazers",
3+
"config": {
4+
"connector.class": "io.confluent.kafka.connect.datagen.DatagenConnector",
5+
"kafka.topic": "github_stargazers",
6+
"quickstart": "github_stargazers",
7+
"key.converter": "org.apache.kafka.connect.storage.StringConverter",
8+
"value.converter": "org.apache.kafka.connect.json.JsonConverter",
9+
"value.converter.schemas.enable": "false",
10+
"max.interval": 100,
11+
"iterations": 100,
12+
"tasks.max": "1"
13+
}
14+
}

‎src/main/java/io/confluent/kafka/connect/datagen/DatagenTask.java‎

Lines changed: 20 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -68,21 +68,26 @@ public class DatagenTask extends SourceTask {
6868
private Random random;
6969

7070
protected enum Quickstart {
71-
CLICKSTREAM_CODES("clickstream_codes_schema.avro", "code"),
72-
CLICKSTREAM("clickstream_schema.avro", "ip"),
73-
CLICKSTREAM_USERS("clickstream_users_schema.avro", "user_id"),
74-
ORDERS("orders_schema.avro", "orderid"),
75-
RATINGS("ratings_schema.avro", "rating_id"),
76-
USERS("users_schema.avro", "userid"),
77-
USERS_("users_array_map_schema.avro", "userid"),
78-
PAGEVIEWS("pageviews_schema.avro", "viewtime"),
79-
STOCK_TRADES("stock_trades_schema.avro", "symbol"),
80-
INVENTORY("inventory.avro", "id"),
81-
PRODUCT("product.avro", "id"),
82-
PURCHASES("purchase.avro", "id"),
83-
TRANSACTIONS("transactions.avro", "transaction_id"),
84-
STORES("stores.avro", "store_id"),
85-
CREDIT_CARDS("credit_cards.avro", "card_id");
71+
CLICKSTREAM_CODES("clickstream_codes_schema.avsc", "code"),
72+
CLICKSTREAM("clickstream_schema.avsc", "ip"),
73+
CLICKSTREAM_USERS("clickstream_users_schema.avsc", "user_id"),
74+
ORDERS("orders_schema.avsc", "orderid"),
75+
RATINGS("ratings_schema.avsc", "rating_id"),
76+
USERS("users_schema.avsc", "userid"),
77+
USERS_("users_array_map_schema.avsc", "userid"),
78+
PAGEVIEWS("pageviews_schema.avsc", "viewtime"),
79+
STOCK_TRADES("stock_trades_schema.avsc", "symbol"),
80+
INVENTORY("inventory.avsc", "id"),
81+
PRODUCT("product.avsc", "id"),
82+
PURCHASES("purchase.avsc", "id"),
83+
TRANSACTIONS("transactions.avsc", "transaction_id"),
84+
STORES("stores.avsc", "store_id"),
85+
CREDIT_CARDS("credit_cards.avsc", "card_id"),
86+
GITHUB_ASSIGNEES("github/assignees.avsc", "id"),
87+
GITHUB_COLLABORATORS("github/collaborators.avsc", "id"),
88+
GITHUB_COMMITS("github/commits.avsc", "id"),
89+
GITHUB_PULL_REQUESTS("github/pull_requests.avsc", "id"),
90+
GITHUB_STARGAZERS("github/stargazers.avsc", "id");
8691

8792
static final Set<String> configValues = new HashSet<>();
8893

‎src/main/resources/github/assignees.avsc‎

Lines changed: 215 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,215 @@
1+
{
2+
"namespace": "datagen.github",
3+
"name": "github_assignees",
4+
"type": "record",
5+
"fields": [
6+
{
7+
"name": "type",
8+
"type": {
9+
"type": "string",
10+
"arg.properties": {
11+
"options": [
12+
"ASSIGNEES"
13+
]
14+
}
15+
}
16+
},
17+
{
18+
"name": "createdAt",
19+
"type": {
20+
"type": "long",
21+
"arg.properties": {
22+
"iteration": {
23+
"start": 1657053439000,
24+
"step": 30000
25+
}
26+
}
27+
}
28+
},
29+
{
30+
"name": "data",
31+
"type": {
32+
"type": "record",
33+
"name": "data_record",
34+
"fields": [
35+
{
36+
"name": "login",
37+
"type":{
38+
"type":"string",
39+
"arg.properties":{
40+
"regex":"user[1-9]{2}"
41+
}
42+
}
43+
},
44+
{
45+
"name": "id",
46+
"type": {
47+
"type": "long",
48+
"arg.properties": {
49+
"iteration": {
50+
"start": 829002,
51+
"step": 20
52+
}
53+
}
54+
}
55+
},
56+
{
57+
"name": "node_id",
58+
"type":{
59+
"type":"string",
60+
"arg.properties":{
61+
"regex":"[a-zA-Z0-9]{40}="
62+
}
63+
}
64+
},
65+
{
66+
"name": "avatar_url",
67+
"type":{
68+
"type":"string",
69+
"arg.properties":{
70+
"regex":"https://avatars\\.githubusercontent\\.com/u/[a-z]{8}"
71+
}
72+
}
73+
},
74+
{
75+
"name": "gravatar_id",
76+
"type":{
77+
"type":"string",
78+
"arg.properties":{
79+
"regex":"[a-z]{8}"
80+
}
81+
}
82+
},
83+
{
84+
"name": "url",
85+
"type":{
86+
"type":"string",
87+
"arg.properties":{
88+
"regex":"https://api\\.github\\.com/users/org[1-9]{2}"
89+
}
90+
}
91+
},
92+
{
93+
"name": "html_url",
94+
"type":{
95+
"type":"string",
96+
"arg.properties":{
97+
"regex":"https://github\\.com/org[1-9]{2}"
98+
}
99+
}
100+
},
101+
{
102+
"name": "followers_url",
103+
"type":{
104+
"type":"string",
105+
"arg.properties":{
106+
"regex":"https://api\\.github\\.com/users/org[1-9]{2}/followers"
107+
}
108+
}
109+
},
110+
{
111+
"name": "following_url",
112+
"type":{
113+
"type":"string",
114+
"arg.properties":{
115+
"regex":"https://api\\.github\\.com/users/org[1-9]{2}/following\\{/other_user\\}"
116+
}
117+
}
118+
},
119+
{
120+
"name": "gists_url",
121+
"type":{
122+
"type":"string",
123+
"arg.properties":{
124+
"regex":"https://api\\.github\\.com/users/org[1-9]{2}/gists\\{/gist_id\\}"
125+
}
126+
}
127+
},
128+
{
129+
"name": "starred_url",
130+
"type":{
131+
"type":"string",
132+
"arg.properties":{
133+
"regex":"https://api\\.github\\.com/users/org[1-9]{2}/starred\\{/owner\\}\\{/repo\\}"
134+
}
135+
}
136+
},
137+
{
138+
"name": "subscriptions_url",
139+
"type":{
140+
"type":"string",
141+
"arg.properties":{
142+
"regex":"https://api\\.github\\.com/users/org[1-9]{2}/subscriptions"
143+
}
144+
}
145+
},
146+
{
147+
"name": "organizations_url",
148+
"type":{
149+
"type":"string",
150+
"arg.properties":{
151+
"regex":"https://api\\.github\\.com/users/org[1-9]{2}/orgs"
152+
}
153+
}
154+
},
155+
{
156+
"name": "repos_url",
157+
"type":{
158+
"type":"string",
159+
"arg.properties":{
160+
"regex":"https://api\\.github\\.com/users/org[1-9]{2}/repos"
161+
}
162+
}
163+
},
164+
{
165+
"name": "events_url",
166+
"type":{
167+
"type":"string",
168+
"arg.properties":{
169+
"regex":"https://api\\.github\\.com/users/org[1-9]{2}/events\\{/privacy\\}"
170+
}
171+
}
172+
},
173+
{
174+
"name": "received_events_url",
175+
"type":{
176+
"type":"string",
177+
"arg.properties":{
178+
"regex":"https://api\\.github\\.com/users/org[1-9]{2}/received_events"
179+
}
180+
}
181+
},
182+
{
183+
"name": "type",
184+
"type":{
185+
"type":"string",
186+
"arg.properties":{
187+
"regex":"User"
188+
}
189+
}
190+
},
191+
{
192+
"name": "site_admin",
193+
"type":{
194+
"type":"boolean",
195+
"arg.properties": {
196+
"options": [
197+
false
198+
]
199+
}
200+
}
201+
}
202+
]
203+
}
204+
},
205+
{
206+
"name": "id",
207+
"type":{
208+
"type":"string",
209+
"arg.properties":{
210+
"regex":"[0-9]{20}"
211+
}
212+
}
213+
}
214+
]
215+
}

0 commit comments

Comments
 (0)