Skip to content

Commit bb91200

Browse files
felixwidjajafelixwidjaja
authored
Merge pull request #41 from datum-cloud/fwidjaja-patch-3
Separating job to prevent hitting limit
2 parents ee6a782 + 6b7e531 commit bb91200

File tree

4 files changed

+175
-32
lines changed

4 files changed

+175
-32
lines changed

‎.github/workflows/monitor-friday.yml‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
name: Monitor Alt Clouds - Friday (Batch 3)
2+
3+
on:
4+
schedule:
5+
# Every Friday at 10 AM UTC
6+
- cron: '0 10 * * 5'
7+
workflow_dispatch: # Manual trigger for testing
8+
9+
jobs:
10+
monitor-and-evaluate:
11+
runs-on: ubuntu-latest
12+
permissions:
13+
contents: write
14+
issues: write
15+
16+
steps:
17+
- name: Checkout repository
18+
uses: actions/checkout@v4
19+
20+
- name: Set up Python
21+
uses: actions/setup-python@v5
22+
with:
23+
python-version: '3.11'
24+
25+
- name: Install dependencies
26+
run: |
27+
pip install anthropic requests beautifulsoup4 lxml
28+
29+
- name: Run news monitoring (Batch 3)
30+
env:
31+
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
32+
CATEGORY_BATCH: "3"
33+
run: |
34+
python scripts/monitor_news.py
35+
36+
- name: Evaluate candidates
37+
env:
38+
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
39+
run: |
40+
python scripts/evaluate_candidates.py
41+
42+
- name: Create GitHub issues for candidates
43+
env:
44+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
45+
run: |
46+
python scripts/create_issues.py
47+
48+
- name: Commit scan results
49+
run: |
50+
git config user.name "github-actions[bot]"
51+
git config user.email "github-actions[bot]@users.noreply.github.com"
52+
git add data/
53+
git diff --staged --quiet || git commit -m "Friday scan (batch 3): $(date +%Y-%m-%d)"
54+
git push
55+
56+
- name: Send Slack notification
57+
env:
58+
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
59+
run: |
60+
python scripts/slack_notifier.py

.github/workflows/weekly-monitor.yml renamed to .github/workflows/monitor-monday.yml

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
name: Weekly Alt Clouds Monitor
1+
name: Monitor Alt Clouds - Monday (Batch 1)
22

33
on:
44
schedule:
@@ -26,9 +26,10 @@ jobs:
2626
run: |
2727
pip install anthropic requests beautifulsoup4 lxml
2828
29-
- name: Run news monitoring
29+
- name: Run news monitoring (Batch 1)
3030
env:
3131
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
32+
CATEGORY_BATCH: "1"
3233
run: |
3334
python scripts/monitor_news.py
3435
@@ -49,7 +50,7 @@ jobs:
4950
git config user.name "github-actions[bot]"
5051
git config user.email "github-actions[bot]@users.noreply.github.com"
5152
git add data/
52-
git diff --staged --quiet || git commit -m "Weekly scan: $(date +%Y-%m-%d)"
53+
git diff --staged --quiet || git commit -m "Monday scan (batch 1): $(date +%Y-%m-%d)"
5354
git push
5455
5556
- name: Send Slack notification

‎.github/workflows/monitor-wednesday.yml‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
name: Monitor Alt Clouds - Wednesday (Batch 2)
2+
3+
on:
4+
schedule:
5+
# Every Wednesday at 10 AM UTC
6+
- cron: '0 10 * * 3'
7+
workflow_dispatch: # Manual trigger for testing
8+
9+
jobs:
10+
monitor-and-evaluate:
11+
runs-on: ubuntu-latest
12+
permissions:
13+
contents: write
14+
issues: write
15+
16+
steps:
17+
- name: Checkout repository
18+
uses: actions/checkout@v4
19+
20+
- name: Set up Python
21+
uses: actions/setup-python@v5
22+
with:
23+
python-version: '3.11'
24+
25+
- name: Install dependencies
26+
run: |
27+
pip install anthropic requests beautifulsoup4 lxml
28+
29+
- name: Run news monitoring (Batch 2)
30+
env:
31+
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
32+
CATEGORY_BATCH: "2"
33+
run: |
34+
python scripts/monitor_news.py
35+
36+
- name: Evaluate candidates
37+
env:
38+
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
39+
run: |
40+
python scripts/evaluate_candidates.py
41+
42+
- name: Create GitHub issues for candidates
43+
env:
44+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
45+
run: |
46+
python scripts/create_issues.py
47+
48+
- name: Commit scan results
49+
run: |
50+
git config user.name "github-actions[bot]"
51+
git config user.email "github-actions[bot]@users.noreply.github.com"
52+
git add data/
53+
git diff --staged --quiet || git commit -m "Wednesday scan (batch 2): $(date +%Y-%m-%d)"
54+
git push
55+
56+
- name: Send Slack notification
57+
env:
58+
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
59+
run: |
60+
python scripts/slack_notifier.py

‎scripts/monitor_news.py‎

Lines changed: 51 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -13,69 +13,90 @@
1313

1414
# Configuration
1515
ANTHROPIC_API_KEY = os.environ.get("ANTHROPIC_API_KEY")
16-
CATEGORIES = {
16+
CATEGORY_BATCH = os.environ.get("CATEGORY_BATCH", "1") # 1, 2, or 3
17+
18+
# All categories split into 3 batches
19+
ALL_CATEGORIES = {
1720
"Infrastructure Clouds": {
1821
"keywords": ["GPU cloud", "cloud compute", "inference API", "serverless GPU"],
19-
"description": "GPU and compute infrastructure providers"
22+
"description": "GPU and compute infrastructure providers",
23+
"batch": "1"
2024
},
21-
"Sovereign Clouds": {
22-
"keywords": ["sovereign cloud", "data residency", "regional compliance cloud"],
23-
"description": "Cloud platforms with data sovereignty"
25+
"Developer Happiness Clouds": {
26+
"keywords": ["PaaS", "deployment platform", "CI/CD", "developer tools cloud"],
27+
"description": "Developer platforms and tools",
28+
"batch": "1"
2429
},
2530
"Data Clouds": {
2631
"keywords": ["database cloud", "data warehouse", "analytics platform", "observability"],
27-
"description": "Data storage, processing, and observability"
28-
},
29-
"Workflow and Operations Clouds": {
30-
"keywords": ["workflow automation", "orchestration platform", "incident management"],
31-
"description": "Workflow automation and operations"
32+
"description": "Data storage, processing, and observability",
33+
"batch": "1"
3234
},
3335
"Network, Connectivity and Security Clouds": {
3436
"keywords": ["CDN", "edge network", "zero trust", "API gateway", "VPN alternative"],
35-
"description": "Networking, security, and connectivity"
36-
},
37-
"Vibe Clouds": {
38-
"keywords": ["AI coding assistant", "LLM API", "AI development platform"],
39-
"description": "AI-powered creativity and development tools"
37+
"description": "Networking, security, and connectivity",
38+
"batch": "1"
4039
},
41-
"Developer Happiness Clouds": {
42-
"keywords": ["PaaS", "deployment platform", "CI/CD", "developer tools cloud"],
43-
"description": "Developer platforms and tools"
40+
"Workflow and Operations Clouds": {
41+
"keywords": ["workflow automation", "orchestration platform", "incident management"],
42+
"description": "Workflow automation and operations",
43+
"batch": "1"
4444
},
4545
"Authorization, Identity, Fraud and Abuse Clouds": {
4646
"keywords": ["authentication service", "identity platform", "fraud detection API"],
47-
"description": "Auth, identity, and fraud prevention"
47+
"description": "Auth, identity, and fraud prevention",
48+
"batch": "2"
4849
},
4950
"Monetization, Finance and Legal Clouds": {
5051
"keywords": ["billing platform", "usage metering", "subscription management API"],
51-
"description": "Billing, payments, and monetization"
52+
"description": "Billing, payments, and monetization",
53+
"batch": "2"
5254
},
53-
"Customer, Marketing and eCommerce Clouds": {
54-
"keywords": ["CRM platform", "marketing automation", "ecommerce platform"],
55-
"description": "Customer engagement and commerce"
55+
"Vibe Clouds": {
56+
"keywords": ["AI coding assistant", "LLM API", "AI development platform"],
57+
"description": "AI-powered creativity and development tools",
58+
"batch": "2"
5659
},
5760
"IoT, Communications, and Media Clouds": {
5861
"keywords": ["IoT platform", "messaging API", "video platform", "SMS gateway"],
59-
"description": "IoT, communications, and media"
62+
"description": "IoT, communications, and media",
63+
"batch": "2"
64+
},
65+
"Sovereign Clouds": {
66+
"keywords": ["sovereign cloud", "data residency", "regional compliance cloud"],
67+
"description": "Cloud platforms with data sovereignty",
68+
"batch": "2"
69+
},
70+
"Customer, Marketing and eCommerce Clouds": {
71+
"keywords": ["CRM platform", "marketing automation", "ecommerce platform"],
72+
"description": "Customer engagement and commerce",
73+
"batch": "3"
6074
},
6175
"Blockchain Clouds": {
6276
"keywords": ["blockchain infrastructure", "web3 cloud", "decentralized compute"],
63-
"description": "Blockchain-based infrastructure"
77+
"description": "Blockchain-based infrastructure",
78+
"batch": "3"
6479
},
6580
"Unikernels & WebAssembly": {
6681
"keywords": ["unikernel platform", "WebAssembly cloud", "wasm runtime"],
67-
"description": "Unikernel and WebAssembly platforms"
82+
"description": "Unikernel and WebAssembly platforms",
83+
"batch": "3"
6884
},
6985
"Source Code Control": {
7086
"keywords": ["git hosting", "version control platform", "code repository"],
71-
"description": "Source code management"
87+
"description": "Source code management",
88+
"batch": "3"
7289
},
7390
"Cloud Adjacent": {
7491
"keywords": ["cloud tools", "cloud utilities", "infrastructure software"],
75-
"description": "Cloud-complementary tools and services"
92+
"description": "Cloud-complementary tools and services",
93+
"batch": "3"
7694
}
7795
}
7896

97+
# Filter categories based on batch
98+
CATEGORIES = {k: v for k, v in ALL_CATEGORIES.items() if v["batch"] == CATEGORY_BATCH}
99+
79100

80101
class AltCloudsMonitor:
81102
def __init__(self):
@@ -182,6 +203,7 @@ def run_daily_monitor(self) -> Dict[str, Any]:
182203
"""
183204
print("🚀 Starting daily alt-clouds monitoring...")
184205
print(f"📅 {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
206+
print(f"📦 Batch: {CATEGORY_BATCH}")
185207
print(f"🔍 Monitoring {len(CATEGORIES)} categories")
186208
print("-" * 60)
187209

@@ -217,7 +239,7 @@ def run_daily_monitor(self) -> Dict[str, Any]:
217239
output["by_category"][cat].append(candidate)
218240

219241
# Save to file
220-
output_file = f"data/candidates/scan-{datetime.now().strftime('%Y%m%d')}.json"
242+
output_file = f"data/candidates/scan-{datetime.now().strftime('%Y%m%d')}-batch{CATEGORY_BATCH}.json"
221243
os.makedirs(os.path.dirname(output_file), exist_ok=True)
222244

223245
with open(output_file, 'w') as f:

0 commit comments

Comments
 (0)