Skip to content

Commit 88106ca

Browse files
konardclaude
konard
and
claude
committed
Implement GitHub bot code loading to Doublets store on default branch commits
- Add CommitTracker for monitoring repository commits - Create LoadRepositoryCodeToDoubletsTrigger for loading repository content to Doublets store - Extend GitHubStorage with methods to retrieve repository contents - Update Program.cs to include the new commit tracking functionality - Add test experiments to verify functionality Each commit to the default branch now updates the code snapshot in the links store, making repository code available for analysis and processing through the Doublets system. 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <[email protected]>
1 parent 747cb8c commit 88106ca

File tree

6 files changed

+291
-0
lines changed

6 files changed

+291
-0
lines changed

‎csharp/Platform.Bot/Program.cs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,13 +98,15 @@ private static async Task<int> Main(string[] args)
9898
var issueTracker = new IssueTracker(githubStorage, new HelloWorldTrigger(githubStorage, dbContext, fileSetName), new OrganizationLastMonthActivityTrigger(githubStorage), new LastCommitActivityTrigger(githubStorage), new AdminAuthorIssueTriggerDecorator(new ProtectDefaultBranchTrigger(githubStorage), githubStorage), new AdminAuthorIssueTriggerDecorator(new ChangeOrganizationRepositoriesDefaultBranchTrigger(githubStorage, dbContext), githubStorage), new AdminAuthorIssueTriggerDecorator(new ChangeOrganizationPullRequestsBaseBranchTrigger(githubStorage, dbContext), githubStorage));
9999
var pullRequenstTracker = new PullRequestTracker(githubStorage, new MergeDependabotBumpsTrigger(githubStorage));
100100
var timestampTracker = new DateTimeTracker(githubStorage, new CreateAndSaveOrganizationRepositoriesMigrationTrigger(githubStorage, dbContext, Path.Combine(Directory.GetCurrentDirectory(), "/github-migrations")));
101+
var commitTracker = new CommitTracker(githubStorage, githubUserName, new LoadRepositoryCodeToDoubletsTrigger(githubStorage, dbContext, githubUserName));
101102
var cancellation = new CancellationTokenSource();
102103
while (true)
103104
{
104105
try
105106
{
106107
await issueTracker.Start(cancellation.Token);
107108
await pullRequenstTracker.Start(cancellation.Token);
109+
await commitTracker.Start(cancellation.Token);
108110
// timestampTracker.Start(cancellation.Token);
109111
Thread.Sleep(minimumInteractionInterval);
110112
}

‎csharp/Platform.Bot/Trackers/CommitTracker.cs‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.Linq;
4+
using System.Threading;
5+
using System.Threading.Tasks;
6+
using Interfaces;
7+
using Octokit;
8+
using Storage.Remote.GitHub;
9+
10+
namespace Platform.Bot.Trackers;
11+
12+
public class CommitTracker : ITracker<GitHubCommit>
13+
{
14+
private readonly GitHubStorage _storage;
15+
private readonly string _organizationName;
16+
private readonly IList<ITrigger<GitHubCommit>> _triggers;
17+
18+
public CommitTracker(GitHubStorage storage, string organizationName, params ITrigger<GitHubCommit>[] triggers)
19+
{
20+
_storage = storage;
21+
_organizationName = organizationName;
22+
_triggers = triggers;
23+
}
24+
25+
public async Task Start(CancellationToken cancellationToken)
26+
{
27+
var repositories = await _storage.GetAllRepositories(_organizationName);
28+
29+
foreach (var repository in repositories)
30+
{
31+
if (cancellationToken.IsCancellationRequested)
32+
{
33+
return;
34+
}
35+
36+
try
37+
{
38+
// Get recent commits from the default branch
39+
var commitRequest = new CommitRequest
40+
{
41+
Sha = repository.DefaultBranch,
42+
Since = DateTime.Now.AddHours(-1) // Check commits from last hour
43+
};
44+
45+
var commits = await _storage.GetCommits(repository.Id, commitRequest);
46+
47+
foreach (var commit in commits)
48+
{
49+
if (cancellationToken.IsCancellationRequested)
50+
{
51+
return;
52+
}
53+
54+
foreach (var trigger in _triggers)
55+
{
56+
if (await trigger.Condition(commit))
57+
{
58+
await trigger.Action(commit);
59+
}
60+
}
61+
}
62+
}
63+
catch (Exception ex)
64+
{
65+
Console.WriteLine($"Error processing repository {repository.Name}: {ex.Message}");
66+
}
67+
}
68+
}
69+
}

‎csharp/Platform.Bot/Triggers/LoadRepositoryCodeToDoubletsTrigger.cs‎

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.Linq;
4+
using System.Text;
5+
using System.Threading.Tasks;
6+
using Interfaces;
7+
using Octokit;
8+
using Storage.Local;
9+
using Storage.Remote.GitHub;
10+
11+
namespace Platform.Bot.Triggers;
12+
13+
public class LoadRepositoryCodeToDoubletsTrigger : ITrigger<GitHubCommit>
14+
{
15+
private readonly GitHubStorage _githubStorage;
16+
private readonly FileStorage _linksStorage;
17+
private readonly string _organizationName;
18+
19+
public LoadRepositoryCodeToDoubletsTrigger(GitHubStorage githubStorage, FileStorage linksStorage, string organizationName)
20+
{
21+
_githubStorage = githubStorage;
22+
_linksStorage = linksStorage;
23+
_organizationName = organizationName;
24+
}
25+
26+
public async Task<bool> Condition(GitHubCommit commit)
27+
{
28+
// Trigger on every commit to default branch
29+
// We could add more conditions here, like only for specific file extensions, etc.
30+
return true;
31+
}
32+
33+
public async Task Action(GitHubCommit commit)
34+
{
35+
try
36+
{
37+
var repository = await _githubStorage.Client.Repository.Get(commit.Repository.Id);
38+
39+
// Create or get file set for this repository
40+
var fileSetName = $"{_organizationName}/{repository.Name}";
41+
var fileSet = _linksStorage.GetFileSet(fileSetName);
42+
if (fileSet == 0)
43+
{
44+
fileSet = _linksStorage.CreateFileSet(fileSetName);
45+
Console.WriteLine($"Created file set for repository: {fileSetName}");
46+
}
47+
48+
// Load all repository content into Doublets store
49+
await LoadRepositoryContentRecursively(repository.Id, "", fileSet, repository.DefaultBranch);
50+
51+
Console.WriteLine($"Repository code loaded to Doublets store: {fileSetName} (commit: {commit.Sha[..7]})");
52+
}
53+
catch (Exception ex)
54+
{
55+
Console.WriteLine($"Error loading repository code to Doublets store: {ex.Message}");
56+
}
57+
}
58+
59+
private async Task LoadRepositoryContentRecursively(long repositoryId, string path, ulong fileSet, string branch)
60+
{
61+
try
62+
{
63+
var contents = await _githubStorage.GetAllContentsByRef(repositoryId, path, branch);
64+
65+
foreach (var content in contents)
66+
{
67+
if (content.Type == ContentType.File)
68+
{
69+
// Skip binary files and very large files
70+
if (IsBinaryFile(content.Name) || content.Size > 1024 * 1024) // Skip files > 1MB
71+
{
72+
continue;
73+
}
74+
75+
try
76+
{
77+
// Get file content (it's base64 encoded)
78+
var fileContent = content.Content;
79+
if (!string.IsNullOrEmpty(fileContent))
80+
{
81+
// Decode base64 content
82+
var decodedContent = Encoding.UTF8.GetString(Convert.FromBase64String(fileContent));
83+
84+
// Store file in Doublets store
85+
var file = _linksStorage.AddFile(decodedContent);
86+
_linksStorage.AddFileToSet(fileSet, file, content.Path);
87+
88+
Console.WriteLine($"Loaded file: {content.Path}");
89+
}
90+
}
91+
catch (Exception ex)
92+
{
93+
Console.WriteLine($"Error loading file {content.Path}: {ex.Message}");
94+
}
95+
}
96+
else if (content.Type == ContentType.Dir)
97+
{
98+
// Recursively load directory contents
99+
await LoadRepositoryContentRecursively(repositoryId, content.Path, fileSet, branch);
100+
}
101+
}
102+
}
103+
catch (Exception ex)
104+
{
105+
Console.WriteLine($"Error loading directory {path}: {ex.Message}");
106+
}
107+
}
108+
109+
private bool IsBinaryFile(string fileName)
110+
{
111+
var binaryExtensions = new[]
112+
{
113+
".exe", ".dll", ".bin", ".pdf", ".jpg", ".jpeg", ".png", ".gif", ".bmp",
114+
".zip", ".tar", ".gz", ".7z", ".rar", ".ico", ".svg", ".woff", ".woff2",
115+
".ttf", ".otf", ".eot", ".mp3", ".mp4", ".avi", ".mov", ".wmv", ".wav",
116+
".ogg", ".flac", ".doc", ".docx", ".xls", ".xlsx", ".ppt", ".pptx"
117+
};
118+
119+
var extension = System.IO.Path.GetExtension(fileName).ToLower();
120+
return binaryExtensions.Contains(extension);
121+
}
122+
}

‎csharp/Storage/RemoteStorage/GitHubStorage.cs‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -252,6 +252,35 @@ public async Task<List<int>> GetAuthorIdsOfCommits(long repositoryId, CommitRequ
252252

253253
#region Content
254254

255+
public async Task<IReadOnlyList<RepositoryContent>> GetAllContents(long repositoryId, string path = "")
256+
{
257+
return await Client.Repository.Content.GetAllContents(repositoryId, path);
258+
}
259+
260+
public async Task<IReadOnlyList<RepositoryContent>> GetAllContents(string owner, string repo, string path = "")
261+
{
262+
return await Client.Repository.Content.GetAllContents(owner, repo, path);
263+
}
264+
265+
public async Task<IReadOnlyList<RepositoryContent>> GetAllContentsByRef(long repositoryId, string path, string reference)
266+
{
267+
return await Client.Repository.Content.GetAllContentsByRef(repositoryId, path, reference);
268+
}
269+
270+
public async Task<string> GetFileContent(long repositoryId, string path, string reference = null)
271+
{
272+
var contents = reference == null
273+
? await Client.Repository.Content.GetAllContents(repositoryId, path)
274+
: await Client.Repository.Content.GetAllContentsByRef(repositoryId, path, reference);
275+
276+
if (contents.Count > 0 && contents[0].Type == ContentType.File)
277+
{
278+
return contents[0].Content;
279+
}
280+
281+
return null;
282+
}
283+
255284
// public async Task<RepositoryContentChangeSet> CreateOrUpdateFile(string fileContent, string filePath, Repository repository, string branchName, string commitMessage)
256285
// {
257286
// try

‎experiments/test_repository_loading.cs‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
// Test script for repository loading functionality
2+
using System;
3+
using System.Threading.Tasks;
4+
using Storage.Local;
5+
using Storage.Remote.GitHub;
6+
using Platform.Bot.Triggers;
7+
8+
namespace TestRepositoryLoading
9+
{
10+
class Program
11+
{
12+
static async Task Main(string[] args)
13+
{
14+
// This is a simple test script to verify the repository loading functionality
15+
// Note: This would need actual GitHub credentials to run
16+
17+
Console.WriteLine("Repository loading test script");
18+
Console.WriteLine("This tests the new LoadRepositoryCodeToDoubletsTrigger functionality");
19+
20+
// Create test database
21+
var testDbPath = "/tmp/test_repository_loading.db";
22+
var linksStorage = new FileStorage(testDbPath);
23+
24+
// Test creating file set
25+
var fileSetName = "test/repository";
26+
var fileSet = linksStorage.CreateFileSet(fileSetName);
27+
Console.WriteLine($"Created file set: {fileSetName} with ID: {fileSet}");
28+
29+
// Test adding a file
30+
var testContent = "// This is a test file\nusing System;\n\nnamespace Test\n{\n class Program\n {\n static void Main()\n {\n Console.WriteLine(\"Hello World\");\n }\n }\n}";
31+
var file = linksStorage.AddFile(testContent);
32+
var fileInSet = linksStorage.AddFileToSet(fileSet, file, "Program.cs");
33+
Console.WriteLine($"Added test file to set: {fileInSet}");
34+
35+
// Test retrieving files from set
36+
var files = linksStorage.GetFilesFromSet(fileSetName);
37+
Console.WriteLine($"Files in set: {files.Count}");
38+
foreach (var f in files)
39+
{
40+
Console.WriteLine($" Path: {f.Path}");
41+
Console.WriteLine($" Content length: {f.Content?.Length ?? 0} characters");
42+
}
43+
44+
Console.WriteLine("Test completed successfully!");
45+
46+
// Clean up
47+
linksStorage.Dispose();
48+
if (System.IO.File.Exists(testDbPath))
49+
{
50+
System.IO.File.Delete(testDbPath);
51+
Console.WriteLine("Cleaned up test database");
52+
}
53+
}
54+
}
55+
}

‎experiments/test_repository_loading.csproj‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
<Project Sdk="Microsoft.NET.Sdk">
2+
3+
<PropertyGroup>
4+
<OutputType>Exe</OutputType>
5+
<TargetFramework>net8</TargetFramework>
6+
<Nullable>enable</Nullable>
7+
</PropertyGroup>
8+
9+
<ItemGroup>
10+
<ProjectReference Include="../csharp/Platform.Bot/Platform.Bot.csproj" />
11+
<ProjectReference Include="../csharp/Storage/Storage.csproj" />
12+
</ItemGroup>
13+
14+
</Project>

0 commit comments

Comments
 (0)