Skip to content

Commit 08b1b58

Browse files
konardclaude
konard
and
claude
committed
Implement repository metadata extraction and statistics functionality for issue #91
Add comprehensive repository metadata extraction system that allows the GitHub bot to extract complete data and metadata about repositories to reduce API requests and enable backup/statistics capabilities. Features implemented: - RepositoryMetadataExtractorTrigger: Automated extraction of repository metadata - RepositoryStatisticsAnalyzer: Statistical analysis and reporting capabilities - CLI commands: metadata extract/stats/report for manual operations - Comprehensive data models for repository, commits, PRs, and issues - JSON export with timestamped backups and current snapshots - Automatic scheduling integration with existing bot architecture - Human-readable report generation in Markdown format The system extracts: - Basic repository information (name, description, language, etc.) - Activity metrics (stars, forks, watchers, open issues/PRs) - Recent commits (last 30 days) with author information - Pull request and issue details with labels and metadata - Organization-wide statistics and trends analysis Benefits: - Reduced GitHub API requests through bulk extraction and caching - Backup capabilities for disaster recovery and historical analysis - Comprehensive statistics for organization insights - Offline data analysis capabilities 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <[email protected]>
1 parent c4f1cb0 commit 08b1b58

File tree

6 files changed

+1190
-1
lines changed

6 files changed

+1190
-1
lines changed

‎csharp/Platform.Bot/Commands/RepositoryMetadataCommand.cs‎

Lines changed: 185 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,185 @@
1+
using System;
2+
using System.CommandLine;
3+
using System.IO;
4+
using System.Threading.Tasks;
5+
using Platform.Bot.Triggers;
6+
using Storage.Local;
7+
using Storage.Remote.GitHub;
8+
using File = System.IO.File;
9+
10+
namespace Platform.Bot.Commands;
11+
12+
/// <summary>
13+
/// Command line interface for repository metadata extraction and statistics.
14+
/// </summary>
15+
public static class RepositoryMetadataCommand
16+
{
17+
public static Command CreateCommand()
18+
{
19+
var extractCommand = new Command("extract", "Extract repository metadata for the organization")
20+
{
21+
CreateGitHubUserNameOption(),
22+
CreateGitHubApiTokenOption(),
23+
CreateGitHubApplicationNameOption(),
24+
CreateOrganizationNameOption(),
25+
CreateOutputDirectoryOption()
26+
};
27+
28+
extractCommand.SetHandler(HandleExtractCommand,
29+
extractCommand.Options[0] as Option<string>,
30+
extractCommand.Options[1] as Option<string>,
31+
extractCommand.Options[2] as Option<string>,
32+
extractCommand.Options[3] as Option<string>,
33+
extractCommand.Options[4] as Option<string>);
34+
35+
var statsCommand = new Command("stats", "Generate statistics from extracted metadata")
36+
{
37+
CreateOutputDirectoryOption()
38+
};
39+
40+
statsCommand.SetHandler(HandleStatsCommand,
41+
statsCommand.Options[0] as Option<string>);
42+
43+
var reportCommand = new Command("report", "Generate a text report from statistics")
44+
{
45+
CreateOutputDirectoryOption()
46+
};
47+
48+
reportCommand.SetHandler(HandleReportCommand,
49+
reportCommand.Options[0] as Option<string>);
50+
51+
var metadataCommand = new Command("metadata", "Repository metadata extraction and analysis commands")
52+
{
53+
extractCommand,
54+
statsCommand,
55+
reportCommand
56+
};
57+
58+
return metadataCommand;
59+
}
60+
61+
private static Option<string> CreateGitHubUserNameOption()
62+
{
63+
return new Option<string>(
64+
name: "--github-user-name",
65+
description: "GitHub user name") { IsRequired = true };
66+
}
67+
68+
private static Option<string> CreateGitHubApiTokenOption()
69+
{
70+
return new Option<string>(
71+
name: "--github-api-token",
72+
description: "GitHub API token") { IsRequired = true };
73+
}
74+
75+
private static Option<string> CreateGitHubApplicationNameOption()
76+
{
77+
return new Option<string>(
78+
name: "--github-application-name",
79+
description: "GitHub application name") { IsRequired = true };
80+
}
81+
82+
private static Option<string> CreateOrganizationNameOption()
83+
{
84+
return new Option<string>(
85+
name: "--organization",
86+
description: "GitHub organization name",
87+
getDefaultValue: () => "linksplatform");
88+
}
89+
90+
private static Option<string> CreateOutputDirectoryOption()
91+
{
92+
return new Option<string>(
93+
name: "--output-dir",
94+
description: "Output directory for metadata and reports",
95+
getDefaultValue: () => "repository-metadata");
96+
}
97+
98+
private static async Task HandleExtractCommand(
99+
string githubUserName,
100+
string githubApiToken,
101+
string githubApplicationName,
102+
string organizationName,
103+
string outputDirectory)
104+
{
105+
try
106+
{
107+
Console.WriteLine($"Starting repository metadata extraction for organization: {organizationName}");
108+
Console.WriteLine($"Output directory: {outputDirectory}");
109+
110+
var githubStorage = new GitHubStorage(githubUserName, githubApiToken, githubApplicationName);
111+
var fileStorage = new FileStorage("temp-db.db"); // Temporary storage, not used in this context
112+
113+
var extractor = new RepositoryMetadataExtractorTrigger(
114+
githubStorage,
115+
fileStorage,
116+
organizationName,
117+
outputDirectory,
118+
TimeSpan.Zero); // Force immediate extraction
119+
120+
// Force extraction by passing null (condition will return true)
121+
if (await extractor.Condition(null))
122+
{
123+
await extractor.Action(null);
124+
Console.WriteLine("Repository metadata extraction completed successfully!");
125+
}
126+
else
127+
{
128+
Console.WriteLine("Extraction not needed at this time.");
129+
}
130+
}
131+
catch (Exception ex)
132+
{
133+
Console.WriteLine($"Error during extraction: {ex.Message}");
134+
Console.WriteLine($"Stack trace: {ex.StackTrace}");
135+
}
136+
}
137+
138+
private static async Task HandleStatsCommand(string outputDirectory)
139+
{
140+
try
141+
{
142+
Console.WriteLine($"Generating statistics from metadata in: {outputDirectory}");
143+
144+
var analyzer = new RepositoryStatisticsAnalyzer(outputDirectory);
145+
var stats = await analyzer.GenerateStatistics();
146+
await analyzer.SaveStatisticsReport(stats);
147+
148+
Console.WriteLine("Statistics generation completed successfully!");
149+
Console.WriteLine($"Total repositories analyzed: {stats.TotalRepositories}");
150+
Console.WriteLine($"Public repositories: {stats.PublicRepositories}");
151+
Console.WriteLine($"Private repositories: {stats.PrivateRepositories}");
152+
Console.WriteLine($"Total stars: {stats.TotalStars:N0}");
153+
Console.WriteLine($"Total forks: {stats.TotalForks:N0}");
154+
}
155+
catch (Exception ex)
156+
{
157+
Console.WriteLine($"Error during statistics generation: {ex.Message}");
158+
}
159+
}
160+
161+
private static async Task HandleReportCommand(string outputDirectory)
162+
{
163+
try
164+
{
165+
Console.WriteLine($"Generating text report from statistics in: {outputDirectory}");
166+
167+
var analyzer = new RepositoryStatisticsAnalyzer(outputDirectory);
168+
var stats = await analyzer.GenerateStatistics();
169+
var textReport = analyzer.GenerateTextSummary(stats);
170+
171+
var reportPath = Path.Combine(outputDirectory, $"report-{DateTime.UtcNow:yyyyMMdd-HHmmss}.md");
172+
await File.WriteAllTextAsync(reportPath, textReport);
173+
174+
Console.WriteLine("Text report generation completed successfully!");
175+
Console.WriteLine($"Report saved to: {reportPath}");
176+
Console.WriteLine();
177+
Console.WriteLine("=== REPORT PREVIEW ===");
178+
Console.WriteLine(textReport);
179+
}
180+
catch (Exception ex)
181+
{
182+
Console.WriteLine($"Error during report generation: {ex.Message}");
183+
}
184+
}
185+
}

‎csharp/Platform.Bot/Program.cs‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
using Platform.Bot.Trackers;
1616
using Platform.Bot.Triggers;
1717
using Platform.Bot.Triggers.Decorators;
18+
using Platform.Bot.Commands;
1819

1920
namespace Platform.Bot
2021
{
@@ -73,7 +74,7 @@ private static async Task<int> Main(string[] args)
7374
description: "Minimum interaction interval in seconds.",
7475
getDefaultValue: () => 60);
7576

76-
var rootCommand = new RootCommand("Sample app for System.CommandLine")
77+
var rootCommand = new RootCommand("Platform Bot - GitHub automation and metadata extraction")
7778
{
7879
githubUserNameOption,
7980
githubApiTokenOption,
@@ -83,6 +84,9 @@ private static async Task<int> Main(string[] args)
8384
minimumInteractionIntervalOption
8485
};
8586

87+
// Add repository metadata commands
88+
rootCommand.AddCommand(RepositoryMetadataCommand.CreateCommand());
89+
8690
rootCommand.SetHandler(async (githubUserName, githubApiToken, githubApplicationName, databaseFilePath, fileSetName, minimumInteractionInterval) =>
8791
{
8892
Debug.WriteLine($"Nickname: {githubUserName}");
@@ -98,6 +102,9 @@ private static async Task<int> Main(string[] args)
98102
var issueTracker = new IssueTracker(githubStorage, new HelloWorldTrigger(githubStorage, dbContext, fileSetName), new OrganizationLastMonthActivityTrigger(githubStorage), new LastCommitActivityTrigger(githubStorage), new AdminAuthorIssueTriggerDecorator(new ProtectDefaultBranchTrigger(githubStorage), githubStorage), new AdminAuthorIssueTriggerDecorator(new ChangeOrganizationRepositoriesDefaultBranchTrigger(githubStorage, dbContext), githubStorage), new AdminAuthorIssueTriggerDecorator(new ChangeOrganizationPullRequestsBaseBranchTrigger(githubStorage, dbContext), githubStorage));
99103
var pullRequenstTracker = new PullRequestTracker(githubStorage, new MergeDependabotBumpsTrigger(githubStorage));
100104
var timestampTracker = new DateTimeTracker(githubStorage, new CreateAndSaveOrganizationRepositoriesMigrationTrigger(githubStorage, dbContext, Path.Combine(Directory.GetCurrentDirectory(), "/github-migrations")));
105+
106+
// Add repository metadata extraction tracker
107+
var metadataExtractionTracker = new DateTimeTracker(githubStorage, new RepositoryMetadataExtractorTrigger(githubStorage, dbContext, "linksplatform", Path.Combine(Directory.GetCurrentDirectory(), "repository-metadata")));
101108
var cancellation = new CancellationTokenSource();
102109
while (true)
103110
{
@@ -106,6 +113,7 @@ private static async Task<int> Main(string[] args)
106113
await issueTracker.Start(cancellation.Token);
107114
await pullRequenstTracker.Start(cancellation.Token);
108115
// timestampTracker.Start(cancellation.Token);
116+
await metadataExtractionTracker.Start(cancellation.Token);
109117
Thread.Sleep(minimumInteractionInterval);
110118
}
111119
catch (Exception ex)

0 commit comments

Comments
 (0)