diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index c7368dcc5..2eee46a1b 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -50,3 +50,7 @@ established in the code. functions. Guidelines are following the [.editorconfig](https://github.com/dzcode-io/dzcode.io/blob/main/.editorconfig) file. + +## Design Decisions + +- Added [bitbucket](../api/src/bitbucket/service.ts) service to support projects hosted in `bitbucket.org`, first one being [Open-listings](../data/models/projects/Open_listings/info.json) diff --git a/api/db/migrations/0008_reflective_whistler.sql b/api/db/migrations/0008_reflective_whistler.sql new file mode 100644 index 000000000..c4a3562d2 --- /dev/null +++ b/api/db/migrations/0008_reflective_whistler.sql @@ -0,0 +1 @@ +ALTER TABLE "contributors" DROP CONSTRAINT "contributors_url_unique"; \ No newline at end of file diff --git a/api/db/migrations/meta/0008_snapshot.json b/api/db/migrations/meta/0008_snapshot.json new file mode 100644 index 000000000..5ca4d3dde --- /dev/null +++ b/api/db/migrations/meta/0008_snapshot.json @@ -0,0 +1,508 @@ +{ + "id": "8989211b-5f96-44fb-8dbd-7a405821a64d", + "prevId": "47729acd-56ae-4a65-931b-5a22f8d471dd", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.ai_prompts": { + "name": "ai_prompts", + "schema": "", + "columns": { + "hash": { + "name": "hash", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "prompt": { + "name": "prompt", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "response": { + "name": "response", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "record_imported_at": { + "name": "record_imported_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "CURRENT_TIMESTAMP" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "public.contributions": { + "name": "contributions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "record_imported_at": { + "name": "record_imported_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "CURRENT_TIMESTAMP" + }, + "title_ar": { + "name": "title_ar", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "title_en": { + "name": "title_en", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "activity_count": { + "name": "activity_count", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "repository_id": { + "name": "repository_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "contributor_id": { + "name": "contributor_id", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "contributions_repository_id_repositories_id_fk": { + "name": "contributions_repository_id_repositories_id_fk", + "tableFrom": "contributions", + "tableTo": "repositories", + "columnsFrom": [ + "repository_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "contributions_contributor_id_contributors_id_fk": { + "name": "contributions_contributor_id_contributors_id_fk", + "tableFrom": "contributions", + "tableTo": "contributors", + "columnsFrom": [ + "contributor_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "contributions_url_unique": { + "name": "contributions_url_unique", + "nullsNotDistinct": false, + "columns": [ + "url" + ] + } + } + }, + "public.contributor_repository_relation": { + "name": "contributor_repository_relation", + "schema": "", + "columns": { + "contributor_id": { + "name": "contributor_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "repository_id": { + "name": "repository_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "record_imported_at": { + "name": "record_imported_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "CURRENT_TIMESTAMP" + }, + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "score": { + "name": "score", + "type": "integer", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "contributor_repository_relation_contributor_id_contributors_id_fk": { + "name": "contributor_repository_relation_contributor_id_contributors_id_fk", + "tableFrom": "contributor_repository_relation", + "tableTo": "contributors", + "columnsFrom": [ + "contributor_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "contributor_repository_relation_repository_id_repositories_id_fk": { + "name": "contributor_repository_relation_repository_id_repositories_id_fk", + "tableFrom": "contributor_repository_relation", + "tableTo": "repositories", + "columnsFrom": [ + "repository_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "contributor_repository_relation_pk": { + "name": "contributor_repository_relation_pk", + "columns": [ + "contributor_id", + "repository_id" + ] + } + }, + "uniqueConstraints": {} + }, + "public.contributors": { + "name": "contributors", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "record_imported_at": { + "name": "record_imported_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "CURRENT_TIMESTAMP" + }, + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name_ar": { + "name": "name_ar", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name_en": { + "name": "name_en", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "username": { + "name": "username", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "avatar_url": { + "name": "avatar_url", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "public.project_tag_relation": { + "name": "project_tag_relation", + "schema": "", + "columns": { + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tag_id": { + "name": "tag_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "record_imported_at": { + "name": "record_imported_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "CURRENT_TIMESTAMP" + }, + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "project_tag_relation_project_id_projects_id_fk": { + "name": "project_tag_relation_project_id_projects_id_fk", + "tableFrom": "project_tag_relation", + "tableTo": "projects", + "columnsFrom": [ + "project_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "project_tag_relation_pk": { + "name": "project_tag_relation_pk", + "columns": [ + "project_id", + "tag_id" + ] + } + }, + "uniqueConstraints": {} + }, + "public.projects": { + "name": "projects", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "record_imported_at": { + "name": "record_imported_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "CURRENT_TIMESTAMP" + }, + "name_ar": { + "name": "name_ar", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name_en": { + "name": "name_en", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "public.repositories": { + "name": "repositories", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "record_imported_at": { + "name": "record_imported_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "CURRENT_TIMESTAMP" + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "stars": { + "name": "stars", + "type": "integer", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "repositories_project_id_projects_id_fk": { + "name": "repositories_project_id_projects_id_fk", + "tableFrom": "repositories", + "tableTo": "projects", + "columnsFrom": [ + "project_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "repositories_provider_owner_name_unique": { + "name": "repositories_provider_owner_name_unique", + "nullsNotDistinct": false, + "columns": [ + "provider", + "owner", + "name" + ] + } + } + }, + "public.tags": { + "name": "tags", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "record_imported_at": { + "name": "record_imported_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "CURRENT_TIMESTAMP" + }, + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + } + }, + "enums": {}, + "schemas": {}, + "sequences": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/api/db/migrations/meta/_journal.json b/api/db/migrations/meta/_journal.json index f8b7c3401..7753c1eb4 100644 --- a/api/db/migrations/meta/_journal.json +++ b/api/db/migrations/meta/_journal.json @@ -57,6 +57,13 @@ "when": 1735852741252, "tag": "0007_rich_trauma", "breakpoints": true + }, + { + "idx": 8, + "version": "7", + "when": 1744408826576, + "tag": "0008_reflective_whistler", + "breakpoints": true } ] } \ No newline at end of file diff --git a/api/docker-compose.yml b/api/docker-compose.yml index f14e426ad..6d6e4740b 100644 --- a/api/docker-compose.yml +++ b/api/docker-compose.yml @@ -11,7 +11,7 @@ services: POSTGRES_DB: db meilisearch: - image: getmeili/meilisearch:latest + image: getmeili/meilisearch:v1.13 ports: - "7700:7700" volumes: diff --git a/api/oracle-cloud/deploy.ts b/api/oracle-cloud/deploy.ts index deb53fa92..2c6e68f8f 100644 --- a/api/oracle-cloud/deploy.ts +++ b/api/oracle-cloud/deploy.ts @@ -50,7 +50,10 @@ const sshServer = isProduction ? process.env.SSH_ADDRESS_PRD : process.env.SSH_A const sshKeyPath = process.env.SSH_PATH; const appPath = "~/app"; const sshPrefix = - "ssh -o StrictHostKeyChecking=no " + (sshKeyPath ? `-i ${sshKeyPath} ` : "") + sshServer + " "; + "ssh -o StrictHostKeyChecking=no -o ServerAliveInterval=60 -o ServerAliveCountMax=60 " + + (sshKeyPath ? `-i ${sshKeyPath} ` : "") + + sshServer + + " "; console.log("⚠️ Cleaning up old containers ..."); logs = execSync(sshPrefix + '"sudo docker container prune --force"'); @@ -76,6 +79,8 @@ logs = execSync( console.log("✅ New code uploaded."); console.log("\n⚙️ Starting up the app"); -logs = execSync(sshPrefix + '"cd ' + appPath + ' && docker compose up -d --build"'); +logs = execSync( + sshPrefix + '"cd ' + appPath + ' && docker compose up -d --build --remove-orphans"', +); console.log(String(logs)); console.log("✅ Deployment successful."); diff --git a/api/src/bitbucket/service.ts b/api/src/bitbucket/service.ts new file mode 100644 index 000000000..ce0a552bd --- /dev/null +++ b/api/src/bitbucket/service.ts @@ -0,0 +1,73 @@ +import { ConfigService } from "src/config/service"; +import { FetchService } from "src/fetch/service"; +import { Service } from "typedi"; + +import { + BitbucketRepositoryContributor, + BitbucketUser, + GetRepositoryInput, + GetRepositoryResponse, + ListRepositoryContributorsResponse, +} from "./types"; + +@Service() +export class BitbucketService { + constructor( + private readonly configService: ConfigService, + private readonly fetchService: FetchService, + ) {} + + public getRepository = async ({ + owner, + repo, + }: GetRepositoryInput): Promise => { + const repoInfo = await this.fetchService.get( + `${this.apiURL}/repositories/${owner}/${repo}`, + { headers: this.bitbucketToken ? { Authorization: `Token ${this.bitbucketToken}` } : {} }, + ); + + return repoInfo; + }; + + public listRepositoryContributors = async ({ + owner, + repo, + }: GetRepositoryInput): Promise => { + interface RepoCommitsResponse { + values: Array<{ author: { user?: BitbucketUser } }>; + next?: string; + } + + const contributorsRecord: Record = {}; + + let url: string | null = `${this.apiURL}/repositories/${owner}/${repo}/commits`; + + while (url) { + const commits: RepoCommitsResponse = await this.fetchService.get(url, { + headers: this.bitbucketToken ? { Authorization: `Token ${this.bitbucketToken}` } : {}, + }); + + for (const commit of commits.values) { + if (!commit.author.user) continue; + + const author = commit.author.user; + if (!contributorsRecord[author.uuid]) { + contributorsRecord[author.uuid] = { ...author, contributions: 0 }; + } + contributorsRecord[author.uuid].contributions += 1; + } + + url = commits.next || null; + } + + const contributors = Object.values(contributorsRecord); + + // @TODO-ZM: validate responses using DTOs, for all fetchService methods + if (!Array.isArray(contributors)) return []; + + return contributors; + }; + + private bitbucketToken = this.configService.env().BITBUCKET_TOKEN; + private apiURL = "https://api.bitbucket.org/2.0"; +} diff --git a/api/src/bitbucket/types.ts b/api/src/bitbucket/types.ts new file mode 100644 index 000000000..4cc5de833 --- /dev/null +++ b/api/src/bitbucket/types.ts @@ -0,0 +1,25 @@ +export interface BitbucketUser { + uuid: string; + username?: string; + display_name: string; + nickname?: string; + links: { avatar: { href: "https://bitbucket.org/account/open-listings/avatar/" } }; + type: "user" | "team"; +} + +export interface BitbucketRepositoryContributor extends BitbucketUser { + contributions: number; +} + +export type ListRepositoryContributorsResponse = BitbucketRepositoryContributor[]; + +export interface GetRepositoryInput { + owner: string; + repo: string; +} + +export interface GetRepositoryResponse { + slug: string; + name: string; + owner: BitbucketUser; +} diff --git a/api/src/config/types.ts b/api/src/config/types.ts index 69faa53c5..0119ef2d8 100644 --- a/api/src/config/types.ts +++ b/api/src/config/types.ts @@ -43,4 +43,8 @@ export class EnvRecord { @IsString() OPENAI_KEY = "no-key"; + + @IsString() + @IsOptional() + BITBUCKET_TOKEN?: string; } diff --git a/api/src/contributor/table.ts b/api/src/contributor/table.ts index 370738a1b..5538912e9 100644 --- a/api/src/contributor/table.ts +++ b/api/src/contributor/table.ts @@ -12,7 +12,7 @@ export const contributorsTable = pgTable("contributors", { name_ar: text("name_ar").notNull(), name_en: text("name_en").notNull(), username: text("username").notNull(), - url: text("url").notNull().unique(), + url: text("url").notNull(), avatarUrl: text("avatar_url").notNull(), }); diff --git a/api/src/digest/cron.ts b/api/src/digest/cron.ts index f5bedcbdf..9706fe327 100644 --- a/api/src/digest/cron.ts +++ b/api/src/digest/cron.ts @@ -16,7 +16,29 @@ import { Service } from "typedi"; import { TagRepository } from "src/tag/repository"; import { AIService } from "src/ai/service"; import { AIResponseTranslateNameDto, AIResponseTranslateTitleDto } from "./dto"; +import { DataProjectEntity } from "src/data/types"; +import { RepositoryEntity } from "@dzcode.io/models/dist/repository"; +import { BitbucketService } from "src/bitbucket/service"; + +type RepoInfo = Pick; +interface RepoContributor { + id: string; + name: string; + username: string; + url: string; + avatarUrl: string; + contributions: number; +} +interface RepoContribution { + user: RepoContributor; + type: "PULL_REQUEST" | "ISSUE"; + title: string; + updatedAt: string; + activityCount: number; + url: string; + id: string; +} @Service() export class DigestCron { private readonly schedule = "15 * * * *"; @@ -33,6 +55,7 @@ export class DigestCron { private readonly searchService: SearchService, private readonly tagRepository: TagRepository, private readonly aiService: AIService, + private readonly bitbucketService: BitbucketService, ) { const SentryCronJob = cron.instrumentCron(CronJob, "DigestCron"); new SentryCronJob( @@ -79,7 +102,7 @@ export class DigestCron { // todo-ZM: make this configurable // uncomment during development // const projectsFromDataFolder = (await this.dataService.listProjects()).filter((p) => - // ["dzcode.io website", "Mishkal", "System Monitor"].includes(p.name), + // ["Open-listings", "dzcode.io website", "Mishkal", "System Monitor"].includes(p.name), // ); // or uncomment to skip the cron // if (Math.random()) return; @@ -128,36 +151,22 @@ it may contain non-translatable parts like acronyms, keep them as is.`; const repositoriesFromDataFolder = project.repositories; for (const repository of repositoriesFromDataFolder) { try { - const repoInfo = await this.githubService.getRepository({ - owner: repository.owner, - repo: repository.name, - }); + const provider = repository.provider; + const repoInfo = await this.getRepoInfo(repository); - const provider = "github"; const [{ id: repositoryId }] = await this.repositoriesRepository.upsert({ + ...repoInfo, provider, - name: repoInfo.name, - owner: repoInfo.owner.login, runId, projectId, - stars: repoInfo.stargazers_count, id: `${provider}-${repoInfo.id}`, }); addedRepositoryCount++; - const issues = await this.githubService.listRepositoryIssues({ - owner: repository.owner, - repo: repository.name, - }); - - for (const issue of issues) { - const githubUser = await this.githubService.getUser({ - username: issue.user.login, - }); - - if (githubUser.type !== "User") continue; + const repoContributions = await this.getRepoContributions(repository); - let name_en = githubUser.name || githubUser.login; + for (const repoContribution of repoContributions) { + let name_en = repoContribution.user.name; let name_ar = name_en; try { const aiRes = await this.aiService.query( @@ -177,17 +186,18 @@ it may contain non-translatable parts like acronyms, keep them as is.`; const contributorEntity: ContributorRow = { name_en, name_ar, - username: githubUser.login, - url: githubUser.html_url, - avatarUrl: githubUser.avatar_url, + username: repoContribution.user.username, + url: repoContribution.user.url, + avatarUrl: repoContribution.user.avatarUrl, runId, - id: `${provider}-${githubUser.login}`, + id: `${provider}-${repoContribution.user.username}`, }; const [{ id: contributorId }] = await this.contributorsRepository.upsert(contributorEntity); await this.searchService.upsert("contributor", contributorEntity); + // todo-zm: insert instead, and allow duplicates, and update the score calculation await this.contributorsRepository.upsertRelationWithRepository({ contributorId, repositoryId, @@ -195,9 +205,9 @@ it may contain non-translatable parts like acronyms, keep them as is.`; score: 1, }); - const type = issue.pull_request ? "PULL_REQUEST" : "ISSUE"; + const type = repoContribution.type; - let title_en = issue.title; + let title_en = repoContribution.title; let title_ar = `ar ${title_en}`; try { const aiRes = await this.aiService.query( @@ -218,33 +228,22 @@ it may contain non-translatable parts like acronyms, keep them as is.`; title_en, title_ar, type, - updatedAt: issue.updated_at, - activityCount: issue.comments, + updatedAt: repoContribution.updatedAt, + activityCount: repoContribution.activityCount, runId, - url: type === "PULL_REQUEST" ? issue.pull_request.html_url : issue.html_url, + url: repoContribution.url, repositoryId, contributorId, - id: `${provider}-${issue.id}`, + id: repoContribution.id, }; await this.contributionsRepository.upsert(contributionEntity); await this.searchService.upsert("contribution", contributionEntity); } - const repoContributors = await this.githubService.listRepositoryContributors({ - owner: repository.owner, - repository: repository.name, - }); - - const repoContributorsFiltered = repoContributors.filter( - (contributor) => contributor.type === "User", - ); - - for (const repoContributor of repoContributorsFiltered) { - const contributor = await this.githubService.getUser({ - username: repoContributor.login, - }); + const repoContributors = await this.getRepoContributors(repository); - let name_en = contributor.name || contributor.login; + for (const repoContributor of repoContributors) { + let name_en = repoContributor.name; let name_ar = `ar ${name_en}`; try { const aiRes = await this.aiService.query( @@ -264,16 +263,17 @@ it may contain non-translatable parts like acronyms, keep them as is.`; const contributorEntity: ContributorRow = { name_en, name_ar, - username: contributor.login, - url: contributor.html_url, - avatarUrl: contributor.avatar_url, + username: repoContributor.username, + url: repoContributor.url, + avatarUrl: repoContributor.avatarUrl, runId, - id: `${provider}-${contributor.login}`, + id: `${provider}-${repoContributor.id}`, }; const [{ id: contributorId }] = await this.contributorsRepository.upsert(contributorEntity); await this.searchService.upsert("contributor", contributorEntity); + // todo-zm: insert instead, and allow duplicates, and update the score calculation await this.contributorsRepository.upsertRelationWithRepository({ contributorId, repositoryId, @@ -320,4 +320,143 @@ it may contain non-translatable parts like acronyms, keep them as is.`; this.logger.info({ message: `Digest cron finished, runId: ${runId}` }); } + + private async getRepoInfo( + reposotory: DataProjectEntity["repositories"][number], + ): Promise { + switch (reposotory.provider) { + case "github": { + const repoInfo = await this.githubService.getRepository({ + owner: reposotory.owner, + repo: reposotory.name, + }); + return { + id: `${repoInfo.id}`, + name: repoInfo.name, + owner: repoInfo.owner.login, + provider: reposotory.provider, + stars: repoInfo.stargazers_count, + }; + } + + case "bitbucket": { + const repoInfo = await this.bitbucketService.getRepository({ + owner: reposotory.owner, + repo: reposotory.name, + }); + return { + id: `${repoInfo.owner.username}-${repoInfo.slug}`, + name: repoInfo.name, + owner: reposotory.owner, + provider: reposotory.provider, + stars: 0, // Bitbucket API doesn't provide stars count + }; + } + + default: + throw new Error(`Unsupported provider: ${reposotory.provider}`); + } + } + + private async getRepoContributors( + reposotory: DataProjectEntity["repositories"][number], + ): Promise { + switch (reposotory.provider) { + case "github": { + const repoContributors = await this.githubService.listRepositoryContributors({ + owner: reposotory.owner, + repository: reposotory.name, + }); + const r = await Promise.all( + repoContributors + .filter(({ type }) => type === "User") + .map(async (contributor) => { + const userInfo = await this.githubService.getUser({ username: contributor.login }); + return { + id: contributor.login, + name: userInfo.name, + avatarUrl: contributor.avatar_url, + url: contributor.html_url, + username: contributor.login, + contributions: contributor.contributions, + }; + }), + ); + + return r; + } + + case "bitbucket": { + const repoContributors = await this.bitbucketService.listRepositoryContributors({ + owner: reposotory.owner, + repo: reposotory.name, + }); + + return repoContributors + .filter(({ type }) => ["user"].includes(type)) + .map((contributor) => ({ + id: contributor.uuid, + name: contributor.display_name, + avatarUrl: contributor.links.avatar.href, + url: "#", // Bitbucket API doesn't provide user URL + username: contributor.username || contributor.display_name.replace(/ /g, "-"), + contributions: contributor.contributions, + })); + } + + default: + throw new Error(`Unsupported provider: ${reposotory.provider}`); + } + } + + private async getRepoContributions( + reposotory: DataProjectEntity["repositories"][number], + ): Promise { + switch (reposotory.provider) { + case "github": { + const repoContributions = await this.githubService.listRepositoryIssues({ + owner: reposotory.owner, + repo: reposotory.name, + }); + return ( + await Promise.all( + repoContributions.map(async (contribution) => { + const githubUser = await this.githubService.getUser({ + username: contribution.user.login, + }); + + if (githubUser.type !== "User") return null; + + return { + user: { + id: githubUser.login, + name: githubUser.name, + avatarUrl: githubUser.avatar_url, + url: githubUser.html_url, + username: githubUser.login, + contributions: 1, + }, + type: contribution.pull_request ? "PULL_REQUEST" : "ISSUE", + title: contribution.title, + updatedAt: contribution.updated_at, + activityCount: contribution.comments, + url: contribution.pull_request + ? contribution.pull_request.html_url + : contribution.html_url, + id: `${reposotory.provider}-${contribution.id}`, + }; + }), + ) + ).filter(Boolean) as RepoContribution[]; + } + + case "bitbucket": { + // todo-ZM: fetch PRs and issues from Bitbucket + return []; + } + + default: + throw new Error(`Unsupported provider: ${reposotory.provider}`); + } + } } diff --git a/api/src/github/types.ts b/api/src/github/types.ts index bd88c484e..8df990dfa 100644 --- a/api/src/github/types.ts +++ b/api/src/github/types.ts @@ -2,7 +2,7 @@ import { GeneralResponse } from "src/app/types"; interface GithubUser { login: string; - name: string | null; + name: string; html_url: string; avatar_url: string; type: "User" | "_other"; diff --git a/package.json b/package.json index fcafaf56a..1b52695bd 100644 --- a/package.json +++ b/package.json @@ -56,10 +56,10 @@ "private": true, "scripts": { "build": "lerna run build:alone --stream", - "build:watch": "lerna run build:alone:watch --parallel --stream", + "build:watch": "lerna run build:alone:watch --parallel", "clean": "lerna run clean:alone --stream", - "deploy": "lerna run deploy --parallel", - "deploy:stg": "lerna run deploy:stg --parallel", + "deploy": "lerna run deploy --parallel --stream", + "deploy:stg": "lerna run deploy:stg --parallel --stream", "dev": "echo \"Please run one of these commands:\\n\\nnpm run dev:web\\nnpm run dev:api\\nnpm run dev:all\n\"", "dev:all": "npm-run-all \"build --include-dependencies {@}\" --parallel \"build:watch --include-dependencies {@}\" \"start:dev {@}\" --", "dev:api": "npm run dev:all --scope=@dzcode.io/api", @@ -67,16 +67,16 @@ "e2e:all": "npm-run-all \"build --include-dependencies {@}\" --parallel \"build:watch --include-dependencies {@}\" \"start:dev {@}\" \"e2e:dev --scope=@dzcode.io/web\" --", "e2e:dev": "lerna run e2e:dev --parallel", "e2e:web": "BROWSER=none npm run e2e:all --scope=@dzcode.io/{web,api}", - "generate:bundle-info": "lerna run generate:bundle-info --parallel --", + "generate:bundle-info": "lerna run generate:bundle-info --parallel --stream --", "generate:sentry-release": "lerna run generate:sentry-release --concurrency 1 --stream --", "lint": "npm run build && npm run lint:alone", - "lint:alone": "lerna run lint:alone --parallel", + "lint:alone": "lerna run lint:alone --parallel --stream", "lint:fix": "npm run build && npm run lint:fix:alone", - "lint:fix:alone": "lerna run lint:fix:alone --parallel", + "lint:fix:alone": "lerna run lint:fix:alone --parallel --stream", "lint:staged": "lerna exec --since HEAD --concurrency 1 --stream -- lint-staged && lint-staged", "patch-package": "patch-package --patch-dir packages/tooling/patches", "postinstall": "npm run patch-package && (husky install && husky set .husky/pre-commit \"npm run lint:staged\") || exit 0", - "pre-deploy": "lerna run pre-deploy --parallel", + "pre-deploy": "lerna run pre-deploy --parallel --stream", "prepare": "ts-patch install -s", "start:dev": "lerna run start:dev --parallel --stream", "test": "npm run build && npm run test:alone", diff --git a/packages/models/src/repository/index.ts b/packages/models/src/repository/index.ts index 83c394754..d35416d0e 100644 --- a/packages/models/src/repository/index.ts +++ b/packages/models/src/repository/index.ts @@ -3,5 +3,6 @@ import { BaseEntity } from "src/_base"; export type RepositoryEntity = BaseEntity & { owner: string; name: string; - provider: "github" | "gitlab"; + provider: "github" | "gitlab" | "bitbucket"; + stars: number; }; diff --git a/web/cloudflare/package.json b/web/cloudflare/package.json index 6ee4a9e02..0c629f5e0 100644 --- a/web/cloudflare/package.json +++ b/web/cloudflare/package.json @@ -14,7 +14,6 @@ }, "private": true, "scripts": { - "ts:check": "tsc --noEmit", "build": "lerna run build:alone --scope=@dzcode.io/web --include-dependencies --stream", "deploy:prd": "npm run generate:config -- production && wrangler pages deploy --branch main", "deploy:stg": "npm run generate:config -- staging && wrangler pages deploy --branch main", @@ -22,6 +21,7 @@ "dev:prepare": "npm run build && cd .. && npm run bundle && npm run pre-deploy", "generate:config": "npx tsx scripts/generate-config.ts", "lint:eslint": "eslint --config ../../packages/tooling/eslint.config.mjs", - "lint:prettier": "prettier --config ../../packages/tooling/.prettierrc --ignore-path ../../packages/tooling/.prettierignore --log-level warn" + "lint:prettier": "prettier --config ../../packages/tooling/.prettierrc --ignore-path ../../packages/tooling/.prettierignore --log-level warn", + "ts:check": "tsc --noEmit" } } diff --git a/web/package.json b/web/package.json index b04bd2fa1..f4f9435f7 100644 --- a/web/package.json +++ b/web/package.json @@ -76,7 +76,6 @@ "generate:robots-txt": "npx tsx src/_build/gen-robots-txt.ts", "generate:sentry-release": "tsx ../packages/tooling/sentry-release.ts web bundle", "generate:sitemap": "npx tsx src/_build/sitemap.ts", - "prune:dictionary": "npx tsx ../packages/tooling/prune-dictionary.ts", "lh:collect": "npx --yes @lhci/cli collect", "lh:upload": "npx --yes @lhci/cli upload", "lint": "npm run build && npm run lint:alone", @@ -88,6 +87,7 @@ "lint:ts-prune": "tsx ../packages/tooling/setup-ts-prune.ts && ts-prune --error", "lint:tsc": "tsc --noEmit", "pre-deploy": "npm run generate:htmls && npm run generate:robots-txt && npm run generate:sitemap && del ./cloudflare/public && cpy \"./bundle/**/*\" ./cloudflare/public", + "prune:dictionary": "npx tsx ../packages/tooling/prune-dictionary.ts", "start:dev": "rsbuild dev --open", "test": "npm run build && npm run test:alone", "test:alone": "jest --config ../packages/tooling/jest.config.ts --rootDir .", diff --git a/web/src/utils/repository.ts b/web/src/utils/repository.ts index 16df67689..b71201293 100644 --- a/web/src/utils/repository.ts +++ b/web/src/utils/repository.ts @@ -10,6 +10,8 @@ export const getRepositoryURL = ({ return `https://www.github.com/${owner}/${name}`; case "gitlab": return `https://www.gitlab.com/${owner}/${name}`; + case "bitbucket": + return `https://bitbucket.org/${owner}/${name}`; default: return ""; }