Skip to content

Commit a48d4f5

Browse files
bjaglinbjaglin
authored
Merge pull request #1751 from scalacenter/sonatype-stats
Add Sonatype statistics
2 parents 87fb44f + a8451f4 commit a48d4f5

File tree

340 files changed

+15432
-0
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

340 files changed

+15432
-0
lines changed

‎.github/scripts/plot.scala‎

Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
package plot
2+
3+
import java.io.File
4+
import java.nio.file.{Files, Paths}
5+
import java.time._
6+
7+
import com.twitter.algebird.Operators._
8+
import plotly._
9+
import plotly.element._
10+
import plotly.layout._
11+
import plotly.Plotly._
12+
import upickle.default._
13+
import ujson.{read => _, _}
14+
15+
import com.github.tototoshi.csv._
16+
17+
object Plot {
18+
def writePlots(): Unit = {
19+
20+
object relevantVersion {
21+
val stableVersionRegex = "^(\\d+)\\.(\\d+)\\.(\\d+)$".r
22+
23+
def unapply(version: String): Option[(Int, Int, Int)] =
24+
version match {
25+
case stableVersionRegex(majorStr, minorStr, patchStr) =>
26+
val major = majorStr.toInt
27+
val minor = minorStr.toInt
28+
val patch = patchStr.toInt
29+
if (major == 2) {
30+
if (minor >= 11) Some((major, minor, patch))
31+
else None
32+
} else Some((major, minor, patch))
33+
case _ => None
34+
}
35+
}
36+
37+
def csvToBars(
38+
dir: File,
39+
allowedVersion: String => Boolean,
40+
filterOutMonths: Set[YearMonth] = Set()
41+
): Seq[Trace] = {
42+
43+
val data = for {
44+
year <- 2015 to Year.now(ZoneOffset.UTC).getValue
45+
month <- 1 to 12
46+
f = new File(dir, f"$year/$month%02d.csv")
47+
if f.exists()
48+
ym = YearMonth.of(year, month)
49+
elem <- CSVReader
50+
.open(f)
51+
.iterator
52+
.map(l => (ym, /* version */ l(0), /* downloads */ l(1).toInt))
53+
.collect {
54+
case (
55+
date,
56+
version @ relevantVersion(major, minor, patch),
57+
downloads
58+
) if allowedVersion(version) =>
59+
(date, (major, minor, patch), downloads)
60+
}
61+
.toVector
62+
} yield elem
63+
64+
data
65+
.groupBy { case (_, version, _) => version }
66+
.mapValues { stats =>
67+
stats
68+
.map { case (date, _, downloads) => (date, downloads) }
69+
.filterNot { case (date, _) => filterOutMonths(date) }
70+
.sortBy { case (date, _) => date }
71+
}
72+
.toSeq
73+
.sortBy { case (version, _) => version }
74+
.map { case ((major, minor, patch), stats) =>
75+
val x = stats.map(_._1).map { m =>
76+
plotly.element.LocalDateTime(m.getYear, m.getMonthValue, 1, 0, 0, 0)
77+
}
78+
val y = stats.map(_._2)
79+
Bar(x, y, name = s"${major}.${minor}.${patch}")
80+
}
81+
}
82+
83+
val dataBase = stats.Params.base
84+
85+
val htmlSnippets =
86+
for {
87+
artifact <- stats.Params.artifacts
88+
(baseDir, divId, title) <- Seq(
89+
(
90+
"per-version-stats",
91+
s"${artifact}-total",
92+
s"${artifact} (total downloads)"
93+
),
94+
(
95+
"per-version-unique-ips",
96+
s"${artifact}-unique",
97+
s"${artifact} (unique IPs)"
98+
)
99+
)
100+
bars = csvToBars(
101+
dataBase.resolve(baseDir).resolve(artifact).toFile,
102+
_ => true /* keep all the versions */
103+
)
104+
} yield s"""
105+
|<h2 id="${divId}-plot">${title} <a href="#${divId}-plot">#</a></h2>
106+
|<div id="${divId}"></div>
107+
|<script>${Plotly.jsSnippet(
108+
divId,
109+
bars,
110+
Layout(barmode = BarMode.Stack)
111+
)}</script>
112+
|""".stripMargin
113+
114+
val html =
115+
s"""<!DOCTYPE html>
116+
|<html>
117+
|<head>
118+
|<title>Scalafix Statistics</title>
119+
|<script src="https://cdn.plot.ly/plotly-${Plotly.plotlyVersion}.min.js"></script>
120+
|</head>
121+
|<body>
122+
|<h1>Scalafix Statistics</h1>
123+
|${htmlSnippets.mkString}
124+
|</body>
125+
|</html>
126+
|""".stripMargin
127+
128+
Files.createDirectories(dataBase)
129+
Files.write(dataBase.resolve("index.html"), html.getBytes("UTF-8"))
130+
131+
}
132+
}

‎.github/scripts/sonatype-stats.scala‎

Lines changed: 184 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,184 @@
1+
package stats
2+
3+
import java.nio.file._
4+
import java.time.{YearMonth, ZoneOffset}
5+
6+
import com.softwaremill.sttp.quick._
7+
import upickle.default._
8+
import ujson.{read => _, _}
9+
10+
object Responses {
11+
12+
case class UniqueIpData(total: Int)
13+
implicit val uniqueIpDataRW: ReadWriter[UniqueIpData] = macroRW
14+
case class UniqueIpResp(data: UniqueIpData)
15+
implicit val uniqueIpRespRW: ReadWriter[UniqueIpResp] = macroRW
16+
17+
case class Elem(id: String, name: String)
18+
implicit val elemRW: ReadWriter[Elem] = macroRW
19+
20+
}
21+
22+
import Responses._
23+
24+
object Params {
25+
26+
// organization one was granted write access to
27+
val proj = sys.env.getOrElse("SONATYPE_PROJECT", "ch.epfl.scala")
28+
// actual organization used for publishing (must have proj as prefix)
29+
val organization = sys.env.getOrElse("SONATYPE_PROJECT", proj)
30+
31+
val sonatypeUser = sys.env.getOrElse(
32+
"SONATYPE_USERNAME",
33+
sys.error("SONATYPE_USERNAME not set")
34+
)
35+
val sonatypePassword: String = sys.env.getOrElse(
36+
"SONATYPE_PASSWORD",
37+
sys.error("SONATYPE_PASSWORD not set")
38+
)
39+
40+
val start = YearMonth.now(ZoneOffset.UTC)
41+
42+
val cutOff = start.minusMonths(4L)
43+
44+
// Note: this assumes the current working directory is the repository root directory!
45+
val base = Paths.get("sonatype-stats")
46+
47+
val artifacts = Set(
48+
"scalafix-core_2.12",
49+
"scalafix-core_2.13",
50+
"scalafix-interfaces"
51+
)
52+
}
53+
54+
case class Data(
55+
base: Path,
56+
ext: String,
57+
empty: String => Boolean,
58+
name: String,
59+
tpe: String,
60+
projId: String,
61+
organization: String,
62+
artifact: Option[String]
63+
) {
64+
65+
def fileFor(monthYear: YearMonth): Path = {
66+
val year = monthYear.getYear
67+
val month = monthYear.getMonth.getValue
68+
base.resolve(f"$year%04d/$month%02d.$ext")
69+
}
70+
71+
def exists(monthYear: YearMonth): Boolean =
72+
Files.isRegularFile(fileFor(monthYear))
73+
74+
def write(monthYear: YearMonth, content: String): Unit = {
75+
System.err.println(s"Writing $monthYear (${content.length} B)")
76+
val f = fileFor(monthYear)
77+
Files.createDirectories(f.getParent)
78+
Files.write(f, content.getBytes("UTF-8"))
79+
}
80+
81+
def urlFor(monthYear: YearMonth) = {
82+
val year = monthYear.getYear
83+
val month = monthYear.getMonth.getValue
84+
85+
uri"https://oss.sonatype.org/service/local/stats/$name?p=$projId&g=$organization&a=${artifact
86+
.getOrElse("")}&t=$tpe&from=${f"$year%04d$month%02d"}&nom=1"
87+
}
88+
89+
def process(monthYears: Iterator[YearMonth]): Iterator[(YearMonth, Boolean)] =
90+
monthYears
91+
.filter { monthYear =>
92+
!exists(monthYear)
93+
}
94+
.map { monthYear =>
95+
val u = urlFor(monthYear)
96+
97+
System.err.println(s"Getting $monthYear: $u")
98+
99+
val statResp = sttp.auth
100+
.basic(Params.sonatypeUser, Params.sonatypePassword)
101+
.header("Accept", "application/json")
102+
.get(u)
103+
.send()
104+
105+
if (!statResp.isSuccess)
106+
sys.error("Error getting project stats: " + statResp.statusText)
107+
108+
val stats = statResp.body.right.get.trim
109+
110+
val empty0 = empty(stats)
111+
if (empty0)
112+
System.err.println(s"Empty response at $monthYear")
113+
else
114+
write(monthYear, stats)
115+
116+
monthYear -> !empty0
117+
}
118+
}
119+
120+
object SonatypeStats {
121+
122+
def collect(): Unit = {
123+
val projId: String = {
124+
val projectIds: Map[String, String] = {
125+
val projResp = sttp.auth
126+
.basic(Params.sonatypeUser, Params.sonatypePassword)
127+
.header("Accept", "application/json")
128+
.get(uri"https://oss.sonatype.org/service/local/stats/projects")
129+
.send()
130+
131+
if (!projResp.isSuccess)
132+
sys.error("Error getting project list: " + projResp.statusText)
133+
134+
val respJson = ujson.read(projResp.body.right.get)
135+
136+
read[Seq[Elem]](respJson("data"))
137+
.map(e => e.name -> e.id)
138+
.toMap
139+
}
140+
141+
projectIds(Params.proj)
142+
}
143+
144+
val artifactStatsPerVersion = Params.artifacts.flatMap { artifact =>
145+
Seq(
146+
Data(
147+
Params.base.resolve("per-version-unique-ips").resolve(artifact),
148+
"csv",
149+
_.isEmpty,
150+
"slices_csv",
151+
"ip",
152+
projId,
153+
Params.organization,
154+
artifact = Some(artifact)
155+
),
156+
Data(
157+
Params.base.resolve("per-version-stats").resolve(artifact),
158+
"csv",
159+
_.isEmpty,
160+
"slices_csv",
161+
"raw",
162+
projId,
163+
Params.organization,
164+
artifact = Some(artifact)
165+
)
166+
)
167+
}
168+
169+
for (data <- artifactStatsPerVersion) {
170+
val it = Iterator.iterate(Params.start)(_.minusMonths(1L))
171+
val processed = data
172+
.process(it)
173+
.takeWhile { case (monthYear, nonEmpty) =>
174+
nonEmpty || monthYear.compareTo(Params.cutOff) >= 0
175+
}
176+
.length
177+
178+
System.err.println(
179+
s"Processed $processed months in ${data.base} for type ${data.tpe}"
180+
)
181+
}
182+
}
183+
184+
}

‎.github/scripts/update.sc‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
#!/usr/bin/env -S scala shebang
2+
3+
// Adapted from https://github.com/alexarchambault/sonatype-stats
4+
//
5+
// /!\ Run it from the repository root directory!
6+
7+
//> using scala "2.12.17"
8+
//> using lib "com.softwaremill.sttp::core:1.5.10"
9+
//> using lib "com.lihaoyi::upickle:2.0.0"
10+
//> using lib "com.github.tototoshi::scala-csv:1.3.5"
11+
//> using lib "com.twitter::algebird-core:0.13.0"
12+
//> using lib "org.plotly-scala::plotly-render:0.5.2"
13+
//> using files "sonatype-stats.scala", "plot.scala"
14+
15+
stats.SonatypeStats.collect()
16+
plot.Plot.writePlots()

‎.github/workflows/sonatype-stats.yml‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
2+
on:
3+
workflow_dispatch:
4+
schedule:
5+
- cron: '0 0 15 * *'
6+
7+
jobs:
8+
update_data:
9+
runs-on: ubuntu-20.04
10+
steps:
11+
- uses: coursier/cache-action@v6
12+
- uses: VirtusLab/[email protected]
13+
- uses: actions/checkout@v3
14+
- name: Update stats
15+
env:
16+
SONATYPE_USERNAME: ${{ secrets.SONATYPE_USERNAME }}
17+
SONATYPE_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }}
18+
run: .github/scripts/update.sc
19+
- name: Push changes
20+
run: |
21+
git config --global user.name 'Scala Center Bot'
22+
git config --global user.email '[email protected]'
23+
git add .
24+
git commit --allow-empty -m "Update stats"
25+
git push
26+
- uses: gautamkrishnar/keepalive-workflow@v1
27+
with:
28+
committer_username: scala-center-bot
29+
committer_email: [email protected]

‎.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,3 +41,6 @@ out/
4141
.metals/
4242
.vscode/
4343
metals.sbt
44+
45+
# Scala CLI specific
46+
.scala-build/

0 commit comments

Comments
 (0)