Skip to content

Commit ce0287d

Browse files
jenzenhojenzenho
authored
Merge pull request #39 from blastnet/firebase
Switch backend from Gist to Firebase
2 parents 945855f + d9bd950 commit ce0287d

File tree

7 files changed

+142
-79
lines changed

7 files changed

+142
-79
lines changed

.github/workflows/update_gist.yml renamed to .github/workflows/update_firebase.yml

Lines changed: 11 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,29 +1,33 @@
1-
name: Update Gist
1+
name: Update Firebase JSON Live
22

33
on:
44
schedule:
5-
- cron: '0 * * * *'
5+
- cron: '0 * * * *' # every hour
66
workflow_dispatch:
77

88
jobs:
9-
update-gist:
9+
update-firebase:
1010
runs-on: ubuntu-latest
1111

1212
steps:
1313
- name: Checkout repository
14-
uses: actions/checkout@v3
14+
uses: actions/checkout@v4
15+
#with:
16+
# repository: blastnet/blastnet.github.io
1517

1618
- name: Setup Python
17-
uses: actions/setup-python@v3
19+
uses: actions/setup-python@v4
1820
with:
1921
python-version: '3.x'
2022

2123
- name: Install dependencies
22-
run: pip install -r requirements.txt
24+
run: |
25+
pip install -r requirements.txt
2326
2427
- name: Run Python script
2528
env:
26-
GIST_TOKEN: ${{ secrets.GIST_TOKEN }}
29+
FIREBASE: ${{ secrets.FIREBASE }}
2730
KAGGLE_USERNAME: ${{ secrets.KAGGLE_USERNAME }}
2831
KAGGLE_KEY: ${{ secrets.KAGGLE_APIKEY }}
2932
run: python kaggle_json.py
33+

‎.gitignore‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,4 +6,5 @@ Gemfile.lock
66
.jekyll-cache
77
*.sqlite
88
kaggle_stats.json
9-
**/.DS_Store
9+
blastnet_backend.json
10+
**/.DS_Store

‎Gemfile‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
source "https://rubygems.org"
22

33
gem "jekyll", "~> 3.10"
4-
54
gem "github-pages", group: :jekyll_plugins
65
gem "jekyll-include-cache", group: :jekyll_plugins
76
gem "jekyll-redirect-from"

‎assets/js/dataset_stats.js‎

Lines changed: 45 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -1,40 +1,57 @@
1-
async function fetchGistData() {
2-
const isLocal = window.location.hostname === "localhost" || window.location.hostname === "127.0.0.1";
3-
const gistURL = 'https://gist.githubusercontent.com/beverleyy/c9112c25c5acd400b90741efa81aa411/raw/kaggle_stats.json';
4-
const localURL = './kaggle_stats.json'; // Path to the local JSON file
5-
6-
let dataURL = gistURL;
7-
8-
if (isLocal) {
9-
try {
10-
// Check if the local JSON file exists
11-
const response = await fetch(localURL, { method: 'HEAD' });
12-
if (response.ok) {
13-
dataURL = localURL;
14-
} else {
15-
console.log("Local JSON file not found. Falling back to Gist.");
16-
}
17-
} catch (error) {
18-
console.log("Error checking local JSON file. Falling back to Gist:", error);
19-
}
1+
async function fetchFirebaseData() {
2+
const firebaseURL = "https://blastnet-backend-default-rtdb.firebaseio.com/kaggle_stats.json";
3+
4+
// Try to get cached ETag and JSON from localStorage
5+
let etag = localStorage.getItem("kaggle_etag");
6+
let cachedJSON = localStorage.getItem("kaggle_stats_json");
7+
8+
let headers = {};
9+
if (etag) {
10+
headers["If-None-Match"] = etag;
11+
} else {
12+
headers["X-Firebase-ETag"] = "true"; // ask Firebase to return an ETag
2013
}
2114

2215
try {
23-
// Fetch data from the appropriate source
24-
console.log(`Fetching data from: ${dataURL}`);
25-
const response = await fetch(dataURL);
26-
const data = await response.json();
16+
console.log(`Fetching data from Firebase REST API: ${firebaseURL}`);
17+
18+
const response = await fetch(firebaseURL, { method: "GET", headers });
19+
20+
let data;
21+
if (response.status === 304) {
22+
// Data hasn't changed — use cached version
23+
data = JSON.parse(cachedJSON);
24+
console.log("Using cached data (not modified).");
25+
} else if (response.ok) {
26+
// New data — save it and the new ETag
27+
const newETag = response.headers.get("ETag");
28+
const jsonString = await response.text();
29+
30+
localStorage.setItem("kaggle_etag", newETag);
31+
localStorage.setItem("kaggle_stats_json", jsonString);
2732

33+
data = JSON.parse(jsonString);
34+
console.log("Loaded fresh data from Firebase.");
35+
} else {
36+
throw new Error(`HTTP error! Status: ${response.status}`);
37+
}
38+
39+
// Parse the data string again
40+
data = JSON.parse(data)
41+
42+
// Extract stats for current page
2843
let fileName = window.location.pathname;
2944
fileName = fileName.split("/").slice(-1)[0].split(".")[0] + ".md";
3045
const stats = data[fileName];
3146

32-
document.querySelectorAll('#kaggle_views')[0].innerHTML = stats.views.toString() + " views";
33-
document.querySelectorAll('#kaggle_downloads')[0].innerHTML = stats.downloads.toString() + " downloads";
34-
document.querySelectorAll('#kaggle_size')[0].innerHTML = stats.size;
47+
document.querySelector('#kaggle_views').innerHTML = stats.views.toString() + " views";
48+
document.querySelector('#kaggle_downloads').innerHTML = stats.downloads.toString() + " downloads";
49+
document.querySelector('#kaggle_size').innerHTML = stats.size;
50+
3551
} catch (error) {
36-
console.log("Failed to fetch data:", error);
52+
console.error("Failed to fetch Firebase data:", error);
3753
}
3854
}
3955

40-
fetchGistData();
56+
fetchFirebaseData();
57+

‎assets/js/homepage_stats.js‎

Lines changed: 48 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -12,44 +12,59 @@ function formatBytes(numBytes) {
1212
return `${numBytes.toFixed(0)} ${units[unitIndex]}`;
1313
}
1414

15-
async function fetchGistData() {
16-
// Determine the data source based on the hostname and local file existence
17-
const isLocal = window.location.hostname === "localhost" || window.location.hostname === "127.0.0.1";
18-
const gistURL = 'https://gist.githubusercontent.com/beverleyy/c9112c25c5acd400b90741efa81aa411/raw/kaggle_stats.json';
19-
const localURL = './kaggle_stats.json'; // Path to the local JSON file
20-
21-
let dataURL = gistURL;
22-
23-
if (isLocal) {
24-
try {
25-
// Check if the local JSON file exists
26-
const response = await fetch(localURL, { method: 'HEAD' });
27-
if (response.ok) {
28-
dataURL = localURL;
29-
} else {
30-
console.log("Local JSON file not found. Falling back to Gist.");
31-
}
32-
} catch (error) {
33-
console.log("Error checking local JSON file. Falling back to Gist:", error);
34-
}
15+
async function fetchFirebaseData() {
16+
const firebaseURL = "https://blastnet-backend-default-rtdb.firebaseio.com/kaggle_stats.json";
17+
18+
// Try to get cached ETag and JSON from localStorage
19+
let etag = localStorage.getItem("kaggle_etag");
20+
let cachedJSON = localStorage.getItem("kaggle_stats_json");
21+
22+
let headers = {};
23+
if (etag) {
24+
headers["If-None-Match"] = etag;
25+
} else {
26+
headers["X-Firebase-ETag"] = "true"; // ask Firebase to return an ETag
3527
}
3628

3729
try {
38-
// Fetch data from the appropriate source
39-
console.log(`Fetching data from: ${dataURL}`);
40-
const response = await fetch(dataURL);
41-
const data = await response.json();
30+
console.log(`Fetching data from Firebase REST API: ${firebaseURL}`);
31+
32+
const response = await fetch(firebaseURL, { method: "GET", headers });
33+
let data;
34+
35+
if (response.status === 304) {
36+
// Data hasn't changed — use cached version
37+
data = JSON.parse(cachedJSON);
38+
console.log("Using cached data (not modified).");
39+
} else if (response.ok) {
40+
// New data — store ETag and JSON string
41+
const newETag = response.headers.get("ETag");
42+
const jsonString = await response.text();
43+
44+
localStorage.setItem("kaggle_etag", newETag);
45+
localStorage.setItem("kaggle_stats_json", jsonString);
4246

43-
// Parse JSON data
47+
data = JSON.parse(jsonString);
48+
console.log("Loaded fresh data from Firebase.");
49+
} else {
50+
throw new Error(`HTTP error! Status: ${response.status}`);
51+
}
52+
53+
// Parse the data again
54+
data = JSON.parse(data)
55+
56+
// Access your data
4457
const total_size = parseInt(data.total_size);
4558
const total_bytes = parseInt(data.total_bytes);
4659

47-
// Print total bytes downloaded
60+
// Format total bytes downloaded
4861
const formattedNumber = total_bytes.toLocaleString();
49-
const wrappedNumber = Array.from(formattedNumber).map(char => (/\d/.test(char) ? `<span>${char}</span>` : `<em>${char}</em>`)).join('');
62+
const wrappedNumber = Array.from(formattedNumber)
63+
.map(char => (/\d/.test(char) ? `<span>${char}</span>` : `<em>${char}</em>`))
64+
.join('');
5065
document.getElementById("kaggle_stat").innerHTML = wrappedNumber + " bytes downloaded";
5166

52-
// Print total dataset size
67+
// Format total dataset size
5368
const formattedSize = formatBytes(total_size);
5469
const wrappedSize = formattedSize.split(/(\s+)/).map(chunk => {
5570
if (/^\d+$/.test(chunk))
@@ -62,7 +77,7 @@ async function fetchGistData() {
6277
document.getElementById("kaggle_size").innerHTML = wrappedSize;
6378
document.getElementById("data_size").innerHTML = formattedSize;
6479

65-
// Extra formatting thanks matthias
80+
// Extra formatting
6681
let children = Array.from(document.getElementById("kaggle_stat").children);
6782
let firstEmIndex = children.findIndex(child => child.tagName.toLowerCase() === 'em');
6883
let byteCount = firstEmIndex === -1
@@ -78,7 +93,9 @@ async function fetchGistData() {
7893
}
7994

8095
} catch (error) {
81-
console.log("Failed to fetch data:", error);
96+
console.error("Failed to fetch Firebase data:", error);
8297
}
8398
}
84-
fetchGistData();
99+
100+
fetchFirebaseData();
101+

‎kaggle_json.py‎

Lines changed: 35 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,19 @@
11
from github import Github
2+
import firebase_admin
23
import traceback,github,requests_cache,re,os,kaggle,json,numpy as np
4+
from firebase_admin import credentials, db
5+
from datetime import datetime,timezone
36

4-
GITHUB_TOKEN = os.getenv("GIST_TOKEN")
5-
GIST_ID = "c9112c25c5acd400b90741efa81aa411"
7+
# Firebase
8+
cred_json = os.getenv("FIREBASE")
9+
cred_dict = json.loads(cred_json)
10+
cred = credentials.Certificate(cred_dict)
11+
#cred = credentials.Certificate("blastnet_backend.json")
612

7-
g = Github(GITHUB_TOKEN)
8-
gist = g.get_gist(GIST_ID)
13+
firebase_admin.initialize_app(cred, {
14+
'databaseURL': "https://blastnet-backend-default-rtdb.firebaseio.com/"
15+
})
16+
ref = db.reference("/")
917

1018
# Format the filesize to unit'ed format
1119
def format_bytes(num_bytes):
@@ -154,7 +162,7 @@ def unformat_bytes(string):
154162
json_dump['total_bytes'] = total_bytes
155163
json_dump['total_size'] = total_size
156164

157-
# Update the gist
165+
# Update the database
158166
# Need the custom encoder class to convert numpy numbers to json readable ones
159167
class NpEncoder(json.JSONEncoder):
160168
def default(self, obj):
@@ -166,12 +174,28 @@ def default(self, obj):
166174
return obj.tolist()
167175
return super(NpEncoder, self).default(obj)
168176

169-
print('Updating {gist}...')
177+
print(f'Updating Firebase…')
178+
170179
try:
171-
gist.edit(files={'kaggle_stats.json': github.InputFileContent(content=json.dumps(json_dump,indent=4,cls=NpEncoder))})
180+
new_json_string = json.dumps(json_dump, indent=4, cls=NpEncoder)
181+
current_json_string = ref.child("kaggle_stats").get()
182+
183+
if current_json_string is None:
184+
current_json_string = ""
185+
if current_json_string.strip() == new_json_string.strip():
186+
print("No change detected, skipping...")
187+
else:
188+
print("Changes detected, writing to firebase...")
189+
ref.update({
190+
"kaggle_stats": new_json_string,
191+
"last_updated": datetime.now(timezone.utc).isoformat()
192+
})
193+
print("Firebase updated.")
194+
172195
except Exception as e:
173-
print(f'Could not update {gist}: {e}')
174-
print(f'Dumping to file...')
175-
with open('kaggle_stats.json','w') as f:
176-
json.dump(json_dump,f,indent=4,cls=NpEncoder)
196+
print(f"Could not update Firebase: {e}")
197+
print(f"Dumping to local file…")
198+
with open("kaggle_stats.json", "w") as f:
199+
json.dump(json_dump, f, indent=4, cls=NpEncoder)
200+
177201
print("Done.")

‎requirements.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
firebase-admin
12
kaggle>=1.6
23
numpy>=1.2
34
requests-cache>=1.2.1

0 commit comments

Comments
 (0)