Skip to content

Add a Colab notebook for monitoring Earth Engine noncommercial EECU usage. #975

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
copybara-service wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Add a Colab notebook for monitoring Earth Engine noncommercial EECU usage. #975

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
247 changes: 247 additions & 0 deletions guides/linked/cloud-monitoring/earth_engine_noncommercial_eecu_monitor.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,247 @@
{
"nbformat": 4,
"nbformat_minor": 0,
"metadata": {
"colab": {
"provenance": []
},
"kernelspec": {
"name": "python3",
"display_name": "Python 3"
},
"language_info": {
"name": "python"
}
},
"cells": [
{
"cell_type": "code",
"source": [
"# @title 🌍 Earth Engine Noncommercial EECU Monitor\n",
"# @markdown ### **Instructions**\n",
"# @markdown 1. Enter your **Project ID**.\n",
"# @markdown 2. Select **User Tier** (Optional: Select \"None\" for raw usage, or a tier for quota comparisons).\n",
"# @markdown 3. Run the cell to check your status.\n",
"\n",
"import pandas as pd\n",
"import plotly.graph_objects as go\n",
"from plotly.subplots import make_subplots\n",
"import time\n",
"import calendar\n",
"from datetime import datetime, timedelta, timezone\n",
"from google.colab import auth\n",
"from google.cloud import monitoring_v3\n",
"from IPython.display import display, Markdown\n",
"\n",
"# --- USER CONFIGURATION ---\n",
"PROJECT_ID = \"ee-yourproject\" # @param {type:\"string\"}\n",
"SHOW_QUOTA_TIER = \"Community\" # @param [\"None\", \"Community\", \"Contributor\", \"Partner\"]\n",
"MONTHS_TO_SHOW = 12 # @param {type:\"slider\", min:1, max:12, step:1}\n",
"\n",
"# Define limits based on tier\n",
"TIER_LIMITS = {\n",
" \"Community\": 150,\n",
" \"Contributor\": 1000,\n",
" \"Partner\": 100000\n",
"}\n",
"QUOTA_LIMIT = TIER_LIMITS.get(SHOW_QUOTA_TIER)\n",
"\n",
"# Authenticate\n",
"try:\n",
" auth.authenticate_user()\n",
"except:\n",
" pass\n",
"\n",
"def get_eecu_data(project_id, months_back):\n",
" if not project_id or project_id == \"YOUR_PROJECT_ID_HERE\":\n",
" return pd.DataFrame()\n",
"\n",
" client = monitoring_v3.MetricServiceClient()\n",
" project_name = f\"projects/{project_id}\"\n",
"\n",
" # --- 1. DEFINE TIME WINDOW (CALENDAR ALIGNED) ---\n",
" now = datetime.now(timezone.utc)\n",
"\n",
" # Calculate start date (1st of the target month)\n",
" # Logic: If months_back=1, we want start date = 1st of current month.\n",
" # So we subtract (months_back - 1).\n",
" start_year = now.year\n",
" start_month = now.month - (months_back - 1)\n",
"\n",
" while start_month <= 0:\n",
" start_month += 12\n",
" start_year -= 1\n",
" start_time = datetime(start_year, start_month, 1, tzinfo=timezone.utc)\n",
"\n",
" interval = monitoring_v3.TimeInterval({\n",
" \"end_time\": {\"seconds\": int(now.timestamp()), \"nanos\": 0},\n",
" \"start_time\": {\"seconds\": int(start_time.timestamp()), \"nanos\": 0}\n",
" })\n",
"\n",
" try:\n",
" results = client.list_time_series(\n",
" request={\n",
" \"name\": project_name,\n",
" \"filter\": 'metric.type = \"earthengine.googleapis.com/project/cpu/usage_time\"',\n",
" \"interval\": interval,\n",
" \"view\": monitoring_v3.ListTimeSeriesRequest.TimeSeriesView.FULL,\n",
" \"aggregation\": {\n",
" \"alignment_period\": {\"seconds\": 86400},\n",
" \"per_series_aligner\": monitoring_v3.Aggregation.Aligner.ALIGN_SUM,\n",
" \"cross_series_reducer\": monitoring_v3.Aggregation.Reducer.REDUCE_SUM\n",
" }\n",
" }\n",
" )\n",
" except Exception as e:\n",
" print(f\"Error fetching data: {e}\")\n",
" return pd.DataFrame()\n",
"\n",
" data = []\n",
" for result in results:\n",
" for point in result.points:\n",
" data.append({\n",
" \"date\": point.interval.end_time,\n",
" \"daily_hours\": point.value.double_value / 3600\n",
" })\n",
"\n",
" if not data:\n",
" return pd.DataFrame(columns=[\"date\", \"daily_hours\"])\n",
"\n",
" df = pd.DataFrame(data)\n",
"\n",
" # --- 2. DATA PROCESSING ---\n",
" df[\"date\"] = pd.to_datetime(df[\"date\"]).dt.tz_localize(None).dt.normalize()\n",
" df = df.groupby(\"date\")[\"daily_hours\"].sum().to_frame()\n",
"\n",
" # --- 3. FILL GAPS ---\n",
" last_day_of_month = calendar.monthrange(now.year, now.month)[1]\n",
" end_of_month_date = datetime(now.year, now.month, last_day_of_month)\n",
"\n",
" full_idx = pd.date_range(\n",
" start=start_time.replace(tzinfo=None).date(),\n",
" end=end_of_month_date.date(),\n",
" freq='D'\n",
" )\n",
" df = df.reindex(full_idx, fill_value=0)\n",
" df.index.name = 'date'\n",
"\n",
" # --- 4. CUMULATIVE MATH ---\n",
" df[\"month_group\"] = df.index.to_period('M')\n",
" df[\"monthly_cumulative\"] = df.groupby(\"month_group\")[\"daily_hours\"].cumsum()\n",
"\n",
" return df\n",
"\n",
"# --- OUTPUT GENERATION ---\n",
"if PROJECT_ID != \"YOUR_PROJECT_ID_HERE\":\n",
" df = get_eecu_data(PROJECT_ID, MONTHS_TO_SHOW)\n",
"\n",
" if not df.empty and 'daily_hours' in df.columns:\n",
"\n",
" # --- STATUS CHECK ---\n",
" current_date = pd.Timestamp.today().normalize()\n",
" if current_date in df.index:\n",
" current_val = df.loc[current_date, \"monthly_cumulative\"]\n",
" else:\n",
" current_val = df[\"monthly_cumulative\"].iloc[-1]\n",
"\n",
" if QUOTA_LIMIT:\n",
" if current_val < QUOTA_LIMIT:\n",
" remaining = QUOTA_LIMIT - current_val\n",
" status_msg = f\"**✅ Within {SHOW_QUOTA_TIER} Tier** ({current_val:.1f} / {QUOTA_LIMIT} hrs used). You have **{remaining:.1f} hours** remaining.\"\n",
" else:\n",
" status_msg = f\"**⛔ Quota Depleted** ({current_val:.1f} hrs used). You have exceeded the {QUOTA_LIMIT}hr {SHOW_QUOTA_TIER} limit.\"\n",
" else:\n",
" status_msg = f\"**{current_val:.1f} hours** used this month.\"\n",
"\n",
" display(Markdown(f\"### Current Month Status: {status_msg}\"))\n",
"\n",
" # --- PLOTLY CHART ---\n",
" fig = make_subplots(specs=[[{\"secondary_y\": True}]])\n",
"\n",
" # A. Bar chart (daily)\n",
" fig.add_trace(\n",
" go.Bar(\n",
" x=df.index,\n",
" y=df['daily_hours'],\n",
" name=\"Daily Usage\",\n",
" marker_color='#4c78a8',\n",
" opacity=0.7,\n",
" hovertemplate='%{x|%b %d}<br>Daily: %{y:.2f} hrs<extra></extra>'\n",
" ),\n",
" secondary_y=False,\n",
" )\n",
"\n",
" # B. Line chart (cumulative)\n",
" fig.add_trace(\n",
" go.Scatter(\n",
" x=df.index,\n",
" y=df['monthly_cumulative'],\n",
" name=\"Month Total\",\n",
" mode='lines',\n",
" line=dict(color='#ff7f0e', width=3),\n",
" hovertemplate='%{x|%b %d}<br>Month Total: %{y:.2f} hrs<extra></extra>'\n",
" ),\n",
" secondary_y=True,\n",
" )\n",
"\n",
" # C. month separators\n",
" month_starts = df.resample('MS').first().index\n",
" for date in month_starts:\n",
" fig.add_vline(x=date, line_width=1, line_dash=\"dash\", line_color=\"gray\", opacity=0.3)\n",
"\n",
" # D. Quota guideline (based on selection)\n",
" max_monthly = df['monthly_cumulative'].max()\n",
" range_max = max_monthly * 1.1\n",
"\n",
" if QUOTA_LIMIT:\n",
" fig.add_hline(\n",
" y=QUOTA_LIMIT, line_dash=\"dot\", line_color=\"orange\", secondary_y=True,\n",
" )\n",
" # Ensure limit is visible if usage is low\n",
" range_max = max(range_max, QUOTA_LIMIT * 1.05)\n",
"\n",
" # Layout\n",
" fig.update_layout(\n",
" title_text=f\"Earth Engine EECU Usage (Last {MONTHS_TO_SHOW} Months)\",\n",
" hovermode=\"x unified\",\n",
" showlegend=False,\n",
" height=500,\n",
" bargap=0.1,\n",
" margin=dict(l=20, r=20, t=50, b=20)\n",
" )\n",
"\n",
" max_daily = df['daily_hours'].max()\n",
" if pd.isna(max_daily) or max_daily == 0: max_daily = 1\n",
"\n",
" fig.update_yaxes(title_text=\"Daily Usage (EECU-Hours)\", color='#4c78a8', secondary_y=False, showgrid=False, range=[0, max_daily * 1.1])\n",
" fig.update_yaxes(title_text=\"Cumulative Monthly Total (EECU-Hours)<br>(resets to zero on the first of each month)\", color='#ff7f0e', secondary_y=True, showgrid=False, range=[0, range_max])\n",
"\n",
" fig.show()\n",
"\n",
" # --- SUMMARY TABLE ---\n",
" summary = df.resample('ME')['daily_hours'].agg(['sum', 'mean', 'max']).round(2)\n",
" summary.columns = ['Total EECU-Hours', 'Avg Daily', 'Peak Day']\n",
" summary.index = summary.index.strftime('%B %Y')\n",
" summary = summary.reset_index().rename(columns={'date': 'Month'})\n",
"\n",
" # Display as a clean table\n",
" print(\"\\n\")\n",
" display(summary.style.format({\n",
" 'Total Hours': '{:.2f}',\n",
" 'Avg Daily': '{:.2f}',\n",
" 'Peak Day': '{:.2f}'}).hide(axis='index'))\n",
"\n",
" else:\n",
" print(\"No data found. Check your Project ID.\")\n",
"else:\n",
" print(\"Please enter a valid Project ID above.\")"
],
"metadata": {
"id": "4xMpkYkO2Z2C",
"cellView": "form"
},
"execution_count": null,
"outputs": []
}
]
}
Loading