Merge pull request #49 from OSLL/developers-google-com-badges-updates

google_export updates

Author: zmm

google_export/Dockerfile

@@ -0,0 +1,12 @@
+FROM python:3.11-slim
+
+WORKDIR /app
+RUN apt update \
+    && apt upgrade -y\
+    && apt install -y curl
+
+COPY requirements.txt requirements.txt
+
+RUN pip3 install -r requirements.txt
+
+COPY . .

google_export/README.md

@@ -7,7 +7,30 @@
 - -k | --key - ключ для экспорта badge'й (брал из dev консоли браузера в запросе, отфильтровав по badges и get запросам (параметр key в конце строки))
 - -c | --curl_args - ключ для нахождения id пользователя, если оно представляется как строковое (аналогично предыдущему, фильтровать по GetProfile, POST-запрос, ключ в header'ах 'X-Goog-Api-Key').
 - -t | --timeout - таймаут 1 запроса
+- -r | --repeat - количество повторных отправок запроса (10 - по умолчнию)
+
+### По аналогии с соседними экспортерами:
+- --google_token - путь до токена для выгрузки в гугл таблицы
+- --table_id - id гугл-таблицы
+- --sheet_id - id листа в гугл таблице
+
+- --yandex_token - токен для яндекс таблиц
+- --yandex_path путь на яндекс диске
+
+### Вместо загрузки id из файлов можно использовать импорт из гугл-таблиц
+- --input_sheet_id - id листа в гугл таблице
+- --input_column_number - номер колонки (например 'B' - 2)
+- --input_column_skip - количество элементов чтобы пропустить
+
+Пример:
+```bash
+python3 main.py -o 'fname.csv' -k 'ключ1' -c 'ключ2' --google_token "токен" --table_id id_таблицы --sheet_id=out --input_sheet_id=in --input_column_number=2 --input_column_skip=3 -t 2
+```
 
 ```bash
 python3 main.py -i 'ids' -o 'fname.csv' -k 'ключ1' -c 'ключ2' -t 0.1
 ```
+### Для запуска через докер
+```bash
+docker run -it --entrypoint python3 <имя контейнера> main.py -i 'ids' -o 'fname.csv' -k 'ключ1' -c 'ключ2' -t 0.1"
+```

google_export/main.py

@@ -1,12 +1,16 @@
-import httpx
-import json
-import csv
 import argparse
-import os
+import csv
+import json
 import subprocess
+import time
 
+import httpx
+import pandas
+
+import sheets
 
-def get_name(user_id: str, curl_args):
+
+def get_name(user_id: str, curl_args, rec_limit=10):
     try:
         if type(user_id) != str or user_id.isdigit():
             status, output = subprocess.getstatusoutput(
@@ -22,10 +26,14 @@ def get_name(user_id: str, curl_args):
         user_id = user_id[1][4][0]
         return user_id
     except httpx.ConnectError:
-        print('ConnectError')
+        if rec_limit == 0:
+            print('ConnectError')
+            return ""
+        time.sleep(0.1)
+        return get_name(user_id, curl_args, rec_limit-1)
     return ""
 
-def get_link(user_id: str, curl_args):
+def get_link(user_id: str, curl_args, rec_limit=10):
     try:
         if type(user_id) != str or user_id.isdigit():
             status, output = subprocess.getstatusoutput(
@@ -41,7 +49,10 @@ def get_link(user_id: str, curl_args):
         user_id = user_id[-1][-1]
         return user_id
     except httpx.ConnectError:
-        print('ConnectError')
+        if rec_limit == 0:
+            print('ConnectError')
+            return ""
+        time.sleep(0.1)
     return ""
 
 def get_id_by_name(user_id: str, curl_args):
@@ -56,12 +67,12 @@ def get_id_by_name(user_id: str, curl_args):
     user_id = user_id[1][31]
     return user_id
 
-def get_awards_by_id(user_id: str | int, key: str, curl_args, timeout) -> dict:
+def get_awards_by_id(user_id: str | int, key: str, curl_args, timeout, rec_limit=10) -> dict:
     print(f'Processing id {user_id}')
     try:
         if not (type(user_id) != str or user_id.isdigit()):
             user_id = get_id_by_name(user_id, curl_args)
-            
+
         c = httpx.get(f'https://developerprofiles-pa.clients6.google.com/v1/awards?access_token&locale&obfuscatedProfileId={user_id}&useBadges=true&key={key}',
             headers={
                 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:128.0) Gecko/20100101 Firefox/128.0',
@@ -96,28 +107,31 @@ def get_awards_by_id(user_id: str | int, key: str, curl_args, timeout) -> dict:
         }
         return award_titles
     except httpx.ConnectError:
-        print('ConnectError')
-        return {}
+        if rec_limit == 0:
+            print('ConnectError')
+            return {}
+        time.sleep(0.1)
+        return get_awards_by_id(user_id, key, curl_args, timeout, rec_limit-1)
 
 
-def get_awards(ids: [str | int], key: str, curl_args, timeout) -> dict[set]:
-    awards = {user_id: get_awards_by_id(user_id, key, curl_args, timeout) for user_id in ids}
+def get_awards(ids: [str | int], key: str, curl_args, timeout, rec_limit=10) -> dict[set]:
+    awards = {user_id: get_awards_by_id(user_id, key, curl_args, timeout, rec_limit) for user_id in ids}
     return awards
 
 
-def write_to_local_csv(awards: dict[set], curl_args, fname: str = 'result.csv') -> None:
+def write_to_local_csv(awards: dict[set], curl_args, fname: str = 'result.csv', rec_limit=10) -> None:
     column_names = set()
     default_columns = [
-        'id', 
-        'name', 
+        'id',
+        'name',
         'link',
-        'public_profile', 
-        'profile created', 
+        'public_profile',
+        'profile created',
     ]
 
     for user_awards in awards.values():
         column_names.update(user_awards)
-    column_names = default_columns + list(column_names)
+    column_names = default_columns + sorted(list(column_names))
     with open(fname, 'w', newline='') as csvfile:
         award_writer = csv.writer(csvfile)
         award_writer.writerow(
@@ -127,10 +141,10 @@ def write_to_local_csv(awards: dict[set], curl_args, fname: str = 'result.csv')
         for user_awards in awards.items():
             row = [
                 get_id_by_name(user_awards[0], curl_args),
-                get_name(user_awards[0], curl_args),
-                get_link(user_awards[0], curl_args),
-                1 if len(user_awards[1]) else 0, 
-                user_awards[1].get('Joined the Google Developer Program'), 
+                get_name(user_awards[0], curl_args, rec_limit),
+                get_link(user_awards[0], curl_args, rec_limit),
+                1 if len(user_awards[1]) else 0,
+                user_awards[1].get('Joined the Google Developer Program'),
             ]
             for award_name in column_names[len(default_columns):]:
                 row.append(user_awards[1][award_name] if award_name in user_awards[1] else 'No')
@@ -150,10 +164,49 @@ def write_to_local_csv(awards: dict[set], curl_args, fname: str = 'result.csv')
     parser.add_argument('-k', '--key')
     parser.add_argument('-c', '--curl_args')
     parser.add_argument('-t', '--timeout', type=float, default=1)
-    args = parser.parse_args()
+    parser.add_argument('-r', '--repeat', type=int, default=10)
+
+    parser.add_argument('--google_token', type=str, required=False, help='Specify path to google token file')
+    parser.add_argument('--table_id', type=str, required=False)
+    parser.add_argument('--sheet_id', type=str, required=False)
+    parser.add_argument('--input_sheet_id', type=str, required=False)
+    parser.add_argument('--input_column_number', type=int, required=False)
+    parser.add_argument('--input_column_skip', type=int, required=False, default=0)
 
-    with open(args.ids_file) as file:
-        lines = [line.rstrip() for line in file]
+    parser.add_argument('--yandex_token', type=str, required=False)
+    parser.add_argument('--yandex_path', type=str, required=False)
+
+    args = parser.parse_args()
+    lines = None
+    if args.ids_file:
+        with open(args.ids_file) as file:
+            lines = [line.rstrip() for line in file]
     # ids = lines
-    q = get_awards(lines, args.key, args.curl_args, args.timeout)
-    write_to_local_csv(q, args.curl_args, args.output)
+    elif args.google_token and args.table_id and args.input_sheet_id:
+        lines = sheets.read_ids_from_table(
+            args.google_token,
+            args.table_id,
+            args.input_sheet_id,
+            args.input_column_number
+        )
+        lines = sheets.cut_lines(lines, args.input_column_skip)
+    else:
+        print('set ids file or google table input')
+    q = get_awards(lines, args.key, args.curl_args, args.timeout, args.repeat)
+    write_to_local_csv(q, args.curl_args, args.output, args.repeat)
+    if args.google_token and args.table_id and args.sheet_id:
+        sheets.write_data_to_table(
+            pandas.read_csv(args.output),
+            args.google_token,
+            args.table_id,
+            args.sheet_id
+        )
+
+    if args.yandex_token and args.yandex_path:
+        import yandex_disk
+        yandex_disk.DiskManager(
+            yatoken=args.yandex_token
+        ).upload(
+            args.output,
+            args.yandex_path
+        )

google_export/requirements.txt

@@ -1,13 +1,43 @@
+annotated-types==0.7.0
 anyio==4.6.0
+cachetools==5.5.1
 certifi==2024.8.30
+charset-normalizer==3.4.1
 distlib==0.3.8
 exceptiongroup==1.2.2
-filelock==3.15.1
+filelock==3.16.1
+google-api-core==2.24.1
+google-api-python-client==2.160.0
+google-auth==2.38.0
+google-auth-httplib2==0.2.0
+google-auth-oauthlib==1.2.1
+googleapis-common-protos==1.66.0
 h11==0.14.0
 httpcore==1.0.5
+httplib2==0.22.0
 httpx==0.27.2
 idna==3.10
-platformdirs==4.2.2
+numpy==2.2.2
+oauthlib==3.2.2
+pandas==2.2.3
+platformdirs==4.3.6
+proto-plus==1.26.0
+protobuf==5.29.3
+pyasn1==0.6.1
+pyasn1_modules==0.4.1
+pydantic_core==2.27.2
+pygsheets==2.0.6
+pyparsing==3.2.1
+python-dateutil==2.9.0.post0
+pytz==2025.1
+requests==2.32.3
+requests-oauthlib==2.0.0
+rsa==4.9
+six==1.17.0
 sniffio==1.3.1
 typing_extensions==4.12.2
-virtualenv==20.26.2
+tzdata==2025.1
+uritemplate==4.1.1
+urllib3==2.3.0
+virtualenv==20.26.6
+yadisk==3.2.0

google_export/sheets.py

@@ -0,0 +1,44 @@
+import pygsheets
+
+def read_ids_from_table(google_token, table_id, sheet_id, column_number):
+    if google_token and sheet_id and table_id:
+        gc = pygsheets.authorize(service_file=google_token)
+        sh = gc.open_by_key(table_id)
+
+    try:
+        sh.worksheets('title', sheet_id)
+    except:
+        sh.add_worksheet(sheet_id)
+
+    wk_content = sh.worksheet_by_title(sheet_id)
+
+    return wk_content.get_col(column_number, include_tailing_empty=False)
+
+def cut_lines(lines: [str], skip=0):
+    prefixes = (
+        'https://g.dev/',
+        'https://developers.google.com/profile/u/'
+    )
+    new_lines = lines[skip:]
+    for i, line in enumerate(new_lines):
+        for prefix in prefixes:
+            if line.startswith(prefix):
+                new_lines[i] = line[len(prefix):]
+    return new_lines
+
+
+def write_data_to_table(df_data, google_token, table_id, sheet_id):
+    df_data = df_data.sort_values(by='Joined the Google Developer Program')
+
+    if google_token and sheet_id and table_id:
+        gc = pygsheets.authorize(service_file=google_token)
+        sh = gc.open_by_key(table_id)
+
+    try:
+        sh.worksheets('title', sheet_id)
+    except:
+        sh.add_worksheet(sheet_id)
+
+    wk_content = sh.worksheet_by_title(sheet_id)
+
+    wk_content.set_dataframe(df_data, 'A1', copy_head=True)

google_export/yandex_disk.py

@@ -0,0 +1,44 @@
+"""Script with DiskManager class and functions for moodle backup
+duplicate moodle_export/*
+"""
+from datetime import datetime
+from os import environ, path
+from logging import getLogger
+
+import yadisk
+
+
+logger = getLogger()
+
+
+class DiskManager():
+    """Light YaDisk manager
+    """
+
+    def __init__(self, yatoken, download_path='./'):
+        self.client = yadisk.Client(token=yatoken or environ.get('YADISK_TOKEN'))
+        self.download_path = download_path
+
+    def upload(self, local_path: str, disk_path: str, overwrite=True):
+        """upload from local_path to disk_path
+
+        Args:
+            local_path (str): path to local file
+            disk_path (str): full path to file on yadisk
+            overwrite (bool): overwrite file. Defaults to true
+        """
+        logger.info("Uploading %s to %s", *(local_path, disk_path))
+        self.client.upload(local_path, disk_path, overwrite=overwrite)
+
+    def download_file_from_disk(self, remote_path: str):
+        """_summary_
+
+        Args:
+            remote_path (str): full path to file on yadisk
+
+        Returns:
+            str: path to downloaded file
+        """
+        local_path = self.download_path + path.basename(remote_path)
+        self.client.download(remote_path, local_path)
+        return local_path