More management scripts

Author: bnmnetp

scripts/active_users.py

@@ -0,0 +1,63 @@
+import datetime
+import os
+
+from sqlalchemy import (
+    Column,
+    Table,
+    ForeignKey,
+    Index,
+    Integer,
+    String,
+    Date,
+    DateTime,
+    Text,
+    create_engine,
+    types,
+    Float,
+    inspect,
+    MetaData,
+)
+from sqlalchemy.orm import declarative_base, sessionmaker
+
+from sqlalchemy.ext.declarative import declared_attr
+from sqlalchemy.sql.schema import UniqueConstraint
+from sqlalchemy.orm import declarative_base
+
+# Local application imports
+# -------------------------
+# This creates the base class we will use to create models
+
+Base = declarative_base()
+
+# authors will have a role defined for their user_id in auth_group and auth_membership
+engine = create_engine(os.environ["DBURL"])
+Session = sessionmaker(expire_on_commit=False)
+engine.connect()
+Session.configure(bind=engine)
+meta = MetaData()
+
+
+class UserActivity(Base):
+    __tablename__ = "user_activity"
+    timestamp = Column(DateTime, primary_key=True)
+    num_users = Column(Integer)
+
+
+Base.metadata.create_all(bind=engine)
+
+
+def create_activity_entry(num_active: int) -> None:
+    new_row = UserActivity(num_users=num_active, timestamp=datetime.datetime.utcnow())
+    with Session.begin() as session:
+        session.add(new_row)
+
+
+last_five = engine.execute(
+    """
+    select count(distinct sid) from useinfo where timestamp > now() - interval '5 minutes'
+    """
+).first()
+
+num_active = last_five[0]
+
+create_activity_entry(num_active)

scripts/database_pruning.py

@@ -0,0 +1,58 @@
+# archive data from runestone database that is more than 2 years old
+import os
+from sqlalchemy import create_engine
+
+dburl = os.environ["DBURL"]
+eng = create_engine(dburl)
+
+for tbl in [
+    "clickablearea_answers",
+    "codelens_answers",
+    "dragndrop_answers",
+    "fitb_answers",
+    "mchoice_answers",
+    "parsons_answers",
+    "shortanswer_answers",
+    "unittest_answers",
+    "code",
+    "unittest",
+][:1]:
+    res = eng.execute(
+        f"""select id from {tbl} where timestamp = now()::date - interval '2 years' limit 1""",
+        eng,
+    ).first()
+
+    max_id = res[0]
+    print(f"copying rows from {tbl} before {max_id}")
+    # this has to be run on the database server machine or it will fail.  Other options including
+    # using pandas may be better if we want to run remotely
+    num = eng.execute(
+        f"""copy (select * from {tbl} where id < {max_id}) to '{os.environ["HOME"]}/archive_{tbl}_{max_id}.csv' csv header""",
+        eng,
+    ).first()
+    if num and num[0] < 1:
+        print("No records copied removing file")
+        # remove the file
+
+    print(f"Copied {num}")
+
+    res = eng.execute(f"""delete from {tbl} where id < {max_id}""")
+    print(f"deleted {res}")
+
+
+# To load the csv into the main table ignoring duplicates follow this pattern
+# CREATE TEMP TABLE tmp_table
+#     ON COMMIT DROP -- omit if doing from psql
+#     AS
+#     SELECT *
+#     FROM main_table
+#     WITH NO DATA;
+#
+#     COPY tmp_table FROM 'full/file/name/here';
+#
+#     INSERT INTO main_table
+#     SELECT DISTINCT ON (PK_field) *
+#     FROM tmp_table
+#     ORDER BY (some_fields)
+
+# There may also be an option for an ON CONFLiCT DO NOTHING

scripts/populate_lib.py

@@ -110,6 +110,7 @@ def update_library(book):
     return True
 
 
+print("Warning -- this script will leave __pycache__ in every book folder")
 os.chdir(cwd)
 for path in cwd.iterdir():
     if path.is_dir():