fixed sorting parameter -> descending=True

Author: rNLKJA

main.py

@@ -9,6 +9,12 @@
 Github: https://github.com/rNLKJA/2023-S1-COMP90024-A1/
 
 """
+from scripts.mpi import gather_task_tdf, get_task_ranks
+from scripts.utils import *
+from scripts.twitter_processor import *
+from scripts.sal_processor import process_salV1
+from scripts.logger import twitter_logger as logger
+from scripts.arg_parser import parser
 import sys
 import time
 import os
@@ -18,12 +24,6 @@
 
 sys.path.append("./scripts")
 
-from scripts.arg_parser import parser
-from scripts.logger import twitter_logger as logger
-from scripts.sal_processor import process_salV1
-from scripts.twitter_processor import *
-from scripts.utils import *
-from scripts.mpi import gather_task_tdf, get_task_ranks
 
 os.environ["NUMEXPR_MAX_THREADS"] = "32"
 PATH = Path()
@@ -54,17 +54,17 @@
         twitter_file, chunk_start[rank], chunk_end[rank], sal_dict
     )
 
-    logger.info(f"Rank {rank}: File Read Completed, cost: {time.time()- start_time}")
+    logger.info(
+        f"Rank {rank}: File Read Completed, cost: {time.time()- start_time}")
 
     # process twitter data based on three task requirements
     t1_tdf = count_number_of_tweets_by_author(tdf)
     t2_tdf = count_number_of_tweets_by_gcc(tdf)
     t3_tdf = count_author_tweets_from_most_different_gcc(tdf)
-
     # =================================== TASK 1 ===================================
     t1_tdfs = gather_task_tdf(rank, task1_rank, size, t1_tdf, comm)
-    
-    if rank == task1_rank: 
+
+    if rank == task1_rank:
         return_twitter_counts_by_author_id(t1_tdfs, path=PATH)
     # =================================== TASK 2 ===================================
     t2_tdfs = gather_task_tdf(rank, task2_rank, size, t2_tdf, comm)

scripts/twitter_processor.py

@@ -291,7 +291,8 @@ def generate_polars_dataframe(
     )
 
     tweet_df1 = tweet_df.with_columns(
-        pl.col("location").apply(lambda x: normalise_location(x), skip_nulls=True)
+        pl.col("location").apply(
+            lambda x: normalise_location(x), skip_nulls=True)
     )
     tweet_df1 = tweet_df1.join(sal_df, on="location", how="left")
 
@@ -362,7 +363,7 @@ def count_number_of_tweets_by_author(tdf: pl.DataFrame) -> pl.DataFrame:
         tdf.select("author_id", "tweet_id")
         .groupby("author_id")
         .agg(pl.count("tweet_id").alias("tweet_count"))
-        .sort("tweet_count", reverse=True)
+        .sort("tweet_count", descending=True)
     )
 
     return author_tweet_count
@@ -466,7 +467,8 @@ def generate_task_3_result(tdf: pl.DataFrame, save: bool, path: Path) -> pl.Data
     )
 
     tdf1 = tdf1.with_columns(
-        pl.col("gcc_count").rank(method="ordinal", descending=True).alias("rank")
+        pl.col("gcc_count").rank(method="ordinal",
+                                 descending=True).alias("rank")
     )
     tdf1 = tdf1.filter(pl.col("rank") < 11)
 
@@ -498,10 +500,12 @@ def generate_task_3_result(tdf: pl.DataFrame, save: bool, path: Path) -> pl.Data
             ).alias("gtc")
         ]
     ).select("rank", "author_id", "gtc")
-    tdf4.columns = ["Rank", "Author Id", "Number of Unique City Locations and #Tweets"]
+    tdf4.columns = ["Rank", "Author Id",
+                    "Number of Unique City Locations and #Tweets"]
 
     if save:
-        tdf4.sort("Rank", descending=False).write_csv(path / "data/result/task3.csv")
+        tdf4.sort("Rank", descending=False).write_csv(
+            path / "data/result/task3.csv")
         return
     return tdf4
 
@@ -537,7 +541,8 @@ def concate_count_dict_with_rank_df(count_dict: dict) -> pl.DataFrame:
     """
     strings = []
     for key in count_dict.keys():
-        strings.append(", ".join([f"#{v}{k[1:]}" for k, v in count_dict[key].items()]))
+        strings.append(
+            ", ".join([f"#{v}{k[1:]}" for k, v in count_dict[key].items()]))
 
     return pl.DataFrame({"author_id": count_dict.keys(), "nugt": strings})