python script to modify date format

.github/workflows/blogs-data.yaml

@@ -0,0 +1,38 @@
+name: Automated WebScraping & Uploading of Blogs
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 0 * * *" # runs every day at 0:00
+
+jobs:
+  scrape-latest:
+    runs-on: ubuntu-latest
+    steps:
+
+      - name: Checkout repo
+        uses: actions/checkout@v2
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.8'
+      - name: Install dependencies & execute script
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install yake
+          python -m pip install scrapy
+          python -m pip install pymongo
+          python -m pip install bs4
+          python -m pip install dnspython
+          ls -la
+          cd ./Auto_Update_Data
+          python Auto_Update_Data/spiders/Auto-Update.py
+      - name: Adding Log Files
+        run: |
+          git config --local user.email "[email protected]"
+          git config --local user.name "GitHub Action"
+          git pull
+          git add .
+          git commit -m ":rocket: Blogs Updated"
+          git push
+

.github/workflows/papers-data.yml

@@ -0,0 +1,51 @@
+name: Automated Uploading of Papers Data
+
+on:
+  workflow_dispatch:
+  # schedule:
+    # - cron: "0 4 * * SUN" # runs every Sunday at 04:00 UTC
+
+jobs:
+  scrape-latest:
+    runs-on: ubuntu-latest
+    steps:
+
+      - name: Checkout repo
+        uses: actions/checkout@v2
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.8'
+      - name: Downloading Papers Dataset from Kaggle
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install kaggle
+          kaggle datasets download Cornell-University/arxiv
+        env:
+          KAGGLE_USERNAME: ${{ secrets.KAGGLEUSERNAME }}
+          KAGGLE_KEY: ${{ secrets.KAGGLEKEY }}
+      - name: Unzip Downloaded File
+        uses: montudor/action-zip@v1
+        with: 
+          args: unzip -qq arxiv.zip -d arxiv
+      - name: Moving Files to Data Upload Directory
+        run: |
+          ls -la
+          sudo mv ./arxiv/arxiv-metadata-oai-snapshot.json ./data-upload
+      - name: Installing Dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install dask[bag] --upgrade
+          python -m pip install pymongo dnspython pdfplumber uuid yake pandas tqdm
+      - name: Executing Python Script to Upload Papers
+        run: |
+          cd ./data-upload
+          python PaperUpdater.py
+      - name: Adding Log Files
+        run: |
+          git config --local user.email "[email protected]"
+          git config --local user.name "GitHub Action"
+          git pull
+          git add .
+          git commit -m ":rocket: Papers Updated"
+          git push

Auto_Update_Data/Auto_Update_Data/appendkeywords.py

@@ -0,0 +1,303 @@
+# -*- coding: utf-8 -*-
+"""AppendKeywords.ipynb
+
+Automatically generated by Colaboratory.
+
+Original file is located at
+    https://colab.research.google.com/drive/1ytvi7MyZlGZfUhinWHJiqKe3agIyW4_x
+"""
+
+
+# Dictionary of keywords
+# Key: Searching words
+# Value: Displayed words
+
+keywords = {"Machine Learning": "Machine Learning",
+            "Supervised Learning": "Supervised Learning",
+            "Unsupervised Learning": "Unsupervised Learning",
+            "Multilabel Classification": "Multilabel Classification",
+            "Clustering": "Clustering",
+            "K-Means": "K-Means",
+            "DBSCAN": "DBSCAN",
+            "Hierarchical Clustering": "Hierarchical Clustering",
+            "Deep Learning": "Deep Learning",
+            "Data Mining": "Data Mining",
+            "Linear regression": "Linear regression",
+            "Logistic regression": "Logistic regression",
+            "SVM": "SVM",
+            "Natural Language Processing": "Natural Language Processing",
+            "Computer Vision": "Computer Vision",
+            "KNN": "KNN",
+            "Random forest": "Random forest",
+            "Decision Tree": "Decision Tree",
+            "Regularization": "Regularization",
+            "Ensemble Learning": "Ensemble Learning",
+            "Gradient Boosting": "Gradient Boosting",
+            "Feature Selection": "Feature Selection",
+            "Reinforcement Learning": "Reinforcement Learning",
+            "Virtual Reality": "Virtual Reality",
+            "Augmented reality": "Augmented reality",
+            "Autonomous driving": "Autonomous driving",
+            "Optics": "Optics",
+            "Biology": "Biology",
+            "C++": "C++",
+            "Java": "Java",
+            "Python": "Python",
+            "React JS": "React JS",
+            "Computer Network": "Computer Networks",  # remove s
+            "Frontend": "Frontend",
+            "Backend": "Backend",
+            "High Scalability": "High Scalability",
+            "Cloud computing": "Cloud computing",
+            "Parallel Computing": "Parallel Computing",
+            "CUDA": "CUDA",
+            "Distributed System": "Distributed Systems",  # remove s
+            "Apache ZooKeeper": "Apache ZooKeeper",
+            "Streaming analytic": "Streaming analytics",
+            "Model Selection": "Model Selection",
+            "Model Evaluation": "Model Evaluation",
+            "Apache Kafka": "Apache Kafka",
+            "HDFS": "HDFS",
+            "Amazon S3": "Amazon S3",
+            "Pub-Sub": "Pub-Sub",
+            "Leader Election": "Leader Election",
+            "Clock Synchronization": "Clock Synchronization",
+            "Graph": "Graphs",  # remove s
+            "Information Retrieval": "Information Retrieval",
+            "SQL": "SQL",
+            "Graph Database": "Graph Database",
+            "Database Management": "Database Management",
+            "Storage": "Storage",
+            "Memor": "Memory",
+            "Garbage Collection": "Garbage Collection",
+            "Map-Reduce": "Map-Reduce",
+            "Network Protocol": "Network Protocols",  # remove s
+            "Cyber Security": "Cyber Security",
+            "Assembly Language": "Assembly Language",
+            "Computational Complexity Theor": "Computational Complexity Theory",
+            "Computer Architecture": "Computer Architecture",
+            "Human-Computer Interface": "Human-Computer Interface",
+            "Data Structure": "Data Structures",  # remove s
+            "Discrete Mathematic": "Discrete Mathematics",
+            "Hacking": "Hacking",
+            "Quantum Computing": "Quantum Computing",
+            "Robotic": "Robotics",  # remove s
+            "Engineering Practice": "Engineering Practices",  # remove s
+            "Software Tool": "Software Tools",  # remove s
+            "Mathematical Logic": "Mathematical Logic",
+            "Graph Theor": "Graph Theory",
+            "Computational Geometr": "Computational Geometry",
+            "Compiler": "Compilers",  # remove s
+            "Distributed Computing": "Distributed Computing",
+            "Software Engineering": "Software Engineering",
+            "Bioinformatic": "Bioinformatics",  # remove s
+            "Computational Chemistry": "Computational Chemistry",
+            "Computational Neuroscience": "Computational Neuroscience",
+            "Computational physics": "Computational physics",
+            "Numerical algorithm": "Numerical algorithms",  # remove s
+            "JavaScript": "JavaScript",
+            "HTML": "HTML",
+            "Web Development": "Web Development",
+            "App Development": "App Development",
+            "CSS": "CSS",
+            "PHP": "PHP",
+            "BlockChain": "BlockChain",
+            "Hardware": "Hardware",
+            "VLSI": "VLSI",
+            "Cluster Computing": "Cluster Computing",
+            "Kubernetes": "Kubernetes",
+            "Go": "Go-Lang",
+            "File System": "File Systems",  # remove s
+            "Statistic": "Statistics",  # remove s
+            "Optimization": "Optimization",
+            "Knowledge Graph": "Knowledge Graph",
+            "RNN": "RNN",
+            "CNN": "CNN",
+            "Physical Design": "Physical Design",
+            "Memory management": "Memory management",
+            "PCA": "PCA",
+            "LDA": "LDA",
+            "Feature Engineering": "Feature Engineering",
+            "Data manipulation": "Data manipulation",
+            "ACID": "ACID",
+            "BASE": "BASE",
+            "Consistency": "Consistency",
+            "Disaster recovery": "Disaster recovery",
+            "Replication": "Replication",
+            "Fault tolerance": "Fault tolerance",
+            "Deployment": "Deployment",
+            "Processor": "Processors",  # remove s
+            "Multi-Threading": "Multi-Threading",
+            "Queue": "Queue",
+            "Stack": "Stack",
+            "Dynamic Programming": "Dynamic Programming",
+            "Graph Traversal": "Graph Traversal",
+            "Device": "Devices",  # remove s
+            "Data analysis": "Data analysis",
+            "Probability": "Probability",
+            "Mathematic": "Mathematics",  # remove s
+            "Genomic": "Genomics",  # remove s
+            "Data Infrastructure": "Data Infrastructure",
+            "Software Principles and Practices": "Software Principles and Practices",
+            "Image Processing": "Image Processing",
+            "Audio Processing": "Audio Processing",
+            "Signal Processing": "Signal Processing",
+            "Pattern Recognition": "Pattern Recognition",
+            "Computation and Language": "Computation and Language",
+            "Artificial Intelligence": "Artificial Intelligence",
+            "Computation and Language": "Computation and Language",
+            "Computational Complexit": "Computational Complexity",
+            "Computational Engineering": "Computational Engineering",
+            "Finance": "Finance",  # remove "and Science" from "Finance, and Science"
+            "Computational Geometry": "Computational Geometry",
+            "Game Theory": "Game Theory",  # remove "Computer Science" from "Computer Science and Game Theory"
+            "Computer Vision": "Computer Vision",  # break down from "Computer Vision and Pattern Recognition"
+            "Pattern Recognition": "Pattern Recognition",  # break down from "Computer Vision and Pattern Recognition"
+            "Computers and Society": "Computers and Society",
+            "Cryptography and Security": "Cryptography and Security",
+            "Data Structure": "Data Structures",  # break down from "Data Structures and Algorithms"
+            "Algorithm": "Algorithms",  # break down from "Data Structures and Algorithms"
+            "Database": "Databases",  # break down from "Databases; Digital Libraries"
+            "Digital Librar": "Digital Libraries",  # break down from "Databases; Digital Libraries"
+            "Distributed Computing": "Distributed Computing",  # break down from "Distributed, Parallel, and Cluster Computing"
+            "Parallel Computing": "Parallel Computing",  # break down from "Distributed, Parallel, and Cluster Computing"
+            "Cluster Computing": "Cluster Computing",  # break down from "Distributed, Parallel, and Cluster Computing"
+            "Emerging Technolog": "Emerging Technologies",
+            "Formal Language": "Formal Languages",  # break down from "Formal Languages and Automata Theory"
+            "Automata Theory": "Automata Theory",  # break down from "Formal Languages and Automata Theory"
+            "General Literature": "General Literature",
+            "Graphic": "Graphics",  # remove s
+            "Human-Computer Interaction": "Human-Computer Interaction",
+            "Information Theory": "Information Theory",
+            "Logic in Computer Science": "Logic in Computer Science",
+            "Mathematical Software": "Mathematical Software",
+            "Multiagent System": "Multi-agent Systems",  # remove s from "Systems"
+            "Multi-agent System": "Multi-agent Systems",  # remove s from "Systems" and add -
+            "Multimedia": "Multimedia",
+            "Networking and Internet Architecture": "Networking and Internet Architecture",
+            "Neural and Evolutionary Computing": "Neural and Evolutionary Computing",
+            "Numerical Analysis": "Numerical Analysis",
+            "Operating System": "Operating Systems",  # remove s from "Systems"
+            "Performance": "Performance",
+            "Programming Language": "Programming Languages",  # remove s 
+            "Social and Information Networks": "Social and Information Networks",
+            "Software Engineering": "Software Engineering",
+            "Sound": "Sound",
+            "Symbolic Computation": "Symbolic Computation",
+            "Systems and Control": "Systems and Control"
+}
+
+def countKeywords(text, keywords):
+    ''' Count occurence of keywords in the text, return a dict of words and its occurence'''
+    d = {}
+    text = ' ' + text + ' '
+    # Abbreviations list
+    abbreviations = ["SVM", "KNN", "CUDA", "HDFS", "SQL", "HTML", "CSS", "PHP",
+                    "VLSI", "RNN", "CNN", "PCA", "LDA", "ACID", "BASE"]
+
+    for search_word, display_word in keywords.items():
+        # 'Go' can be a sub-string of many words, to be precise, we'll search for the word " Go "
+        if search_word == "Go":
+            search_word = ' ' + search_word + ' '
+
+        # Lower the word if it's not an abbreviation
+        elif search_word not in abbreviations:
+            search_word = search_word.lower()
+
+        # Count occurence of searching word (case sensitive)
+        oc = text.count(search_word)
+
+        # Append to the dictionary if the word occurs 1 or more time
+        if oc > 0:
+            d[display_word] = oc
+
+    return d
+
+from bs4 import BeautifulSoup
+import re
+import string
+import copy
+
+# Funtion to remove HTML tags
+# def removeHTMLTags(text):
+#     return BeautifulSoup(text, 'html.parser').get_text()
+
+
+# Function to remove more special characters and escape characters
+def removeExtraWhitespaceEsc(text):
+    #pattern = r'^\s+$|\s+$'
+    pat = r'^\s*|\s\s*'
+    return re.sub(pat, ' ', text).strip()
+
+
+# Function to remove commas and periods
+def removeCommasPeriods(text):
+    pat = r'[.,]+'
+    return re.sub(pat, '', text)
+
+
+# Function to remove words that include special character
+def removeSpecialCharacterWords(text):
+    # define the pattern to keep only letters, numbers, dash and white spaces
+    pat = r'[a-zA-Z0-9]*[^a-zA-Z0-9_\s]+[a-zA-Z0-9]*'
+    return re.sub(pat, '', text)
+
+
+def clean_data(text):
+    '''
+    Clean text
+    '''
+    #clean_text = removeHTMLTags(text)
+    clean_text = removeExtraWhitespaceEsc(text)
+    clean_text = removeCommasPeriods(clean_text)
+    clean_text = removeSpecialCharacterWords(clean_text)
+
+    return clean_text
+
+import yake
+def keywordsFromYAKE(text, numOfKeywords):
+    '''
+    Extracts keywords from text by using YAKE
+    '''
+
+    kw_extractor = yake.KeywordExtractor()
+    language = "en"
+    max_ngram_size = 2  # max number of words in generated keywords
+    deduplication_threshold = 0.1
+    numOfKeywords = numOfKeywords
+    custom_kw_extractor = yake.KeywordExtractor(lan=language, n=max_ngram_size, dedupLim=deduplication_threshold, top=numOfKeywords, features=None)
+    kws = custom_kw_extractor.extract_keywords(text)
+    result = [x for x, y in kws]
+
+    return result
+
+def keywordsFromBlog(blog, keywords):
+    '''
+    Extracts keywords from a blog.
+      Dict blog: a dictionary of blog content
+      Dict keywords: a dictionary of searched word and displayed word
+    Return
+      A list of five words
+    '''
+
+    text = blog
+
+    N_KEYWORDS = 5
+
+    # Keywords from performing keywords matching
+    occ = countKeywords(text, keywords)
+
+    # get a list of top 5 words
+    result = list(dict(sorted(occ.items(), key=lambda x: x[1], reverse=True)).keys())[:N_KEYWORDS]
+    #print('kwm: {}'.format(result))
+
+    # If the result has less than 5 keywords then use YAKE
+    if len(result) < N_KEYWORDS:
+        n_left = N_KEYWORDS - len(result)
+        yake_kws = keywordsFromYAKE(clean_data(text), n_left)
+        #print('yake_kws: {}'.format(yake_kws))
+        result += [x for x in yake_kws if x not in result]
+
+    return result
+
+