Merge pull request #107 from fossology/feat/package/use-poetry

feat(python): use poetry as package manager

Author: GMishx

.github/ISSUE_TEMPLATE.md

@@ -0,0 +1,28 @@
+<!-- SPDX-FileCopyrightText: © Fossology contributors
+     SPDX-License-Identifier: GPL-2.0-only
+-->
+
+<!-- Search if the Issue do not already exists in the issues (https://github.com/fossology/atarashi/issues). -->
+
+## Description
+
+Please describe your situation in few words here.
+
+### How to reproduce
+
+For a bug: Describe the bug and list the steps you used when the issue occurred.
+
+For an enhancement or new feature: Describe your needs/expected results.
+
+### Screenshots
+
+If applicable, add screenshots to help explain your problem.
+
+### Versions
+
+* Last commit id on master:
+
+
+### Logs
+
+Any logs (if any) generated in the process of reproducing the issue.

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,23 @@
+<!-- SPDX-FileCopyrightText: © Fossology contributors
+     SPDX-License-Identifier: GPL-2.0-only
+-->
+
+<!-- Search if the bug do not already exists in the issues (https://github.com/fossology/atarashi/issues). -->
+<!-- Please refer to CONTRIBUTING.md (https://github.com/fossology/fossology/blob/master/CONTRIBUTING.md)
+before creating the pull request to make sure you follow all the standards. -->
+
+## Description
+
+Please describe the changes in your pull request in few words here.
+
+### Changes
+
+List the changes done to fix a bug or introducing a new feature.
+
+## How to test
+
+Describe the steps required to test the changes proposed in the pull request.
+
+Please consider using the closing keyword if the pull request is proposed to
+fix an issue already created in the repository
+(https://help.github.com/articles/closing-issues-using-keywords/)

.github/depandabot.yml

@@ -0,0 +1,24 @@
+# SPDX-FileCopyrightText: © 2025 Siemens AG
+# SPDX-FileContributor: Kaushlendra Pratap <kaushlendra-pratap.singh@siemens.com>
+
+# SPDX-License-Identifier: GPL-2.0-only
+
+version: 2
+updates:
+  - package-ecosystem: "pip"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+  - package-ecosystem: "docker"
+    directories:
+      - "/"
+    groups:
+      docker:
+        applies-to: security-updates
+        patterns: [ "*" ]
+    schedule:
+      interval: "weekly"

.github/workflows/build-test.yml

@@ -1,48 +1,52 @@
-# SPDX-FileCopyrightText: 2022 Gaurav Mishra <mishra.gaurav@siemens.com>
-# SPDX-License-Identifier: GPL-2.0
+# SPDX-FileCopyrightText: 2025 Gaurav Mishra <mishra.gaurav@siemens.com>
+# SPDX-FileCopyrightText: 2025 Kaushlendra Pratap <kaushlendra-pratap.singh@siemens.com>
+# SPDX-License-Identifier: GPL-2.0-only
 
 name: Build and test packages
 
 on:
-  - "pull_request"
-  - "push"
+  push:
+    branches: [master]
+  pull_request:
+    branches: [master]
 
 jobs:
   build:
 
     strategy:
       matrix:
-        python: [3.5, 3.6, 3.7, 3.8, 3.9]
+        python: ['3.10', '3.11', '3.12']
 
     runs-on: ubuntu-latest
+    env:
+      PYTHONDONTWRITEBYTECODE: "1"
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
       - name: Setup python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python }}
-          architecture: 'x64'
 
       - name: Install build dependencies
         run: |
-          python3 -m pip install --upgrade pip
-          python3 -m pip install --requirement requirements.txt
+          python3 -m pip install --upgrade pip poetry
 
       - name: Build and install
         run: |
-          python3 setup.py build
-          python3 -m pip install .
+          poetry install
+          poetry run preprocess
+          poetry build
 
       - name: Test
         run: |
-          atarashi -h
-          atarashi -a wordFrequencySimilarity ./atarashi/atarashii.py
-          atarashi -a DLD ./atarashi/atarashii.py
-          atarashi -a tfidf -s ScoreSim ./atarashi/atarashii.py
-          atarashi -a tfidf -s CosineSim ./atarashi/atarashii.py
-          atarashi -a Ngram -s CosineSim ./atarashi/atarashii.py
-          atarashi -a Ngram -s DiceSim ./atarashi/atarashii.py
-          atarashi -a Ngram -s BigramCosineSim ./atarashi/atarashii.py
-          atarashi -a Ngram -s BigramCosineSim ./atarashi/agents
+          poetry run atarashi -h
+          poetry run atarashi -a wordFrequencySimilarity ./atarashi/atarashii.py
+          poetry run atarashi -a DLD ./atarashi/atarashii.py
+          poetry run atarashi -a tfidf -s ScoreSim ./atarashi/atarashii.py
+          poetry run atarashi -a tfidf -s CosineSim ./atarashi/atarashii.py
+          poetry run atarashi -a Ngram -s CosineSim ./atarashi/atarashii.py
+          poetry run atarashi -a Ngram -s DiceSim ./atarashi/atarashii.py
+          poetry run atarashi -a Ngram -s BigramCosineSim ./atarashi/atarashii.py
+          poetry run atarashi -a Ngram -s BigramCosineSim ./atarashi/agents

.github/workflows/release-publish.yml

@@ -11,23 +11,24 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
       - name: Setup python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v5
         with:
-          python-version: '3.8'
-          architecture: 'x64'
+          python-version: '3.11'
 
       - name: Install build dependencies
-        run: python3 -m pip install --upgrade wheel --requirement requirements.txt
+        run: python3 -m pip install --upgrade poetry
 
       - name: Build packages
-        run: python3 setup.py sdist bdist_wheel
+        run: |
+          poetry install
+          poetry run preprocess
+          poetry build
 
       - name: Upload Packages to PyPI
         uses: pypa/gh-action-pypi-publish@v1.4.1
         with:
           user: __token__
           password: ${{ secrets.PYPI_API_TOKEN }}
-

.gitignore

@@ -8,6 +8,10 @@ __pycache__/
 
 TestFiles/
 
+# Processed files
+atarashi/data/licenses/processedLicenses.csv
+atarashi/data/Ngram_keywords.json
+
 # Distribution / packaging
 .Python
 build/

README.md

@@ -19,8 +19,9 @@ https://fossology.github.io/atarashi
 
 ### Requirements
 
-- Python >= v3.5
-- pip >= 18.1
+- Python >= v3.10
+- pip >= 25.0
+- poetry >= 2.0.0
 
 ## Steps for Installation
 
@@ -32,21 +33,20 @@ https://fossology.github.io/atarashi
 
 #### Source install
 
-- `pip install .`
+- ```shell
+  poetry install
+  poetry run preprocess
+  ```
 - It will download all dependencies required and trigger build as well.
 - Build will generate 3 new files in your current directory
     1.  `data/Ngram_keywords.json`
     2.  `licenses/<SPDX-version>.csv`
     3.  `licenses/processedList.csv`
 - These files will be placed to their appropriate places by the install script.
 
-### Installing just dependencies
-
-- `pip install -r requirements.txt`
-
 ### Build (optional)
 
-- `$ python3 setup.py build`
+- `poetry build`
 
 ## How to run
 

atarashi/agents/tfidf.py

@@ -78,7 +78,7 @@ def __tfidfsumscore(self, inputFile):
 
     all_documents = self.licenseList['processed_text'].tolist()
     all_documents.append(processedData1)
-    sklearn_tfidf = TfidfVectorizer(min_df=0, use_idf=True, smooth_idf=True,
+    sklearn_tfidf = TfidfVectorizer(min_df=1, use_idf=True, smooth_idf=True,
                                     sublinear_tf=True, tokenizer=tokenize,
                                     vocabulary=processedData)
 
@@ -115,7 +115,7 @@ def __tfidfcosinesim(self, inputFile):
     startTime = time.time()
 
     all_documents = self.licenseList['processed_text'].tolist()
-    sklearn_tfidf = TfidfVectorizer(min_df=0, max_df=0.10, use_idf=True, smooth_idf=True,
+    sklearn_tfidf = TfidfVectorizer(min_df=1, max_df=0.10, use_idf=True, smooth_idf=True,
                                     sublinear_tf=True, tokenizer=tokenize)
 
     all_documents_matrix = sklearn_tfidf.fit_transform(all_documents).toarray()

atarashi/atarashii.py

@@ -18,7 +18,7 @@
 with this program; if not, write to the Free Software Foundation, Inc.,
 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 """
-from pkg_resources import resource_filename
+from importlib_resources import files
 import argparse
 import errno
 import json
@@ -121,8 +121,8 @@ def main():
   Calls atarashii_runner for each file in the folder/ repository specified by user
   Prints the Input path and the JSON output from atarashii_runner
   '''
-  defaultProcessed = resource_filename("atarashi", "data/licenses/processedLicenses.csv")
-  defaultJSON = resource_filename("atarashi", "data/Ngram_keywords.json")
+  defaultProcessed = str(files("atarashi.data.licenses").joinpath("processedLicenses.csv"))
+  defaultJSON = str(files("atarashi.data").joinpath("Ngram_keywords.json"))
   parser = argparse.ArgumentParser()
   parser.add_argument("inputPath", help="Specify the input file/directory path to scan")
   parser.add_argument("-l", "--processedLicenseList", required=False,

atarashi/license/license_merger.py

@@ -73,8 +73,8 @@ def license_merger(licenseList, requiredlicenseList, verbose=0):
         spdx_compatible_shortname, case=False, regex=False).any():
       # SPDX style short name match
       continue
-    licenses_merge = licenses_merge.append(
-      licenses.loc[idx], ignore_index=True, sort=False)
+    licenses_merge = pd.concat([licenses_merge,
+      licenses.loc[idx]], ignore_index=True, sort=False)
 
   if verbose > 0:
     print("Licenses to Merge", len(licenses_merge))
@@ -100,18 +100,7 @@ def license_merger(licenseList, requiredlicenseList, verbose=0):
 
   requiredlicenses = requiredlicenses.drop_duplicates(
     subset='shortname').sort_values(by=['shortname']).reset_index(drop=True)
-  indexesToDrop = []
-  for idx, row in requiredlicenses.iterrows():
-    if len(requiredlicenses.loc[requiredlicenses['shortname'] == \
-                                row['shortname'] + '-only']['deprecated'] == \
-                                True) > 0:
-      indexesToDrop.append(idx)
-    if row['shortname'].endswith('+') and \
-        len(requiredlicenses.loc[requiredlicenses['shortname'] == \
-                                 row['shortname'][:-1] + \
-                                 "-or-later" ]['deprecated'] == True) > 0:
-      indexesToDrop.append(idx)
-  requiredlicenses.drop(indexesToDrop, inplace=True)
+  requiredlicenses = requiredlicenses[requiredlicenses.deprecated == False]
   requiredlicenses.to_csv(str(requiredlicenseList), index=False, encoding='utf-8')
 
   return str(Path(os.path.abspath(requiredlicenseList)))