Skip to content

Commit 109d769

Browse files
leovschleovsch
committed
Finish up initial implementation and version for packaging
1 parent a3288ff commit 109d769

File tree

12 files changed

+437
-21297
lines changed

12 files changed

+437
-21297
lines changed

‎.github/pypi-publish.yml‎

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
name: Publish Python 🐍 distribution 📦 to PyPI
2+
3+
on: push
4+
5+
jobs:
6+
build:
7+
name: Build distribution 📦
8+
runs-on: ubuntu-latest
9+
10+
steps:
11+
- uses: actions/checkout@v4
12+
with:
13+
persist-credentials: false
14+
- name: Set up Python
15+
uses: actions/setup-python@v5
16+
with:
17+
python-version: "3.x"
18+
- name: Install pypa/build
19+
run: >-
20+
python3 -m
21+
pip install
22+
build
23+
--user
24+
- name: Build a binary wheel and a source tarball
25+
run: python3 -m build
26+
- name: Store the distribution packages
27+
uses: actions/upload-artifact@v4
28+
with:
29+
name: python-package-distributions
30+
path: dist/
31+
32+
publish-to-pypi:
33+
name: >-
34+
Publish Python 🐍 distribution 📦 to PyPI
35+
if: startsWith(github.ref, 'refs/tags/') # only publish to PyPI on tag pushes
36+
needs:
37+
- build
38+
runs-on: ubuntu-latest
39+
environment:
40+
name: pypi
41+
url: https://pypi.org/project/dots-infrastructure/ # Replace <package-name> with your PyPI project name
42+
permissions:
43+
id-token: write # IMPORTANT: mandatory for trusted publishing
44+
45+
steps:
46+
- name: Download all the dists
47+
uses: actions/download-artifact@v4
48+
with:
49+
name: python-package-distributions
50+
path: dist/
51+
- name: Publish distribution 📦 to PyPI
52+
uses: pypa/gh-action-pypi-publish@release/v1
53+
54+
github-release:
55+
name: >-
56+
Sign the Python 🐍 distribution 📦 with Sigstore
57+
and upload them to GitHub Release
58+
needs:
59+
- publish-to-pypi
60+
runs-on: ubuntu-latest
61+
62+
permissions:
63+
contents: write # IMPORTANT: mandatory for making GitHub Releases
64+
id-token: write # IMPORTANT: mandatory for sigstore
65+
66+
steps:
67+
- name: Download all the dists
68+
uses: actions/download-artifact@v4
69+
with:
70+
name: python-package-distributions
71+
path: dist/
72+
- name: Sign the dists with Sigstore
73+
uses: sigstore/gh-action-sigstore-python@v3.0.0
74+
with:
75+
inputs: >-
76+
./dist/*.tar.gz
77+
./dist/*.whl
78+
- name: Create GitHub Release
79+
env:
80+
GITHUB_TOKEN: ${{ github.token }}
81+
run: >-
82+
gh release create
83+
"$GITHUB_REF_NAME"
84+
--repo "$GITHUB_REPOSITORY"
85+
--notes ""
86+
- name: Upload artifact signatures to GitHub Release
87+
env:
88+
GITHUB_TOKEN: ${{ github.token }}
89+
# Upload to GitHub Release using the `gh` CLI.
90+
# `dist/` contains the built packages, and the
91+
# sigstore-produced signatures and certificates.
92+
run: >-
93+
gh release upload
94+
"$GITHUB_REF_NAME" dist/**
95+
--repo "$GITHUB_REPOSITORY"

‎README.md‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,17 @@
1-
# Python-project-template
2-
A template repository for new python projects
1+
# Network file parser
2+
A python package parsing gaia files.
3+
4+
Example usage:
5+
6+
```python
7+
from ElectricityNetworFileParser.GnfParser import GnfParser
8+
9+
gnf_parser = GnfParser("test.gnf")
10+
gnf_parser.parse_file()
11+
12+
# Write all data to a single excel file each tab in the excel file represents an entity in the gaia file
13+
gnf_parser.write_all_data_frames()
14+
```
315

416
## Installation
517

‎pyproject.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ requires = ["hatchling"]
33
build-backend = "hatchling.build"
44

55
[project]
6-
name = "NetworFileParser"
6+
name = "ElectricityNetworFileParser"
77
version = "0.0.1"
88
authors = [
99
{ name="Leo van Schooten", email="l.g.t.v.schooten@tue.nl" },

src/NetworFileParser/FileParser.py renamed to src/ElectricityNetworFileParser/FileParser.py

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33

44
from pathlib import Path
55

6-
from NetworFileParser.dataclasses import PropertyDescription
6+
from ElectricityNetworFileParser.dataclasses import PropertyDescription
77

88
class FileParser:
99
def __init__(self, file_path):
@@ -90,6 +90,9 @@ def parse_entities(self, lines : List[str], property_attributes_to_parse : List[
9090
self.extend_dictionary(data_instance, general_properties.property_attributes)
9191
parsed_property_types.append(general_properties.property_type)
9292

93+
if len(data_instance.items()) > 0:
94+
data_instances.append(data_instance)
95+
9396
return pd.DataFrame(data_instances)
9497

9598
def create_entity_dict(self, lines):
@@ -106,9 +109,13 @@ def create_entity_dict(self, lines):
106109

107110
def group_data_frame_by_columns(self, df : pd.DataFrame, columns_to_group_by : List[str]) -> pd.DataFrame:
108111
for col in columns_to_group_by:
109-
df[col] = df.apply(lambda x, col=col: -1 if pd.isnull(x[col]) else x[col], axis=1)
112+
df[col] = df.apply(lambda x, col=col: -1 if pd.isnull(x[col]) or pd.isna(x[col]) else x[col], axis=1)
110113
result = df.groupby(columns_to_group_by).size().reset_index().rename(columns={0:'count'})
111114
return result
115+
116+
def get_records_containing_field_values(self, df : pd.DataFrame, fields : dict) -> pd.DataFrame:
117+
query = " and ".join([f"{key} == {value}" for key, value in fields.items()])
118+
return df.query(query)
112119

113120
def parse_cable_types(self, cables_df : pd.DataFrame) -> pd.DataFrame:
114121
pass
@@ -121,14 +128,14 @@ def write_all_data_frames(self, file_name : str = "data.xlsx"):
121128

122129
def parse_file(self):
123130
for key, value in self.parse_entities_dict.items():
124-
if key in self.parse_entities_dict.keys():
131+
if key in self.parse_entities_dict.keys() and key in self.entity_dict.keys():
125132
if key not in self.data_frames.keys():
126133
self.data_frames[key] = pd.DataFrame()
127134
self.data_frames[key] = pd.concat([self.data_frames[key], self.parse_entities(self.entity_dict[key], value)])
128-
129-
def write_all_data_frames(self, file_name : str = "data.xlsx"):
130135
if "CABLE" in self.data_frames.keys():
131136
self.data_frames["CABLETYPE"] = self.parse_cable_types(self.data_frames["CABLE"])
137+
138+
def write_all_data_frames(self, file_name : str = "data.xlsx"):
132139
with pd.ExcelWriter(file_name) as writer:
133140
for name, dataframe in self.data_frames.items():
134141
dataframe.to_excel(writer, sheet_name=name, index=False)

src/NetworFileParser/GnfParser.py renamed to src/ElectricityNetworFileParser/GnfParser.py

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
import pandas as pd
2-
from NetworFileParser.FileParser import FileParser
2+
from ElectricityNetworFileParser.FileParser import FileParser
33

44
class GnfParser(FileParser):
55

@@ -30,4 +30,18 @@ def parse_cable_types(self, cables_df : pd.DataFrame) -> pd.DataFrame:
3030
"X_ch_o", "R_hh_n", "X_hh_n", "R_hh_o", "X_hh_o", "R_he", "X_he",
3131
"Inom_h", "Ik1s_h"]
3232

33-
return self.group_data_frame_by_columns(cables_df, columns_to_group_by)
33+
unique_cable_types = self.group_data_frame_by_columns(cables_df, columns_to_group_by)
34+
dictionaries = unique_cable_types.to_dict('records')
35+
long_names = []
36+
short_names = []
37+
for dictionary in dictionaries:
38+
dictionary.pop("count")
39+
cables_with_cable_type_params = self.get_records_containing_field_values(cables_df, dictionary)
40+
cable_types = cables_with_cable_type_params["CableType"].unique()
41+
short_names_values = cables_with_cable_type_params["ShortName"].unique()
42+
long_names.append(",".join(cable_types))
43+
short_names.append(",".join(short_names_values))
44+
45+
unique_cable_types["Longnames"] = long_names
46+
unique_cable_types["Shortnames"] = short_names
47+
return unique_cable_types

src/NetworFileParser/__init__.py renamed to src/ElectricityNetworFileParser/__init__.py

File renamed without changes.

src/NetworFileParser/dataclasses.py renamed to src/ElectricityNetworFileParser/dataclasses.py

File renamed without changes.

‎test/Alliander_Arch1_Net1_anoniem_2023_Q2.gnf‎

Lines changed: 0 additions & 21273 deletions
This file was deleted.

‎test/TestGnfParser.py‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
import unittest
2+
3+
import pandas as pd
4+
from ElectricityNetworFileParser.GnfParser import GnfParser
5+
6+
class Test(unittest.TestCase):
7+
8+
def setUp(self):
9+
self.gnf_parser = GnfParser("test/test.gnf")
10+
11+
def test_gnf_parser(self):
12+
self.gnf_parser.parse_file()
13+
14+
def test_parse_property_line_function(self):
15+
property_line = "#General StringValue:'Test string 1' NumberValue:1 BooleanValue:True FloatValue:2,53"
16+
expected_output = {
17+
"StringValue": "Test string 1",
18+
"NumberValue": 1,
19+
"BooleanValue": True,
20+
"FloatValue": 2.53
21+
}
22+
result = self.gnf_parser.parse_property_line(property_line)
23+
self.assertEqual(result.property_attributes, expected_output)
24+
self.assertEqual(result.property_type, "General")
25+
26+
def test_parse_entities_function(self):
27+
# Arrange
28+
lines = [
29+
"#General StringValue:'Test string 1' NumberValue:1 BooleanValue:True FloatValue:2,53",
30+
"#Attributes StringValue2:'Test string 2' NumberValue2:2 BooleanValue2:False FloatValue2:3,53",
31+
"#General StringValue:'Test string 3' NumberValue:3 BooleanValue:False FloatValue:3,14",
32+
"#Attributes StringValue2:'Test string 4' NumberValue2:4 BooleanValue2:False FloatValue2:4,53"
33+
]
34+
35+
# Execute
36+
result_df = self.gnf_parser.parse_entities(lines, ["General", "Attributes"])
37+
38+
# Assert
39+
expected_output_dict = {
40+
"StringValue": ["Test string 1", "Test string 3"],
41+
"NumberValue": [1, 3],
42+
"BooleanValue": [True, False],
43+
"FloatValue": [2.53, 3.14],
44+
"StringValue2": ["Test string 2", "Test string 4"],
45+
"NumberValue2": [2, 4],
46+
"BooleanValue2": [False, False],
47+
"FloatValue2": [3.53, 4.53]
48+
}
49+
expected_output_df = pd.DataFrame(expected_output_dict)
50+
self.assertTrue(expected_output_df.equals(result_df))
51+
52+
def test_parse_cable_types(self):
53+
# Arrange
54+
cables_df = self.gnf_parser.parse_entities(self.gnf_parser.entity_dict["CABLE"], ["General", "CablePart", "CableType"])
55+
56+
# Execute
57+
cabletype_df = self.gnf_parser.parse_cable_types(cables_df)
58+
59+
# Assert
60+
expected_number_of_rows = 1
61+
expected_long_names = "4x95Al+4x6Cu+35CuAs(V-VMvKhsas),4x95Al+4x6Cu+35CuAs(V-VMvKhsas)-test"
62+
self.assertEqual(cabletype_df.shape[0], expected_number_of_rows)
63+
self.assertEqual(cabletype_df["Longnames"].iloc[0], expected_long_names)
64+
65+
if __name__ == '__main__':
66+
unittest.main()

‎test/TestParser.py‎

Lines changed: 0 additions & 14 deletions
This file was deleted.

0 commit comments

Comments
 (0)