code cleanup (#73)

* code cleanup

* fixed unused local vars

Author: ronanstokes-db

dbldatagen/data_generator.py

@@ -12,7 +12,7 @@
 from pyspark.sql.types import LongType, IntegerType, StringType, StructType, StructField
 
 from .column_generation_spec import ColumnGenerationSpec
-from .datagen_constants import DEFAULT_RANDOM_SEED, RANDOM_SEED_RANDOM, RANDOM_SEED_FIXED, RANDOM_SEED_HASH_FIELD_NAME
+from .datagen_constants import DEFAULT_RANDOM_SEED, RANDOM_SEED_FIXED, RANDOM_SEED_HASH_FIELD_NAME
 from .spark_singleton import SparkSingleton
 from .utils import ensure, topologicalSort, DataGenError, deprecated
 

dbldatagen/daterange.py

@@ -1,11 +1,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-from datetime import date, datetime, timedelta, timezone
 import math
-
-from pyspark.sql.types import LongType, FloatType, IntegerType, StringType, DoubleType, BooleanType, ShortType, \
-    StructType, StructField, TimestampType, DataType, DateType, ByteType
+from datetime import datetime, timedelta
 
 from .datarange import DataRange
 from .utils import parse_time_interval
@@ -104,7 +101,7 @@ def computeDateRange(cls, begin, end, interval, unique_values):
             assert type(unique_values) is int, "unique_values must be integer"
             assert unique_values >= 1, "unique_values must be positive integer"
 
-            effective_begin = effective_end - effective_interval * (unique_values - 1 )
+            effective_begin = effective_end - effective_interval * (unique_values - 1)
 
         result = DateRange(effective_begin, effective_end, effective_interval)
         return result

dbldatagen/utils.py

@@ -130,7 +130,6 @@ def topologicalSort(sources, initial_columns=None, flatten=True):
             if deps:
                 next_pending.append((name, set(deps)))
             elif name in provided:
-                pass
                 value_emitted |= True
             else:
                 gen.append(name)
@@ -151,6 +150,7 @@ def topologicalSort(sources, initial_columns=None, flatten=True):
 
 PATTERN_NAME_EQUALS_VALUE = re.compile(r"(\w+)\s*\=\s*([0-9]+)")
 PATTERN_VALUE_SPACE_NAME = re.compile(r"([0-9]+)\s+(\w+)")
+_WEEKS_PER_YEAR = 52
 
 
 def parse_time_interval(spec):
@@ -202,14 +202,14 @@ def parse_time_interval(spec):
         elif time_type in ["milliseconds", "millisecond"]:
             milliseconds = time_value
 
-    td = delta = timedelta(
+    delta = timedelta(
         days=days,
         seconds=seconds,
         microseconds=microseconds,
         milliseconds=milliseconds,
         minutes=minutes,
         hours=hours,
-        weeks=weeks
+        weeks=weeks + (years * _WEEKS_PER_YEAR)
     )
 
-    return td
+    return delta

examples/example1.py

@@ -1,9 +1,10 @@
 from datetime import timedelta, datetime
-import math
-from pyspark.sql.types import StructType, StructField, IntegerType, StringType, FloatType, TimestampType
-import dbldatagen as dg
+
 from pyspark.sql import SparkSession
-from pyspark.sql.functions import count, when, isnan, isnull, col, lit, countDistinct
+from pyspark.sql.functions import count, when, isnan, col, lit, countDistinct
+from pyspark.sql.types import StructType, StructField, IntegerType, StringType
+
+import dbldatagen as dg
 
 interval = timedelta(days=1, hours=1)
 start = datetime(2017, 10, 1, 0, 0, 0)
@@ -55,7 +56,6 @@
 print("Summary;", analyzer.summarize())
 
 
-
 def extended_summary(df):
     colnames = [c for c in df.columns]
     colnames2 = ["summary"]

examples/example4.py

@@ -1,10 +1,9 @@
 from datetime import timedelta, datetime
-import math
-from pyspark.sql.types import StructType, StructField, IntegerType, StringType, FloatType, TimestampType
-# from dbldatagen.data_generator import DataGenerator,ensure
-import dbldatagen as dg
-from pyspark.conf import SparkConf
+
 from pyspark.sql import SparkSession
+from pyspark.sql.types import StructType, StructField, IntegerType, StringType
+
+import dbldatagen as dg
 
 interval = timedelta(days=1, hours=1)
 start = datetime(2017, 10, 1, 0, 0, 0)

examples/example5.py

@@ -1,11 +1,9 @@
 from datetime import timedelta, datetime
-import math
-from pyspark.sql.types import StructType, StructField, IntegerType, StringType, FloatType, TimestampType
 
-import dbldatagen as dg
-from pyspark.conf import SparkConf
 from pyspark.sql import SparkSession
 
+import dbldatagen as dg
+
 interval = timedelta(days=1, hours=1)
 start = datetime(2017, 10, 1, 0, 0, 0)
 end = datetime(2018, 10, 1, 6, 0, 0)

examples/example6.py

@@ -1,11 +1,9 @@
 from datetime import timedelta, datetime
-import math
-from pyspark.sql.types import StructType, StructField, IntegerType, StringType, FloatType, TimestampType
-from dbldatagen import DateRange
-import dbldatagen as dg
-from pyspark.conf import SparkConf
+
 from pyspark.sql import SparkSession
 
+import dbldatagen as dg
+
 interval = timedelta(days=1, hours=1)
 start = datetime(2017, 10, 1, 0, 0, 0)
 end = datetime(2018, 10, 1, 6, 0, 0)
@@ -37,7 +35,7 @@
       .withColumn("sector_status_desc", "string", dataRange=range(1, 5), prefix='status', random=True)
       # withColumn adds specification for new column
       .withColumn("rand", "float", expr="floor(rand() * 350) * (86400 + 3600)")
-      .withColumn("last_sync_dt", "timestamp", dataRange=DateRange(start, end, timedelta(days=1, hours=1)),
+      .withColumn("last_sync_dt", "timestamp", dataRange=dg.DateRange(start, end, timedelta(days=1, hours=1)),
                   random=True)
       .withColumnSpec("sector_technology_desc", values=["GSM", "UMTS", "LTE", "UNKNOWN"], random=True)
       .withColumn("test_cell_flg", "int", values=[0, 1], random=True)

tutorial/2-Basics.py

@@ -104,9 +104,7 @@
 # COMMAND ----------
 
 from datetime import timedelta, datetime
-import math
-from pyspark.sql.types import StructType, StructField, IntegerType, StringType, FloatType, TimestampType
-# from dbldatagen.data_generator import DataGenerator,ensure
+from pyspark.sql.types import StructType, StructField, IntegerType, StringType
 import dbldatagen as dg
 
 interval = timedelta(days=1, hours=1)