Merge pull request #194 from jecisc/fix-various-bugs

Fix various bugs

Author: jecisc

src/DataFrame-Math/DataPearsonCorrelationMethod.class.st

@@ -9,12 +9,10 @@ DataPearsonCorrelationMethod class >> between: x and: y [
 	"Calcualte the Pearson correlation coefficient between two data series"
 
 	| xDeviation yDeviation |
+	x size = y size ifFalse: [ SizeMismatch signal: 'Correlation can not be calculated for two series of different size' ].
 
-	x size = y size ifFalse: [
-		SizeMismatch signal: 'Correlation can not be calculated for two series of different size' ].
-
-	xDeviation := x - x average.
-	yDeviation := y - y average.
+	xDeviation := (x values replaceAll: nil with: 0) - x average.
+	yDeviation := (y values replaceAll: nil with: 0) - y average.
 
 	^ (xDeviation * yDeviation) sum / ((xDeviation ** 2) sum * (yDeviation ** 2) sum) sqrt
 ]

src/DataFrame-Tests/DataFrameStatsTest.class.st

@@ -32,33 +32,58 @@ DataFrameStatsTest >> testAverage [
 	self assert: actual closeTo: expected
 ]
 
+{ #category : #tests }
+DataFrameStatsTest >> testAverageWithNils [
+
+	| expected actual |
+	df := DataFrame withRows: #( #( 1 1 ) #( 2 nil ) #( nil 3 ) #( 4 4 ) ).
+	df columnNames: #( dogs cats ).
+
+	expected := {
+		            (7 / 3).
+		            (8 / 3) } asDataSeries.
+	expected name: #average.
+	expected keys: df columnNames.
+
+	actual := df average.
+	self assert: actual closeTo: expected
+]
+
 { #category : #tests }
 DataFrameStatsTest >> testCorrelationMatrix [
 
 	| expectedCorrelationMatrix actualCorrelationMatrix |
 	expectedCorrelationMatrix := DataFrame withRows:
-		                             #( #( 1 0.311398 0.538922 0.454601 )
-		                                #( 0.311398 1 -0.321281 -0.308023 )
-		                                #( 0.538922 -0.321281 1 0.982956 )
-		                                #( 0.454601 -0.308023 0.982956
-		                                   1 ) ).
-	expectedCorrelationMatrix columnNames:
-		#( sepalLength sepalWidth petalLength petalWidth ).
-	expectedCorrelationMatrix rowNames:
-		#( sepalLength sepalWidth petalLength petalWidth ).
+		                             #( #( 1 0.311398 0.538922 0.454601 ) #( 0.311398 1 -0.321281 -0.308023 ) #( 0.538922 -0.321281 1 0.982956 )
+		                                #( 0.454601 -0.308023 0.982956 1 ) ).
+	expectedCorrelationMatrix columnNames: #( sepalLength sepalWidth petalLength petalWidth ).
+	expectedCorrelationMatrix rowNames: #( sepalLength sepalWidth petalLength petalWidth ).
 	actualCorrelationMatrix := df correlationMatrix.
-	self
-		assert: actualCorrelationMatrix rowNames
-		equals: expectedCorrelationMatrix rowNames.
-	self
-		assert: actualCorrelationMatrix columnNames
-		equals: expectedCorrelationMatrix columnNames.
+	self assert: actualCorrelationMatrix rowNames equals: expectedCorrelationMatrix rowNames.
+	self assert: actualCorrelationMatrix columnNames equals: expectedCorrelationMatrix columnNames.
+
+	1 to: actualCorrelationMatrix numberOfColumns do: [ :j |
+		1 to: actualCorrelationMatrix numberOfRows do: [ :i | self assert: (actualCorrelationMatrix at: i at: j) closeTo: (expectedCorrelationMatrix at: i at: j) ] ]
+]
+
+{ #category : #tests }
+DataFrameStatsTest >> testCorrelationMatrixWithNils [
+
+	| expectedCorrelationMatrix actualCorrelationMatrix |
+	df := DataFrame withRows: #( #( 1 1 ) #( 2 nil ) #( nil 3 ) #( 4 4 ) ).
+	df columnNames: #( dogs cats ).
+
+	expectedCorrelationMatrix := DataFrame withRows: #( #( 1 0.4174555390689118 ) #( 0.4174555390689118 1 ) ).
+	expectedCorrelationMatrix columnNames: #( dogs cats ).
+	expectedCorrelationMatrix rowNames: #( dogs cats ).
+
+	actualCorrelationMatrix := df correlationMatrix.
+
+	self assert: actualCorrelationMatrix rowNames equals: expectedCorrelationMatrix rowNames.
+	self assert: actualCorrelationMatrix columnNames equals: expectedCorrelationMatrix columnNames.
 
 	1 to: actualCorrelationMatrix numberOfColumns do: [ :j |
-		1 to: actualCorrelationMatrix numberOfRows do: [ :i |
-			self
-				assert: (actualCorrelationMatrix at: i at: j)
-				closeTo: (expectedCorrelationMatrix at: i at: j) ] ]
+		1 to: actualCorrelationMatrix numberOfRows do: [ :i | self assert: (actualCorrelationMatrix at: i at: j) closeTo: (expectedCorrelationMatrix at: i at: j) ] ]
 ]
 
 { #category : #tests }

src/DataFrame-Tests/DataFrameTest.class.st

@@ -898,6 +898,33 @@ DataFrameTest >> testColumnsSubset [
 	self assert: actualDataFrame equals: expectedDataFrame
 ]
 
+{ #category : #tests }
+DataFrameTest >> testCopy [
+
+	| copy |
+	copy := df copy.
+
+	self assert: copy equals: df.
+	self deny: copy identicalTo: df.
+
+	df addRow: #( 'Paris' 7 false ) named: 'D'.
+
+	self assert: df size equals: 4.
+	self assert: copy size equals: 3
+]
+
+{ #category : #tests }
+DataFrameTest >> testCopy2 [
+
+	| copy |
+	copy := df copy.
+
+	df addColumn: #( false true true ) named: 'Like it'.
+
+	self assert: df numberOfColumns equals: 4.
+	self assert: copy numberOfColumns equals: 3
+]
+
 { #category : #tests }
 DataFrameTest >> testCreateDataFrameWith3ColumnsAndNoRows [
 	| dataFrame |

src/DataFrame-Tests/DataSeriesTest.class.st

@@ -531,6 +531,18 @@ DataSeriesTest >> testAtTransformIfAbsent [
 	self assert: exceptionBlockEvaluated
 ]
 
+{ #category : #'tests - arithmetic' }
+DataSeriesTest >> testAverage [
+
+	self assert: #( 1 2 3 4 ) asDataSeries average equals: 5 / 2
+]
+
+{ #category : #'tests - arithmetic' }
+DataSeriesTest >> testAverageWithNils [
+
+	self assert: #( 2 nil 4 ) asDataSeries average equals: 3
+]
+
 { #category : #'tests - comparing' }
 DataSeriesTest >> testBooleanGreaterThanEqualFromScalar [
 
@@ -981,6 +993,21 @@ DataSeriesTest >> testCollectWithNotNils [
 	self assert: actual equals: expected
 ]
 
+{ #category : #running }
+DataSeriesTest >> testCopy [
+
+	| copy |
+	copy := series copy.
+
+	self assert: copy equals: series.
+	self deny: copy identicalTo: series.
+
+	series add: $l -> 30.
+
+	self assert: series size equals: 12.
+	self assert: copy size equals: 11
+]
+
 { #category : #'tests - copying' }
 DataSeriesTest >> testCopyCanBeChanged [
 
@@ -1392,6 +1419,19 @@ DataSeriesTest >> testInjectInto [
 	self assert: actual equals: expected
 ]
 
+{ #category : #'tests - testing' }
+DataSeriesTest >> testIsNumerical [
+
+	self assert: #( 1 2 3 ) asDataSeries isNumerical.
+	self deny: #( 1 2 '3' ) asDataSeries isNumerical
+]
+
+{ #category : #'tests - testing' }
+DataSeriesTest >> testIsNumericalWithNils [
+
+	self assert: #( 1 nil 3 ) asDataSeries isNumerical
+]
+
 { #category : #'tests - accessing' }
 DataSeriesTest >> testLast [
 
@@ -2095,6 +2135,19 @@ DataSeriesTest >> testStatsZerothQuartileEqualsMin [
 	self assert: series zerothQuartile equals: series min
 ]
 
+{ #category : #'tests - arithmetic' }
+DataSeriesTest >> testSum [
+
+	self assert: #( 1 2 3 4 ) asDataSeries sum equals: 10
+]
+
+{ #category : #'tests - arithmetic' }
+DataSeriesTest >> testSumWithNils [
+
+	self assert: #( 1 2 nil 4 ) asDataSeries sum equals: 7.
+	self assert: #( nil nil nil ) asDataSeries sum equals: 0
+]
+
 { #category : #'tests - head/tail' }
 DataSeriesTest >> testTail [
 	| expected actual |

src/DataFrame/DataFrame.class.st

@@ -1412,6 +1412,16 @@ DataFrame >> outerJoin: aDataFrame onLeft: leftColumn onRight: rightColumn [
 	^ outputDf
 ]
 
+{ #category : #copying }
+DataFrame >> postCopy [
+
+	super postCopy.
+	contents := contents copy.
+	rowNames := rowNames copy.
+	columnNames := columnNames copy.
+	dataTypes := dataTypes copy
+]
+
 { #category : #printing }
 DataFrame >> printOn: aStream [
 

src/DataFrame/DataSeries.class.st

@@ -160,6 +160,13 @@ DataSeries >> atIndex: aNumber transform: aBlock [
 	self at: key transform: aBlock
 ]
 
+{ #category : #information }
+DataSeries >> average [
+	"We do not count the nils"
+
+	^ (self values reject: #isNil) average
+]
+
 { #category : #'data-types' }
 DataSeries >> calculateDataType [
 
@@ -420,7 +427,8 @@ DataSeries >> isCategorical [
 
 { #category : #'categorical-numerical' }
 DataSeries >> isNumerical [
-	^ forcedIsNumerical ifNil: [self uniqueValues allSatisfy: [:each|each isNumber]]
+
+	^ forcedIsNumerical ifNil: [ (self uniqueValues copyWithout: nil) allSatisfy: [ :each | each isNumber ] ]
 ]
 
 { #category : #testing }
@@ -659,6 +667,16 @@ DataSeries >> sortedDescending [
 	^ self sorted: [ :a :b | a > b ]
 ]
 
+{ #category : #transformation }
+DataSeries >> sum [
+	"Return the sum of the values over the requested axis. Nil values are excluded."
+
+	| result |
+	result := 0.
+	self do: [ :each | each ifNotNil: [ result := result + each ] ].
+	^ result
+]
+
 { #category : #statistics }
 DataSeries >> summary [
 	| summary |