@@ -72,51 +72,6 @@ def split_dense_variable(var_list,
72
72
return blocks
73
73
74
74
75
- def split_selected_rows (var ,
76
- pserver_count ,
77
- min_block_size = 1024 ,
78
- max_block_size = 1048576 ):
79
- assert ((len (var .shape )) <= 1 )
80
-
81
- split_count = pserver_count
82
- indices = var .desc .selected_rows ().dims ()
83
- var_width = reduce (lambda x , y : x * y , var .shape [1 :])
84
- row_count = len (indices )
85
- rows_per_block = 1
86
- if var_width < min_block_size :
87
- rows_per_block = 1
88
- split_count = row_count
89
- else :
90
- rows_per_block = row_count / pserver_count
91
- if not rows_per_block % pserver_count :
92
- rows_per_block += 1
93
- split_count = row_count / rows_per_block
94
- if not row_count % rows_per_block :
95
- split_count += 1
96
- blocks = []
97
- for block_id in xrange (split_count ):
98
- curr_block_rows = min (rows_per_block ,
99
- row_count - (block_id * rows_per_block ))
100
- block = VarBlock (var .name , block_id , curr_block_rows )
101
- blocks .append (block )
102
- return blocks
103
-
104
-
105
- def split_variable (var_list ,
106
- pserver_count ,
107
- min_block_size = 1024 ,
108
- max_block_size = 1048576 ):
109
- for var in var_list :
110
- if var .type == core .VarDesc .VarType .LOD_TENSOR :
111
- split_dense_variable (var_list , pserver_count , min_block_size ,
112
- max_block_size )
113
- elif var .type == core .VarDesc .VarType .SELECTED_ROWS :
114
- split_selected_rows (var_list , pserver_count , min_block_size ,
115
- max_block_size )
116
- else :
117
- raise TypeError ("variable must be lodtensor or selected rows" )
118
-
119
-
120
75
class DistributeTranspiler :
121
76
def transpile (self ,
122
77
optimize_ops ,
0 commit comments