Skip to content

Commit 510cbcc

Browse files
committed
add note
1 parent f722f47 commit 510cbcc

File tree

2 files changed

+172
-8
lines changed

2 files changed

+172
-8
lines changed

notebooks/.ipynb_checkpoints/dask_2d_subset-checkpoint.ipynb

Lines changed: 86 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -755,12 +755,94 @@
755755
},
756756
{
757757
"cell_type": "code",
758-
"execution_count": null,
758+
"execution_count": 30,
759759
"metadata": {
760-
"collapsed": true
760+
"collapsed": false
761761
},
762-
"outputs": [],
763-
"source": []
762+
"outputs": [
763+
{
764+
"name": "stdout",
765+
"output_type": "stream",
766+
"text": [
767+
" 80055494 function calls (60052157 primitive calls) in 19.425 seconds\n",
768+
"\n",
769+
" Ordered by: internal time\n",
770+
"\n",
771+
" ncalls tottime percall cumtime percall filename:lineno(function)\n",
772+
"10001670/3 5.032 0.000 6.671 2.224 slicing.py:623(check_index)\n",
773+
" 1 3.459 3.459 4.272 4.272 slicing.py:398(partition_by_size)\n",
774+
"10001671/1 3.287 0.000 6.378 6.378 slicing.py:540(posify_index)\n",
775+
" 40006704 2.999 0.000 2.999 0.000 {built-in method builtins.isinstance}\n",
776+
" 1 1.731 1.731 6.378 6.378 slicing.py:563(<listcomp>)\n",
777+
" 1 0.849 0.849 0.849 0.849 slicing.py:44(<listcomp>)\n",
778+
" 10011685 0.433 0.000 0.433 0.000 {built-in method builtins.len}\n",
779+
" 10015670 0.381 0.000 0.381 0.000 {method 'append' of 'list' objects}\n",
780+
" 1 0.355 0.355 0.355 0.355 slicing.py:420(issorted)\n",
781+
" 1 0.196 0.196 0.196 0.196 {method 'tolist' of 'numpy.ndarray' objects}\n",
782+
" 1 0.193 0.193 0.193 0.193 slicing.py:479(<listcomp>)\n",
783+
" 1 0.157 0.157 0.157 0.157 {built-in method builtins.sorted}\n",
784+
" 1 0.085 0.085 4.707 4.707 slicing.py:441(take_sorted)\n",
785+
" 1 0.085 0.085 19.425 19.425 <string>:1(<module>)\n",
786+
" 1 0.079 0.079 19.341 19.341 core.py:1024(__getitem__)\n",
787+
" 1 0.034 0.034 18.157 18.157 slicing.py:142(slice_with_newaxes)\n",
788+
" 2 0.033 0.017 0.033 0.017 {built-in method _hashlib.openssl_md5}\n",
789+
" 1 0.026 0.026 1.071 1.071 slicing.py:15(sanitize_index)\n",
790+
" 4001 0.007 0.000 0.009 0.000 slicing.py:567(insert_many)\n",
791+
" 2001 0.002 0.000 0.011 0.000 slicing.py:156(<genexpr>)\n",
792+
" 12003 0.001 0.000 0.001 0.000 {method 'pop' of 'list' objects}\n",
793+
" 4 0.000 0.000 0.000 0.000 {method 'update' of 'dict' objects}\n",
794+
" 1 0.000 0.000 19.228 19.228 slicing.py:60(slice_array)\n",
795+
" 1 0.000 0.000 0.000 0.000 slicing.py:464(<listcomp>)\n",
796+
" 1 0.000 0.000 0.000 0.000 slicing.py:162(<genexpr>)\n",
797+
" 1 0.000 0.000 0.033 0.033 base.py:314(normalize_array)\n",
798+
" 1 0.000 0.000 18.111 18.111 slicing.py:170(slice_wrap_lists)\n",
799+
" 1 0.000 0.000 0.000 0.000 slicing.py:465(<listcomp>)\n",
800+
" 1 0.000 0.000 5.062 5.062 slicing.py:487(take)\n",
801+
" 1 0.000 0.000 0.033 0.033 base.py:343(tokenize)\n",
802+
" 1 0.000 0.000 19.425 19.425 {built-in method builtins.exec}\n",
803+
" 2 0.000 0.000 0.000 0.000 functoolz.py:217(__call__)\n",
804+
" 3 0.000 0.000 0.000 0.000 {built-in method builtins.sum}\n",
805+
" 2 0.000 0.000 0.000 0.000 abc.py:178(__instancecheck__)\n",
806+
" 1 0.000 0.000 0.000 0.000 core.py:1455(normalize_chunks)\n",
807+
" 2 0.000 0.000 0.000 0.000 dicttoolz.py:19(merge)\n",
808+
" 4 0.000 0.000 0.000 0.000 _weakrefset.py:70(__contains__)\n",
809+
" 2 0.000 0.000 0.000 0.000 dicttoolz.py:11(_get_factory)\n",
810+
" 1 0.000 0.000 0.000 0.000 exceptions.py:15(merge)\n",
811+
" 1 0.000 0.000 0.000 0.000 core.py:794(__init__)\n",
812+
" 4 0.000 0.000 0.000 0.000 {built-in method builtins.all}\n",
813+
" 1 0.000 0.000 0.000 0.000 slicing.py:467(<listcomp>)\n",
814+
" 1 0.000 0.000 0.000 0.000 {method 'view' of 'numpy.ndarray' objects}\n",
815+
" 4 0.000 0.000 0.000 0.000 slicing.py:183(<genexpr>)\n",
816+
" 2 0.000 0.000 0.000 0.000 {method 'hexdigest' of '_hashlib.HASH' objects}\n",
817+
" 1 0.000 0.000 0.000 0.000 slicing.py:606(replace_ellipsis)\n",
818+
" 1 0.000 0.000 0.000 0.000 slicing.py:192(<listcomp>)\n",
819+
" 4 0.000 0.000 0.000 0.000 slicing.py:207(<genexpr>)\n",
820+
" 1 0.000 0.000 0.000 0.000 slicing.py:475(<listcomp>)\n",
821+
" 2 0.000 0.000 0.033 0.017 utils.py:502(__call__)\n",
822+
" 2 0.000 0.000 0.000 0.000 slicing.py:125(<genexpr>)\n",
823+
" 2 0.000 0.000 0.000 0.000 core.py:1043(<genexpr>)\n",
824+
" 4 0.000 0.000 0.000 0.000 slicing.py:197(<genexpr>)\n",
825+
" 1 0.000 0.000 0.000 0.000 core.py:826(_get_chunks)\n",
826+
" 2 0.000 0.000 0.000 0.000 {built-in method builtins.hasattr}\n",
827+
" 1 0.000 0.000 0.000 0.000 {method 'ravel' of 'numpy.ndarray' objects}\n",
828+
" 1 0.000 0.000 0.000 0.000 {method 'disable' of '_lsprof.Profiler' objects}\n",
829+
" 1 0.000 0.000 0.000 0.000 {method 'encode' of 'str' objects}\n",
830+
" 1 0.000 0.000 0.000 0.000 slicing.py:613(<listcomp>)\n",
831+
" 1 0.000 0.000 0.000 0.000 core.py:1452(<lambda>)\n",
832+
" 1 0.000 0.000 0.000 0.000 slicing.py:149(<listcomp>)\n",
833+
" 2 0.000 0.000 0.000 0.000 {method 'pop' of 'dict' objects}\n",
834+
" 2 0.000 0.000 0.000 0.000 {method 'items' of 'dict' objects}\n",
835+
" 1 0.000 0.000 0.000 0.000 slicing.py:150(<listcomp>)\n",
836+
" 1 0.000 0.000 0.000 0.000 {method 'count' of 'tuple' objects}\n",
837+
"\n",
838+
"\n"
839+
]
840+
}
841+
],
842+
"source": [
843+
"# problem is in fact just the dim0 selection\n",
844+
"cProfile.run('d[c]', sort='time')"
845+
]
764846
}
765847
],
766848
"metadata": {

notebooks/dask_2d_subset.ipynb

Lines changed: 86 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -755,12 +755,94 @@
755755
},
756756
{
757757
"cell_type": "code",
758-
"execution_count": null,
758+
"execution_count": 30,
759759
"metadata": {
760-
"collapsed": true
760+
"collapsed": false
761761
},
762-
"outputs": [],
763-
"source": []
762+
"outputs": [
763+
{
764+
"name": "stdout",
765+
"output_type": "stream",
766+
"text": [
767+
" 80055494 function calls (60052157 primitive calls) in 19.425 seconds\n",
768+
"\n",
769+
" Ordered by: internal time\n",
770+
"\n",
771+
" ncalls tottime percall cumtime percall filename:lineno(function)\n",
772+
"10001670/3 5.032 0.000 6.671 2.224 slicing.py:623(check_index)\n",
773+
" 1 3.459 3.459 4.272 4.272 slicing.py:398(partition_by_size)\n",
774+
"10001671/1 3.287 0.000 6.378 6.378 slicing.py:540(posify_index)\n",
775+
" 40006704 2.999 0.000 2.999 0.000 {built-in method builtins.isinstance}\n",
776+
" 1 1.731 1.731 6.378 6.378 slicing.py:563(<listcomp>)\n",
777+
" 1 0.849 0.849 0.849 0.849 slicing.py:44(<listcomp>)\n",
778+
" 10011685 0.433 0.000 0.433 0.000 {built-in method builtins.len}\n",
779+
" 10015670 0.381 0.000 0.381 0.000 {method 'append' of 'list' objects}\n",
780+
" 1 0.355 0.355 0.355 0.355 slicing.py:420(issorted)\n",
781+
" 1 0.196 0.196 0.196 0.196 {method 'tolist' of 'numpy.ndarray' objects}\n",
782+
" 1 0.193 0.193 0.193 0.193 slicing.py:479(<listcomp>)\n",
783+
" 1 0.157 0.157 0.157 0.157 {built-in method builtins.sorted}\n",
784+
" 1 0.085 0.085 4.707 4.707 slicing.py:441(take_sorted)\n",
785+
" 1 0.085 0.085 19.425 19.425 <string>:1(<module>)\n",
786+
" 1 0.079 0.079 19.341 19.341 core.py:1024(__getitem__)\n",
787+
" 1 0.034 0.034 18.157 18.157 slicing.py:142(slice_with_newaxes)\n",
788+
" 2 0.033 0.017 0.033 0.017 {built-in method _hashlib.openssl_md5}\n",
789+
" 1 0.026 0.026 1.071 1.071 slicing.py:15(sanitize_index)\n",
790+
" 4001 0.007 0.000 0.009 0.000 slicing.py:567(insert_many)\n",
791+
" 2001 0.002 0.000 0.011 0.000 slicing.py:156(<genexpr>)\n",
792+
" 12003 0.001 0.000 0.001 0.000 {method 'pop' of 'list' objects}\n",
793+
" 4 0.000 0.000 0.000 0.000 {method 'update' of 'dict' objects}\n",
794+
" 1 0.000 0.000 19.228 19.228 slicing.py:60(slice_array)\n",
795+
" 1 0.000 0.000 0.000 0.000 slicing.py:464(<listcomp>)\n",
796+
" 1 0.000 0.000 0.000 0.000 slicing.py:162(<genexpr>)\n",
797+
" 1 0.000 0.000 0.033 0.033 base.py:314(normalize_array)\n",
798+
" 1 0.000 0.000 18.111 18.111 slicing.py:170(slice_wrap_lists)\n",
799+
" 1 0.000 0.000 0.000 0.000 slicing.py:465(<listcomp>)\n",
800+
" 1 0.000 0.000 5.062 5.062 slicing.py:487(take)\n",
801+
" 1 0.000 0.000 0.033 0.033 base.py:343(tokenize)\n",
802+
" 1 0.000 0.000 19.425 19.425 {built-in method builtins.exec}\n",
803+
" 2 0.000 0.000 0.000 0.000 functoolz.py:217(__call__)\n",
804+
" 3 0.000 0.000 0.000 0.000 {built-in method builtins.sum}\n",
805+
" 2 0.000 0.000 0.000 0.000 abc.py:178(__instancecheck__)\n",
806+
" 1 0.000 0.000 0.000 0.000 core.py:1455(normalize_chunks)\n",
807+
" 2 0.000 0.000 0.000 0.000 dicttoolz.py:19(merge)\n",
808+
" 4 0.000 0.000 0.000 0.000 _weakrefset.py:70(__contains__)\n",
809+
" 2 0.000 0.000 0.000 0.000 dicttoolz.py:11(_get_factory)\n",
810+
" 1 0.000 0.000 0.000 0.000 exceptions.py:15(merge)\n",
811+
" 1 0.000 0.000 0.000 0.000 core.py:794(__init__)\n",
812+
" 4 0.000 0.000 0.000 0.000 {built-in method builtins.all}\n",
813+
" 1 0.000 0.000 0.000 0.000 slicing.py:467(<listcomp>)\n",
814+
" 1 0.000 0.000 0.000 0.000 {method 'view' of 'numpy.ndarray' objects}\n",
815+
" 4 0.000 0.000 0.000 0.000 slicing.py:183(<genexpr>)\n",
816+
" 2 0.000 0.000 0.000 0.000 {method 'hexdigest' of '_hashlib.HASH' objects}\n",
817+
" 1 0.000 0.000 0.000 0.000 slicing.py:606(replace_ellipsis)\n",
818+
" 1 0.000 0.000 0.000 0.000 slicing.py:192(<listcomp>)\n",
819+
" 4 0.000 0.000 0.000 0.000 slicing.py:207(<genexpr>)\n",
820+
" 1 0.000 0.000 0.000 0.000 slicing.py:475(<listcomp>)\n",
821+
" 2 0.000 0.000 0.033 0.017 utils.py:502(__call__)\n",
822+
" 2 0.000 0.000 0.000 0.000 slicing.py:125(<genexpr>)\n",
823+
" 2 0.000 0.000 0.000 0.000 core.py:1043(<genexpr>)\n",
824+
" 4 0.000 0.000 0.000 0.000 slicing.py:197(<genexpr>)\n",
825+
" 1 0.000 0.000 0.000 0.000 core.py:826(_get_chunks)\n",
826+
" 2 0.000 0.000 0.000 0.000 {built-in method builtins.hasattr}\n",
827+
" 1 0.000 0.000 0.000 0.000 {method 'ravel' of 'numpy.ndarray' objects}\n",
828+
" 1 0.000 0.000 0.000 0.000 {method 'disable' of '_lsprof.Profiler' objects}\n",
829+
" 1 0.000 0.000 0.000 0.000 {method 'encode' of 'str' objects}\n",
830+
" 1 0.000 0.000 0.000 0.000 slicing.py:613(<listcomp>)\n",
831+
" 1 0.000 0.000 0.000 0.000 core.py:1452(<lambda>)\n",
832+
" 1 0.000 0.000 0.000 0.000 slicing.py:149(<listcomp>)\n",
833+
" 2 0.000 0.000 0.000 0.000 {method 'pop' of 'dict' objects}\n",
834+
" 2 0.000 0.000 0.000 0.000 {method 'items' of 'dict' objects}\n",
835+
" 1 0.000 0.000 0.000 0.000 slicing.py:150(<listcomp>)\n",
836+
" 1 0.000 0.000 0.000 0.000 {method 'count' of 'tuple' objects}\n",
837+
"\n",
838+
"\n"
839+
]
840+
}
841+
],
842+
"source": [
843+
"# problem is in fact just the dim0 selection\n",
844+
"cProfile.run('d[c]', sort='time')"
845+
]
764846
}
765847
],
766848
"metadata": {

0 commit comments

Comments
 (0)