change url to local mirroring

Author: asahi417

benchmark/agreement_all.heatmap.pdf

@@ -1,16 +1,16 @@
-,domain,type,data size,avg word number,avg vocabulary size,avg keyword number,avg keyword number (tractable only),avg keyword number (tractable only and multiwords),avg vocabulary diversity
-citeulike180,BI,Full,183,3799.2677595628415,1023.1857923497267,16.469945355191257,7.775956284153006,1.0546448087431695,0.2693113139431527
-fao30,AG,Full,30,3672.7,983.3666666666667,31.933333333333334,15.9,5.466666666666667,0.26775033808006826
-fao780,AG,Full,779,3567.3478818998715,942.2387676508343,7.961489088575096,4.193838254172015,1.5571245186136071,0.26412864650279744
-Inspec,CS,Abst,2000,85.3395,53.0095,13.4925,5.836,4.811,0.6211601895956738
-kdd,CS,Abst,755,52.688741721854306,33.71920529801324,4.079470198675497,0.7456953642384105,0.5576158940397351,0.6399698340874811
-Krapivin2009,CS,Full,2304,5451.236545138889,902.3663194444445,5.329861111111111,3.8094618055555554,2.9479166666666665,0.16553424383117712
-Nguyen2007,-,Full,209,3656.186602870813,752.933014354067,10.736842105263158,7.177033492822966,4.760765550239235,0.20593396785680168
-PubMed,BM,Full,500,2958.59,682.442,16.142,5.682,1.512,0.23066460712704362
-Schutz2008,BM,Full,1231,3207.568643379366,1101.2908204711616,43.56539398862713,28.499593826157597,10.148659626320065,0.34334131016784275
-SemEval2010,CS,Full,243,6221.6378600823045,1037.1851851851852,15.209876543209877,11.567901234567902,8.77366255144033,0.16670613245423843
-SemEval2017,-,Para,493,126.50507099391481,77.13590263691684,17.11764705882353,9.27789046653144,6.320486815415822,0.6097455385059406
-theses100,-,Full,100,3635.28,984.87,6.78,2.39,0.75,0.2709199841552783
-wiki20,CS,Report,20,4489.65,923.95,35.15,12.75,6.65,0.20579555199180338
-www,CS,Abst,1330,57.50375939849624,36.47744360902256,4.8007518796992485,0.8842105263157894,0.5428571428571428,0.6343488493723849
-500N-KPCrowd-v1.1,-,News,500,269.696,144.12,47.978,16.49,3.74,0.5343794494542002
+,500N-KPCrowd-v1.1,Inspec,Krapivin2009,Nguyen2007,PubMed,Schutz2008,SemEval2010,SemEval2017,citeulike180,fao30,fao780,theses100,kdd,wiki20,www
+Data size,500,2000,2304,209,500,1231,243,493,183,30,779,100,755,20,1330
+Domain,-,CS,CS,-,BM,BM,CS,-,BI,AG,AG,-,CS,CS,CS
+Type,-,CS,CS,-,BM,BM,CS,-,BI,AG,AG,-,CS,CS,CS
+Avg phrase,77.384,26.967,814.49609375,617.9856459330143,566.298,630.2558895207148,897.8312757201646,39.699797160243406,822.2677595628415,774.0,776.2708600770218,728.36,15.954966887417218,816.7,17.9
+Std phrase,62.00755231421413,12.314946650310784,252.14317468513156,113.86454129909198,196.48920885381975,287.6595497592065,207.70384981408176,12.94657221855953,173.00291983671727,93.19942775217739,147.1711422801908,131.26046777305035,17.04914862119257,322.3617688250268,16.517061126928027
+Avg word,447.302,137.762,9130.471788194445,5931.397129186603,4461.298,4200.692120227457,9740.218106995884,198.01825557809332,5521.131147540984,5437.833333333333,5590.523748395379,5396.72,81.93774834437086,7145.55,90.62781954887218
+Std word,476.7283406679322,66.55563354067026,2524.422828713077,1023.0662095302862,1626.42691849219,2251.1355744927955,2443.441728214183,60.297350338210734,978.82878840376,927.5287626567466,902.3992981704127,958.4219747063399,93.01618511994454,3609.7554692111767,89.12352992882947
+Avg vocab,197.14,76.232,1081.1414930555557,908.4784688995215,799.822,1217.1722177091794,1217.9053497942386,106.12576064908723,1171.120218579235,1125.0666666666666,1087.3196405648266,1134.03,48.10728476821192,1088.4,52.74661654135338
+Std vocab,140.59065545049575,28.59461795513273,256.2237773449945,142.4035332525959,223.72044679912463,468.37048133960053,209.0611466958877,27.475683619235635,202.9375439628631,157.13941439230183,210.26987796235758,192.34289459192405,45.666634100960046,295.40470544661264,43.33231971821068
+Avg keyword,47.978,13.4925,5.329861111111111,10.736842105263158,16.142,43.56539398862713,15.209876543209877,17.11764705882353,16.469945355191257,31.933333333333334,7.961489088575096,6.78,4.079470198675497,35.15,4.8007518796992485
+Avg keyword (in candidate),16.49,5.836,3.8094618055555554,7.177033492822966,5.682,28.499593826157597,11.567901234567902,9.27789046653144,7.775956284153006,15.9,4.193838254172015,2.39,0.7456953642384105,12.75,0.8842105263157894
+Std keyword (in candidate),11.971545430728654,3.540353654650908,2.146071359435905,4.258652751428725,2.6714932154134297,10.253637574384731,3.347887749806898,4.866770425384573,3.413100931015965,5.61159513863928,2.3493725225755795,1.5420440979427268,0.9272715155165945,3.1760824926314495,0.9818535567010371
+Avg keyword (in candidate & multi),3.74,4.811,2.9479166666666665,4.760765550239235,1.512,10.148659626320065,8.77366255144033,6.320486815415822,1.0546448087431695,5.466666666666667,1.5571245186136071,0.75,0.5576158940397351,6.65,0.5428571428571428
+Std keyword (in candidate & multi),3.778941650780024,3.1664300087006554,1.9007205779066374,3.2148268033683105,1.3303593499502333,4.928390150828587,3.285177736220138,3.4309659794374823,1.0012384460016208,2.6042699979499475,1.34546638502203,0.8170067319184096,0.763612630732103,2.650943228362313,0.7499659856232418
+Vocab diversity,2.268956071827128,1.8071413579599118,8.445214476404585,6.528935282716778,5.577863574645359,3.4511896173028935,7.997516480768776,1.8658830275229359,4.714401698434547,4.833343209291301,5.1415642096663365,4.758886449212102,1.7032295366317007,6.565187431091511,1.7181731358601913

benchmark/agreement_all.heatmap.png

@@ -1,16 +1,16 @@
 ,FirstN,TF,LexSpec,TFIDF,TextRank,SingleRank,PositionRank,LexRank,TFIDFRank,SingleTPR,TopicRank
-500N-KPCrowd-v1.1,24.7,17.2,23.7,23.9,18.1,18.4,19.8,19.6,19.8,17.2,24.7
-Inspec,32.9,22.0,34.3,34.5,37.2,37.5,35.8,36.5,36.6,33.4,32.9
-Krapivin2009,19.5,0.1,9.9,8.5,7.2,9.9,15.5,10.8,10.6,8.1,10.1
-Nguyen2007,16.5,0.7,14.8,13.9,13.0,15.8,18.2,16.3,16.5,13.7,13.9
-PubMed,10.0,4.2,6.9,6.3,9.9,10.6,10.1,8.4,8.2,9.0,8.5
-Schutz2008,7.5,2.2,14.6,14.5,13.2,14.1,8.9,14.7,14.9,6.6,21.3
-SemEval2010,11.1,0.9,10.0,9.8,10.1,12.7,16.1,12.2,12.4,10.0,13.3
-SemEval2017,28.9,19.2,42.4,43.1,39.3,40.1,38.4,42.5,42.5,33.8,31.7
-citeulike180,6.5,8.2,13.9,11.9,17.5,18.4,16.7,16.9,18.0,17.8,14.0
-fao30,11.4,9.9,12.0,10.7,15.9,17.4,14.8,15.6,16.4,18.2,14.0
-fao780,10.2,3.6,13.3,11.3,13.6,15.3,14.7,14.8,14.5,15.8,13.9
-kdd,35.2,10.1,32.3,33.7,29.3,32.1,33.9,34.8,36.8,16.5,33.9
-theses100,7.1,0.8,15.9,15.1,9.2,13.0,13.8,15.5,14.6,12.1,12.6
-wiki20,11.6,8.0,10.7,10.7,10.7,13.4,11.2,12.5,13.4,12.9,14.3
-www,32.2,14.0,32.1,33.2,26.9,27.9,32.7,27.8,31.0,17.9,30.9
+500N-KPCrowd-v1.1,24.708624708624708,17.249417249417252,23.679768507354716,23.872679045092838,18.053211156659433,18.406880475845995,19.83763363073708,19.580419580419576,19.78940599630255,17.152961980548188,24.67647295233502
+Inspec,32.854498986516255,21.99700361328986,34.34387943949943,34.546576187538555,37.17282100995858,37.5429628976822,35.76275667577333,36.48541464704327,36.63523398255046,33.40089891601304,32.925002203225525
+Krapivin2009,19.45288753799392,0.12616849228651716,9.875551987153752,8.522108160807479,7.203073923266616,9.932901301829443,15.518724551241611,10.804610884899924,10.575213626197169,8.109193095142512,10.070539657051098
+Nguyen2007,16.5266106442577,0.7002801120448179,14.775910364145659,13.865546218487395,12.955182072829134,15.756302521008402,18.207282913165265,16.316526610644257,16.5266106442577,13.65546218487395,13.935574229691877
+PubMed,10.02834868887314,4.216867469879518,6.874557051736358,6.27214741318214,9.922041105598865,10.559886605244508,10.063784549964563,8.433734939759036,8.22111977321049,8.965272856130401,8.46917080085046
+Schutz2008,7.512354450783685,2.1633544719930433,14.600521750196185,14.485991219325967,13.192220407643854,14.087255297037052,8.94186515090458,14.69808479501156,14.850792169505187,6.638528918959044,21.340855585484313
+SemEval2010,11.095917171322526,0.8595428794686462,10.001953506544247,9.806602852119553,10.119163899199062,12.697792537604998,16.05782379370971,12.189880836100802,12.385231490525495,10.001953506544247,13.28384450087908
+SemEval2017,28.9199856476498,19.23214926444205,42.38727424949168,43.08097117569669,39.3254395407248,40.06697763425428,38.41645736155962,42.53079775146514,42.48295658414065,33.847625882071526,31.694773352469802
+citeulike180,6.547835576573299,8.22117133503092,13.89596216806111,11.858857766460531,17.53364859949072,18.40669334303383,16.660603855947617,16.878865041833393,17.97017097126228,17.824663514005092,13.968715896689703
+fao30,11.443433029908972,9.882964889466841,11.963589076723018,10.663198959687907,15.864759427828352,17.425227568270483,14.824447334200261,15.604681404421324,16.38491547464239,18.20546163849155,14.044213263979193
+fao780,10.18092609628948,3.5878564857405713,13.30880098129408,11.315547378104876,13.61545538178473,15.332720024532351,14.688745783501991,14.842072983747315,14.504753143207608,15.823367065317386,13.922109782275378
+kdd,35.168738898756665,10.124333925399645,32.326820603907635,33.747779751332146,29.307282415630553,32.149200710479576,33.92539964476021,34.81349911190053,36.767317939609235,16.518650088809945,33.92539964476021
+theses100,7.112970711297072,0.8368200836820083,15.899581589958158,15.062761506276152,9.205020920502092,12.97071129707113,13.807531380753138,15.481171548117153,14.644351464435147,12.133891213389122,12.552301255230125
+wiki20,11.607142857142858,8.035714285714285,10.714285714285714,10.714285714285714,10.714285714285714,13.392857142857142,11.160714285714283,12.499999999999996,13.392857142857142,12.946428571428573,14.285714285714283
+www,32.22789115646258,14.030612244897958,32.0578231292517,33.16326530612245,26.87074829931973,27.89115646258503,32.6530612244898,27.806122448979593,30.952380952380953,17.857142857142858,30.86734693877551

benchmark/data_statistics.csv

@@ -1,16 +1,16 @@
 ,FirstN,TF,LexSpec,TFIDF,TextRank,SingleRank,PositionRank,LexRank,TFIDFRank,SingleTPR,TopicRank
-500N-KPCrowd-v1.1,25.3,18.0,24.3,24.4,18.8,19.0,20.5,20.4,20.6,18.0,24.9
-Inspec,33.7,26.4,36.3,37.1,39.7,40.0,38.3,38.9,39.4,35.9,31.8
-Krapivin2009,15.6,0.1,8.7,7.8,8.0,10.3,14.5,10.2,10.3,8.6,10.1
-Nguyen2007,16.3,0.7,13.9,12.8,12.3,15.0,17.9,15.6,15.4,13.8,13.8
-PubMed,10.5,4.5,6.5,6.1,9.2,9.3,9.7,7.8,7.5,8.3,8.7
-Schutz2008,7.5,2.2,14.6,14.5,13.2,14.1,9.0,14.7,14.9,6.7,21.3
-SemEval2010,11.2,0.8,10.2,9.8,10.2,12.8,16.0,12.1,12.4,10.2,13.4
-SemEval2017,29.1,21.1,41.9,42.7,40.3,40.7,39.4,42.2,42.5,35.0,30.7
-citeulike180,6.2,8.6,13.0,11.2,16.2,17.9,16.4,16.4,17.5,16.8,13.4
-fao30,11.8,9.8,11.8,10.6,15.7,17.2,14.7,15.7,16.2,18.3,13.9
-fao780,9.5,4.9,10.0,8.6,12.2,13.1,13.4,12.3,12.1,13.8,12.2
-kdd,18.6,14.7,20.8,21.6,19.9,20.9,21.7,21.0,21.3,19.0,17.0
-theses100,5.6,1.2,10.4,10.0,7.6,9.7,10.7,10.7,10.2,9.1,9.0
-wiki20,11.4,8.4,10.5,10.5,10.5,13.2,11.0,12.3,13.2,13.6,14.1
-www,18.6,14.4,20.2,20.4,18.8,19.3,21.0,19.8,19.9,18.6,17.1
+500N-KPCrowd-v1.1,25.268776210446052,18.04041174227983,24.26229508196721,24.369043080442243,18.75714830346931,19.046892870758676,20.526115135341215,20.44986656500191,20.63286313381624,17.99466260007625,24.887533358749522
+Inspec,33.656854606985505,26.384011238107398,36.332290402911696,37.08575442181215,39.73564906455526,40.02298703786475,38.305344486303554,38.931102739288676,39.352531766809264,35.9300172402784,31.84981801928358
+Krapivin2009,15.639915289060276,0.13275595031134432,8.736605872870374,7.78202737301261,7.952713594841483,10.335998988526095,14.520972279293234,10.2285298858931,10.291747005088977,8.641780194076555,10.070487087903404
+Nguyen2007,16.292134831460675,0.6741573033707865,13.93258426966292,12.752808988764045,12.303370786516853,15.0,17.865168539325847,15.561797752808989,15.449438202247192,13.820224719101123,13.764044943820226
+PubMed,10.51278755943966,4.549543760442103,6.528723814419742,6.066058347256138,9.201902069142784,9.304716617401363,9.690271173371032,7.8396093047166175,7.505462022876237,8.276571134815576,8.713532964914535
+Schutz2008,7.496359453812548,2.165333558449233,14.608614904079523,14.50731274929827,13.181942890910245,14.106325053289154,8.973682544372455,14.714137981976657,14.857649367916764,6.660616676867232,21.294557119642064
+SemEval2010,11.181072314443808,0.8395344399923679,10.188895248998282,9.769128029002099,10.188895248998282,12.783819881701966,15.989315016218278,12.096928067162754,12.440373974432362,10.150734592634993,13.356229727151309
+SemEval2017,29.14382117250105,21.067060312104598,41.94432728806411,42.74567692956559,40.278363559679455,40.70012652889076,39.41374947279628,42.19738506959089,42.47153099957824,34.985238296077604,30.746520455504005
+citeulike180,6.167129201356769,8.572309589885908,13.012642614862783,11.22417514646932,16.2195497995683,17.946345975948198,16.404563675609005,16.404563675609005,17.51464693185322,16.836262719703978,13.444341658957756
+fao30,11.840411840411841,9.78120978120978,11.840411840411841,10.553410553410554,15.7014157014157,17.245817245817246,14.671814671814671,15.7014157014157,16.216216216216218,18.275418275418275,13.8996138996139
+fao780,9.477274822531575,4.886143634184568,10.030423158476998,8.647552318613442,12.187701668664147,13.128053839771367,13.38618972987923,12.279893057988385,12.09551027933991,13.791831842905871,12.24301650225869
+kdd,18.594945513563648,14.6533735219105,20.820774402967775,21.56271736610248,19.893345699049387,20.91351727335961,21.70183167169024,21.006260143751447,21.284488754926965,19.01228843032692,17.01831671690239
+theses100,5.618964003511852,1.2291483757682176,10.35996488147498,10.008779631255488,7.550482879719053,9.657594381035997,10.711150131694469,10.711150131694469,10.184372256365231,9.13081650570676,8.955223880597014
+wiki20,11.428571428571429,8.351648351648352,10.54945054945055,10.54945054945055,10.54945054945055,13.186813186813188,10.98901098901099,12.307692307692308,13.186813186813188,13.626373626373628,14.065934065934066
+www,18.56853477380149,14.38217420661715,20.166554130092283,20.391627278865627,18.771100607697498,19.288768849876206,20.99932478055368,19.806437092054914,19.91897366644159,18.56853477380149,17.060544677020033

benchmark/result.10.f1.fixed.csv

@@ -1,16 +1,16 @@
 ,FirstN,TF,LexSpec,TFIDF,TextRank,SingleRank,PositionRank,LexRank,TFIDFRank,SingleTPR,TopicRank
-500N-KPCrowd-v1.1,36.6,25.6,35.1,35.4,26.8,27.3,29.4,29.0,29.3,25.4,36.6
-Inspec,33.8,22.6,35.4,35.6,38.3,38.6,36.8,37.6,37.7,34.4,33.9
-Krapivin2009,19.6,0.1,9.9,8.6,7.3,10.0,15.6,10.9,10.6,8.2,10.1
-Nguyen2007,17.4,0.7,15.6,14.6,13.6,16.6,19.2,17.2,17.4,14.4,14.7
-PubMed,10.1,4.2,6.9,6.3,10.0,10.6,10.1,8.5,8.3,9.0,8.5
-Schutz2008,14.7,4.2,28.5,28.3,25.8,27.5,17.5,28.7,29.0,13.0,41.7
-SemEval2010,12.3,1.0,11.1,10.9,11.2,14.1,17.8,13.5,13.7,11.1,14.7
-SemEval2017,31.9,21.2,46.8,47.6,43.4,44.2,42.4,47.0,46.9,37.4,35.0
-citeulike180,6.8,8.5,14.4,12.3,18.2,19.1,17.3,17.5,18.6,18.5,14.5
-fao30,15.1,13.0,15.8,14.0,20.9,22.9,19.5,20.5,21.6,24.0,18.5
-fao780,10.2,3.6,13.3,11.3,13.6,15.4,14.7,14.9,14.5,15.9,13.9
-kdd,35.2,10.1,32.3,33.7,29.3,32.1,33.9,34.8,36.8,16.5,33.9
-theses100,7.1,0.8,15.9,15.1,9.2,13.0,13.8,15.5,14.6,12.1,12.6
-wiki20,13.5,9.3,12.4,12.4,12.4,15.5,13.0,14.5,15.5,15.0,16.6
-www,32.2,14.0,32.1,33.2,26.9,27.9,32.7,27.8,31.0,17.9,30.9
+500N-KPCrowd-v1.1,36.630123927550045,25.571973307912298,35.10486177311725,35.3908484270734,26.763584366062915,27.28789323164919,29.408960915157294,29.027645376549096,29.337464251668255,25.428979980934223,36.58245948522402
+Inspec,33.82326256577753,22.645617855198694,35.356559608056614,35.56523317002359,38.26891671203048,38.64997278170931,36.81727454182544,37.56124115405552,37.71547813463981,34.38577390673199,33.89584467428779
+Krapivin2009,19.584295612009235,0.12702078521939955,9.942263279445728,8.579676674364896,7.2517321016166285,10.0,15.623556581986142,10.877598152424943,10.646651270207851,8.163972286374134,10.138568129330254
+Nguyen2007,17.404129793510325,0.7374631268436578,15.56047197640118,14.601769911504425,13.643067846607671,16.5929203539823,19.174041297935105,17.182890855457227,17.404129793510325,14.380530973451327,14.67551622418879
+PubMed,10.096325365679629,4.24545130217624,6.921155904388156,6.314662861220121,9.989297181591153,10.631466286122013,10.132001427042454,8.49090260435248,8.276846236175526,9.026043524794861,8.526578665715306
+Schutz2008,14.677606497596551,4.226752859273994,28.52643792474722,28.302668655726837,25.7749046908669,27.523620089507705,17.470578484999173,28.717056190949776,29.01541521631029,12.970329852478038,41.69567379413227
+SemEval2010,12.305025996533796,0.9532062391681109,11.091854419410744,10.875216637781628,11.221837088388215,14.081455805892547,17.807625649913344,13.518197573656845,13.734835355285963,11.091854419410744,14.731369150779896
+SemEval2017,31.925006601531553,21.2305254819118,46.791655664114074,47.55743332453129,43.411671507789805,44.23026142064959,42.40823871138104,46.950092421441774,46.8972801689992,37.36466860311592,34.98811724320042
+citeulike180,6.787330316742081,8.521870286576169,14.404223227752638,12.29260935143288,18.17496229260935,19.079939668174962,17.26998491704374,17.49622926093514,18.627450980392158,18.47662141779789,14.479638009049776
+fao30,15.068493150684931,13.013698630136986,15.753424657534246,14.04109589041096,20.89041095890411,22.945205479452056,19.52054794520548,20.54794520547945,21.575342465753426,23.972602739726025,18.493150684931507
+fao780,10.199692780337942,3.5944700460829497,13.333333333333334,11.336405529953918,13.640552995391705,15.360983102918588,14.715821812596005,14.869431643625191,14.531490015360985,15.85253456221198,13.947772657450077
+kdd,35.168738898756665,10.124333925399645,32.326820603907635,33.747779751332146,29.307282415630553,32.149200710479576,33.92539964476021,34.81349911190053,36.767317939609235,16.518650088809945,33.92539964476021
+theses100,7.112970711297072,0.8368200836820083,15.899581589958158,15.062761506276152,9.205020920502092,12.97071129707113,13.807531380753138,15.481171548117153,14.644351464435147,12.133891213389122,12.552301255230125
+wiki20,13.471502590673575,9.32642487046632,12.435233160621761,12.435233160621761,12.435233160621761,15.544041450777202,12.953367875647666,14.507772020725387,15.544041450777202,15.025906735751295,16.580310880829018
+www,32.22789115646258,14.030612244897958,32.0578231292517,33.16326530612245,26.87074829931973,27.89115646258503,32.6530612244898,27.806122448979593,30.952380952380953,17.857142857142858,30.86734693877551