Skip to content

Commit a4a1e03

Browse files
NexesenexNexesenex
committed
Merge branch 'concedo_experimental' into croco_nex
2 parents 13c6690 + 4ae06b4 commit a4a1e03

File tree

6 files changed

+434
-46
lines changed

6 files changed

+434
-46
lines changed

‎.github/workflows/kcpp-build-release-win-full-cu12.yaml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,15 @@ jobs:
3333
run: |
3434
echo "$(Get-Location)\w64devkit\bin" | Out-File -Append -FilePath $env:GITHUB_PATH -Encoding utf8
3535
36+
- name: Print System Environment Variables
37+
id: printvars
38+
run: |
39+
echo "Number of processors: $NUMBER_OF_PROCESSORS"
40+
echo "Processor Architecture: $PROCESSOR_ARCHITECTURE"
41+
echo "Computer Name: $COMPUTERNAME"
42+
wmic cpu get name
43+
wmic os get TotalVisibleMemorySize, FreePhysicalMemory
44+
3645
- name: Build Non-CUDA
3746
id: make_build
3847
run: |

‎.github/workflows/kcpp-build-release-win-full.yaml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,15 @@ jobs:
3333
run: |
3434
echo "$(Get-Location)\w64devkit\bin" | Out-File -Append -FilePath $env:GITHUB_PATH -Encoding utf8
3535
36+
- name: Print System Environment Variables
37+
id: printvars
38+
run: |
39+
echo "Number of processors: $NUMBER_OF_PROCESSORS"
40+
echo "Processor Architecture: $PROCESSOR_ARCHITECTURE"
41+
echo "Computer Name: $COMPUTERNAME"
42+
wmic cpu get name
43+
wmic os get TotalVisibleMemorySize, FreePhysicalMemory
44+
3645
- name: Build Non-CUDA
3746
id: make_build
3847
run: |

‎gpttype_adapter.cpp‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -450,6 +450,15 @@ void ContextRewind(std::vector<int> &embd, std::vector<int> &current_context_tok
450450
last_n_tokens.resize(last_n_tokens.size() - amount_rewind);
451451
}
452452

453+
if(amount_rewind >= top_picks_history.size())
454+
{
455+
top_picks_history.clear();
456+
}
457+
else
458+
{
459+
top_picks_history.resize(top_picks_history.size() - amount_rewind);
460+
}
461+
453462
if (amount_rewind >= current_context_tokens.size())
454463
{
455464
current_context_tokens.clear();

‎kcpp_docs.embd‎

Lines changed: 214 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -257,6 +257,11 @@
257257
"minimum": 0,
258258
"type": "number"
259259
},
260+
"logprobs": {
261+
"default": false,
262+
"description": "If true, return up to 5 top logprobs for generated tokens. Incurs performance overhead.",
263+
"type": "boolean"
264+
},
260265
},
261266
"required": [
262267
"prompt"
@@ -808,6 +813,215 @@
808813
]
809814
}
810815
},
816+
"/api/extra/last_logprobs": {
817+
"post": {
818+
"description": "Obtains the token logprobs of the most recent request. A unique genkey previously submitted is required in multiuser mode.",
819+
"requestBody": {
820+
"content": {
821+
"application/json": {
822+
"example": {
823+
"genkey": "KCPP2342"
824+
},
825+
"schema": {
826+
"properties": {
827+
"genkey": {
828+
"type": "string",
829+
"description": "A unique key used to identify the previous generation."
830+
}
831+
},
832+
"type": "object"
833+
}
834+
}
835+
},
836+
"required": false
837+
},
838+
"responses": {
839+
"200": {
840+
"content": {
841+
"application/json": {
842+
"example": {
843+
"logprobs": {
844+
"content": [
845+
{
846+
"token": "Hello",
847+
"logprob": -0.31725305,
848+
"bytes": [72, 101, 108, 108, 111],
849+
"top_logprobs": [
850+
{
851+
"token": "Hello",
852+
"logprob": -0.31725305,
853+
"bytes": [72, 101, 108, 108, 111]
854+
},
855+
{
856+
"token": "Hi",
857+
"logprob": -1.3190403,
858+
"bytes": [72, 105]
859+
}
860+
]
861+
},
862+
{
863+
"token": "!",
864+
"logprob": -0.02380986,
865+
"bytes": [
866+
33
867+
],
868+
"top_logprobs": [
869+
{
870+
"token": "!",
871+
"logprob": -0.02380986,
872+
"bytes": [33]
873+
},
874+
{
875+
"token": " there",
876+
"logprob": -3.787621,
877+
"bytes": [32, 116, 104, 101, 114, 101]
878+
}
879+
]
880+
},
881+
{
882+
"token": " How",
883+
"logprob": -0.000054669687,
884+
"bytes": [32, 72, 111, 119],
885+
"top_logprobs": [
886+
{
887+
"token": " How",
888+
"logprob": -0.000054669687,
889+
"bytes": [32, 72, 111, 119]
890+
},
891+
{
892+
"token": "<|end|>",
893+
"logprob": -10.953937,
894+
"bytes": null
895+
}
896+
]
897+
},
898+
{
899+
"token": " can",
900+
"logprob": -0.015801601,
901+
"bytes": [32, 99, 97, 110],
902+
"top_logprobs": [
903+
{
904+
"token": " can",
905+
"logprob": -0.015801601,
906+
"bytes": [32, 99, 97, 110]
907+
},
908+
{
909+
"token": " may",
910+
"logprob": -4.161023,
911+
"bytes": [32, 109, 97, 121]
912+
}
913+
]
914+
},
915+
{
916+
"token": " I",
917+
"logprob": -3.7697225e-6,
918+
"bytes": [
919+
32,
920+
73
921+
],
922+
"top_logprobs": [
923+
{
924+
"token": " I",
925+
"logprob": -3.7697225e-6,
926+
"bytes": [32, 73]
927+
},
928+
{
929+
"token": " assist",
930+
"logprob": -13.596657,
931+
"bytes": [32, 97, 115, 115, 105, 115, 116]
932+
}
933+
]
934+
},
935+
{
936+
"token": " assist",
937+
"logprob": -0.04571125,
938+
"bytes": [32, 97, 115, 115, 105, 115, 116],
939+
"top_logprobs": [
940+
{
941+
"token": " assist",
942+
"logprob": -0.04571125,
943+
"bytes": [32, 97, 115, 115, 105, 115, 116]
944+
},
945+
{
946+
"token": " help",
947+
"logprob": -3.1089056,
948+
"bytes": [32, 104, 101, 108, 112]
949+
}
950+
]
951+
},
952+
{
953+
"token": " you",
954+
"logprob": -5.4385737e-6,
955+
"bytes": [32, 121, 111, 117],
956+
"top_logprobs": [
957+
{
958+
"token": " you",
959+
"logprob": -5.4385737e-6,
960+
"bytes": [32, 121, 111, 117]
961+
},
962+
{
963+
"token": " today",
964+
"logprob": -12.807695,
965+
"bytes": [32, 116, 111, 100, 97, 121]
966+
}
967+
]
968+
},
969+
{
970+
"token": " today",
971+
"logprob": -0.0040071653,
972+
"bytes": [32, 116, 111, 100, 97, 121],
973+
"top_logprobs": [
974+
{
975+
"token": " today",
976+
"logprob": -0.0040071653,
977+
"bytes": [32, 116, 111, 100, 97, 121]
978+
},
979+
{
980+
"token": "?",
981+
"logprob": -5.5247097,
982+
"bytes": [63]
983+
}
984+
]
985+
},
986+
{
987+
"token": "?",
988+
"logprob": -0.0008108172,
989+
"bytes": [63],
990+
"top_logprobs": [
991+
{
992+
"token": "?",
993+
"logprob": -0.0008108172,
994+
"bytes": [63]
995+
},
996+
{
997+
"token": "?\n",
998+
"logprob": -7.184561,
999+
"bytes": [63, 10]
1000+
}
1001+
]
1002+
}
1003+
]
1004+
}
1005+
},
1006+
"schema": {
1007+
"properties": {
1008+
"logprobs": {
1009+
"type": "object",
1010+
"description": "A logprobs object in the same format as OpenAI API."
1011+
}
1012+
}
1013+
}
1014+
}
1015+
},
1016+
"description": "Successful request"
1017+
}
1018+
},
1019+
"summary": "Obtains the token logprobs of the most recent request.",
1020+
"tags": [
1021+
"api/extra"
1022+
]
1023+
}
1024+
},
8111025
"/api/extra/tokencount": {
8121026
"post": {
8131027
"description": "Counts the number of tokens in a string.",

0 commit comments

Comments
 (0)