File tree Expand file tree Collapse file tree 2 files changed +1
-16
lines changed
src/main/java/com/example/model/format Expand file tree Collapse file tree 2 files changed +1
-16
lines changed Original file line number Diff line number Diff line change @@ -57,7 +57,7 @@ public List<Integer> encodeMessage(Message message) {
57
57
public List <Integer > encodeDialogPrompt (boolean appendAssistantTurn , List <Message > dialog ) {
58
58
List <Integer > tokens = new ArrayList <>();
59
59
tokens .add (beginOfText );
60
- for (LlamaChatFormat . Message message : dialog ) {
60
+ for (Message message : dialog ) {
61
61
tokens .addAll (encodeMessage (message ));
62
62
}
63
63
if (appendAssistantTurn ) {
Original file line number Diff line number Diff line change @@ -58,26 +58,11 @@ public List<Integer> encodeHeader(Message message) {
58
58
@ Override
59
59
public List <Integer > encodeMessage (Message message ) {
60
60
List <Integer > tokens = encodeHeader (message );
61
- //tokens.add(beginOfInstruction);
62
61
tokens .addAll (tokenizer .encodeAsList (message .content ().strip ()));
63
62
tokens .add (endOfInstruction );
64
63
return tokens ;
65
64
}
66
65
67
- public List <Integer > encodeMessage (String userMessage , boolean addHeader , boolean addFooter ) {
68
- List <Integer > tokens = new ArrayList <>();
69
- if (addHeader ) {
70
- tokens .add (this .beginOfInstruction );
71
- }
72
- if (userMessage != null ) {
73
- tokens .addAll (this .tokenizer .encodeAsList (userMessage .strip ()));
74
- }
75
- if (addFooter ) {
76
- tokens .add (endOfInstruction );
77
- }
78
- return tokens ;
79
- }
80
-
81
66
public List <Integer > encodeFillInTheMiddle (String prefix , String suffix ) {
82
67
List <Integer > tokens = new ArrayList <>();
83
68
// dummy - empty string set to comply with encode method signature.
You can’t perform that action at this time.
0 commit comments