diff --git a/.github/workflows/run-examples.yml b/.github/workflows/run-examples.yml index 58d5aa751..42f68fa59 100644 --- a/.github/workflows/run-examples.yml +++ b/.github/workflows/run-examples.yml @@ -44,4 +44,5 @@ jobs: WATSONX_PROJECT_ID: ${{ secrets.WATSONX_PROJECT_ID }} WATSONX_APIKEY: ${{ secrets.WATSONX_APIKEY }} WATSONX_URL: ${{ secrets.WATSONX_URL }} + REPLICATE_API_TOKEN: ${{ secrets.REPLICATE_API_TOKEN }} run: py.test -v --capture=tee-sys tests/test_examples_run.py diff --git a/README.md b/README.md index 69f3f531a..a4ccf49a8 100644 --- a/README.md +++ b/README.md @@ -39,13 +39,16 @@ pip install prompt-declaration-language To install the dependencies for development of PDL and execute all the example, execute the command: ``` -pip install 'prompt-declaration-language[dev]' pip install 'prompt-declaration-language[examples]' -pip install 'prompt-declaration-language[docs]' ``` -In order to run the examples that use foundation models hosted on [watsonx](https://www.ibm.com/watsonx) via LiteLLM, you need a watsonx account (a free plan is available) and set up the following environment variables: -- `WATSONX_URL`, the API url (set to `https://{region}.ml.cloud.ibm.com`) of your watsonx instance +Most examples in this repository use IBM Granite models on [Replicate](https://replicate.com/). +In order to run these examples, you need to create a free account +on Replicate, get an API key and store it in the environment variable: +- `REPLICATE_API_TOKEN` + +In order to use foundation models hosted on [Watsonx](https://www.ibm.com/watsonx) via LiteLLM, you need a WatsonX account (a free plan is available) and set up the following environment variables: +- `WATSONX_URL`, the API url (set to `https://{region}.ml.cloud.ibm.com`) of your WatsonX instance - `WATSONX_APIKEY`, the API key (see information on [key creation](https://cloud.ibm.com/docs/account?topic=account-userapikey&interface=ui#create_user_key)) - `WATSONX_PROJECT_ID`, the project hosting the resources (see information about [project creation](https://www.ibm.com/docs/en/watsonx/saas?topic=projects-creating-project) and [finding project ID](https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-project-id.html?context=wx)). @@ -117,16 +120,14 @@ In PDL, we can write some YAML to create a prompt and call an LLM: ```yaml description: Hello world text: -- Hello, -- model: watsonx/ibm/granite-34b-code-instruct +- "Hello\n" +- model: replicate/ibm-granite/granite-20b-code-instruct-8k parameters: - decoding_method: greedy - stop: - - '!' - include_stop_sequence: true + stop_sequences: '!' + temperature: 0 ``` -The `description` field is a description for the program. Field `text` contains a list of either strings or *block*s which together form the text to be produced. In this example, the text starts with the string `"Hello"` followed by a block that calls out to a model. In this case, it is model with id `watsonx/ibm/granite-34b-code-instruct` from [watsonx](https://www.ibm.com/watsonx), via LiteLLM, with the indicated parameters: the stop sequence is `!`, which is to be included in the output. The input to the model call is everything that has been produced so far in the document (here `Hello`). +The `description` field is a description for the program. Field `text` contains a list of either strings or *block*s which together form the text to be produced. In this example, the text starts with the string `"Hello\n"` followed by a block that calls out to a model. In this case, it is model with id `replicate/ibm-granite/granite-20b-code-instruct-8k` on Replicate, via LiteLLM, with the indicated parameters: the stop sequence is `!`, and temperature set to `0`. Stop sequences are provided with a comman separated list of strings. The input to the model call is everything that has been produced so far in the program (here `"Hello\n"`). When we execute this program using the PDL interpreter: @@ -134,13 +135,29 @@ When we execute this program using the PDL interpreter: pdl examples/hello/hello.pdl ``` -we obtain the following document: +we obtain the following: ``` -Hello, World! +Hello +Hello ``` -where the portion `, World!` was produced by granite. In general, PDL provides blocks for calling models, Python code, and makes it easy to compose them together with control structures (sequencing, conditions, loops). +where the second `Hello` was produced by Granite. In general, PDL provides blocks for calling models, Python code, and makes it easy to compose them together with control structures (sequencing, conditions, loops). + +A similar example on WatsonX would look as follows: + +```yaml +description: Hello world +text: +- Hello, +- model: watsonx/ibm/granite-34b-code-instruct + parameters: + decoding_method: greedy + stop: + - '!' +``` + +Notice the syntactic differences. Model ids on WatsonX start with `watsonx`. The `decoding_method` can be set to `greedy`, rather than setting the temperature to `0`. Also, `stop_sequences` are indicated with the keyword `stop` instead as a list of strings. A PDL program computes 2 data structures. The first is a JSON corresponding to the result of the overall program, obtained by aggregating the results of each block. This is what is printed by default when we run the interpreter. The second is a conversational background context, which is a list of role/content pairs, where we implicitly keep track of roles and content for the purpose of communicating with models that support chat APIs. The contents in the latter correspond to the results of each block. The conversational background context is what is used to make calls to LLMs via LiteLLM. @@ -210,7 +227,7 @@ defs: parser: yaml text: - "\n${ CODE.source_code }\n" -- model: watsonx/ibm/granite-34b-code-instruct +- model: replicate/ibm-granite/granite-3.0-8b-instruct input: - | Here is some info about the location of the function in the repo. @@ -248,11 +265,19 @@ public static Map deserializeOffsetMap(String lastSourceOffset) return offsetMap; } +This Java function, `deserializeOffsetMap`, is designed to deserialize a JSON string into a `Map`. Here's a breakdown of what it does: + +1. It takes a single argument, `lastSourceOffset`, which is expected to be a JSON string. + +2. It initializes a `Map` called `offsetMap`. +3. If `lastSourceOffset` is either `null` or an empty string, it creates a new `HashMap` and assigns it to `offsetMap`. -The function `deserializeOffsetMap` takes a string as input and returns a map. It first checks if the input string is null or empty. If it is, it creates a new empty map and returns it. Otherwise, it uses the Jackson library to parse the input string into a map and returns it. +4. If `lastSourceOffset` is not `null` or an empty string, it uses the `JSON_MAPPER` object (which is presumably an instance of a JSON deserialization library like Jackson) to deserialize the JSON string into a `Map` and assigns it to `offsetMap`. -The `@SuppressWarnings("unchecked")` annotation is used to suppress the warning that the type of the parsed map is not checked. This is because the Jackson library is used to parse the input string into a map, but the specific type of the map is not known at compile time. Therefore, the warning is suppressed to avoid potential issues. +5. Finally, it returns the `offsetMap`. + +The `@SuppressWarnings("unchecked")` annotation is used to suppress a compile-time warning about the raw use of the `Map` type. This is because the `JSON_MAPPER.readValue` method returns a `Map` object, but the compiler doesn't know that this `Map` will be a `Map`. The `unchecked` warning is suppressed to avoid this compile-time warning. ``` Notice that in PDL variables are used to templatize any entity in the document, not just textual prompts to LLMs. We can add a block to this document to evaluate the quality of the output using a similarity metric with respect to our [ground truth](https://github.com/IBM/prompt-declaration-language/blob/main/examples/code/ground_truth.txt). See [file](https://github.com/IBM/prompt-declaration-language/blob/main/examples/code/code-eval.pdl): @@ -267,7 +292,7 @@ defs: read: ./ground_truth.txt text: - "\n${ CODE.source_code }\n" -- model: watsonx/ibm/granite-34b-code-instruct +- model: replicate/ibm-granite/granite-3.0-8b-instruct def: EXPLANATION input: | Here is some info about the location of the function in the repo. @@ -316,14 +341,23 @@ public static Map deserializeOffsetMap(String lastSourceOffset) return offsetMap; } +This Java method, `deserializeOffsetMap`, is designed to convert a JSON string into a `Map`. Here's a breakdown of the code: + +1. The method takes a single argument, `lastSourceOffset`, which is expected to be a JSON string. + +2. It initializes a `Map` called `offsetMap`. -The function `deserializeOffsetMap` takes a string as input and returns a map. It first checks if the input string is null or empty. If it is, it creates a new empty map and returns it. Otherwise, it uses the Jackson library to parse the input string into a map and returns it. +3. If `lastSourceOffset` is either `null` or an empty string, it creates a new `HashMap` and assigns it to `offsetMap`. -The `@SuppressWarnings("unchecked")` annotation is used to suppress the warning that the type of the parsed map is not checked. This is because the Jackson library is used to parse the input string into a map, but the specific type of the map is not known at compile time. Therefore, the warning is suppressed to avoid potential issues. +4. If `lastSourceOffset` is not `null` or empty, it uses the `JSON_MAPPER` object (which is presumably an instance of `ObjectMapper` from the Jackson library) to convert the JSON string into a `Map` and assigns it to `offsetMap`. + +5. Finally, it returns the `offsetMap`. + +The `@SuppressWarnings("unchecked")` annotation is used to suppress a potential unchecked warning that might occur if the JSON string does not match the expected `Map` type. EVALUATION: The similarity (Levenshtein) between this answer and the ground truth is: -0.9967637540453075 +0.30199115044247793 ``` PDL allows rapid prototyping of prompts by allowing the user to change prompts and see their immediate effects on metrics. Try it! @@ -339,7 +373,7 @@ defs: TRUTH: read: ./ground_truth.txt text: -- model: watsonx/ibm/granite-34b-code-instruct +- model: replicate/ibm-granite/granite-3.0-8b-instruct def: EXPLANATION contribute: [] input: @@ -401,8 +435,8 @@ This is similar to a spreadsheet for tabular data, where data is in the forefron ## Additional Notes -When using Granite models on watsonx, we use the following defaults for model parameters (except `granite-20b-code-instruct-r1.1`): - - `decoding_method`: `greedy` +When using Granite models, we use the following defaults for model parameters (except `granite-20b-code-instruct-r1.1`): + - `decoding_method`: `greedy`, (`temperature`: 0) - `max_new_tokens`: 1024 - `min_new_tokens`: 1 - `repetition_penalty`: 1.05 diff --git a/docs/README.md b/docs/README.md index e673543e5..ad17f782a 100644 --- a/docs/README.md +++ b/docs/README.md @@ -44,13 +44,16 @@ pip install prompt-declaration-language To install the dependencies for development of PDL and execute all the example, execute the command: ``` -pip install 'prompt-declaration-language[dev]' pip install 'prompt-declaration-language[examples]' -pip install 'prompt-declaration-language[docs]' ``` -In order to run the examples that use foundation models hosted on [watsonx](https://www.ibm.com/watsonx) via LiteLLM, you need a watsonx account (a free plan is available) and set up the following environment variables: -- `WATSONX_URL`, the API url (set to `https://{region}.ml.cloud.ibm.com`) of your watsonx instance +Most examples in this repository use IBM Granite models on [Replicate](https://replicate.com/). +In order to run these examples, you need to create a free account +on Replicate, get an API key and store it in the environment variable: +- `REPLICATE_API_TOKEN` + +In order to use foundation models hosted on [Watsonx](https://www.ibm.com/watsonx) via LiteLLM, you need a WatsonX account (a free plan is available) and set up the following environment variables: +- `WATSONX_URL`, the API url (set to `https://{region}.ml.cloud.ibm.com`) of your WatsonX instance - `WATSONX_APIKEY`, the API key (see information on [key creation](https://cloud.ibm.com/docs/account?topic=account-userapikey&interface=ui#create_user_key)) - `WATSONX_PROJECT_ID`, the project hosting the resources (see information about [project creation](https://www.ibm.com/docs/en/watsonx/saas?topic=projects-creating-project) and [finding project ID](https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-project-id.html?context=wx)). @@ -122,16 +125,14 @@ In PDL, we can write some YAML to create a prompt and call an LLM: ```yaml description: Hello world text: -- Hello, -- model: watsonx/ibm/granite-34b-code-instruct +- "Hello\n" +- model: replicate/ibm-granite/granite-20b-code-instruct-8k parameters: - decoding_method: greedy - stop: - - '!' - include_stop_sequence: true + stop_sequences: '!' + temperature: 0 ``` -The `description` field is a description for the program. Field `text` contains a list of either strings or *block*s which together form the text to be produced. In this example, the text starts with the string `"Hello"` followed by a block that calls out to a model. In this case, it is model with id `watsonx/ibm/granite-34b-code-instruct` from [watsonx](https://www.ibm.com/watsonx), via LiteLLM, with the indicated parameters: the stop sequence is `!`, which is to be included in the output. The input to the model call is everything that has been produced so far in the document (here `Hello`). +The `description` field is a description for the program. Field `text` contains a list of either strings or *block*s which together form the text to be produced. In this example, the text starts with the string `"Hello\n"` followed by a block that calls out to a model. In this case, it is model with id `replicate/ibm-granite/granite-20b-code-instruct-8k` on Replicate, via LiteLLM, with the indicated parameters: the stop sequence is `!`, and temperature set to `0`. Stop sequences are provided with a comman separated list of strings. The input to the model call is everything that has been produced so far in the program (here `"Hello\n"`). When we execute this program using the PDL interpreter: @@ -139,13 +140,29 @@ When we execute this program using the PDL interpreter: pdl examples/hello/hello.pdl ``` -we obtain the following document: +we obtain the following: ``` -Hello, World! +Hello +Hello ``` -where the portion `, World!` was produced by granite. In general, PDL provides blocks for calling models, Python code, and makes it easy to compose them together with control structures (sequencing, conditions, loops). +where the second `Hello` was produced by Granite. In general, PDL provides blocks for calling models, Python code, and makes it easy to compose them together with control structures (sequencing, conditions, loops). + +A similar example on WatsonX would look as follows: + +```yaml +description: Hello world +text: +- Hello, +- model: watsonx/ibm/granite-34b-code-instruct + parameters: + decoding_method: greedy + stop: + - '!' +``` + +Notice the syntactic differences. Model ids on WatsonX start with `watsonx`. The `decoding_method` can be set to `greedy`, rather than setting the temperature to `0`. Also, `stop_sequences` are indicated with the keyword `stop` instead as a list of strings. A PDL program computes 2 data structures. The first is a JSON corresponding to the result of the overall program, obtained by aggregating the results of each block. This is what is printed by default when we run the interpreter. The second is a conversational background context, which is a list of role/content pairs, where we implicitly keep track of roles and content for the purpose of communicating with models that support chat APIs. The contents in the latter correspond to the results of each block. The conversational background context is what is used to make calls to LLMs via LiteLLM. @@ -215,7 +232,7 @@ defs: parser: yaml text: - "\n${ CODE.source_code }\n" -- model: watsonx/ibm/granite-34b-code-instruct +- model: replicate/ibm-granite/granite-3.0-8b-instruct input: - | Here is some info about the location of the function in the repo. @@ -253,11 +270,19 @@ public static Map deserializeOffsetMap(String lastSourceOffset) return offsetMap; } +This Java function, `deserializeOffsetMap`, is designed to deserialize a JSON string into a `Map`. Here's a breakdown of what it does: + +1. It takes a single argument, `lastSourceOffset`, which is expected to be a JSON string. + +2. It initializes a `Map` called `offsetMap`. +3. If `lastSourceOffset` is either `null` or an empty string, it creates a new `HashMap` and assigns it to `offsetMap`. -The function `deserializeOffsetMap` takes a string as input and returns a map. It first checks if the input string is null or empty. If it is, it creates a new empty map and returns it. Otherwise, it uses the Jackson library to parse the input string into a map and returns it. +4. If `lastSourceOffset` is not `null` or an empty string, it uses the `JSON_MAPPER` object (which is presumably an instance of a JSON deserialization library like Jackson) to deserialize the JSON string into a `Map` and assigns it to `offsetMap`. -The `@SuppressWarnings("unchecked")` annotation is used to suppress the warning that the type of the parsed map is not checked. This is because the Jackson library is used to parse the input string into a map, but the specific type of the map is not known at compile time. Therefore, the warning is suppressed to avoid potential issues. +5. Finally, it returns the `offsetMap`. + +The `@SuppressWarnings("unchecked")` annotation is used to suppress a compile-time warning about the raw use of the `Map` type. This is because the `JSON_MAPPER.readValue` method returns a `Map` object, but the compiler doesn't know that this `Map` will be a `Map`. The `unchecked` warning is suppressed to avoid this compile-time warning. ``` Notice that in PDL variables are used to templatize any entity in the document, not just textual prompts to LLMs. We can add a block to this document to evaluate the quality of the output using a similarity metric with respect to our [ground truth](https://github.com/IBM/prompt-declaration-language/blob/main/examples/code/ground_truth.txt). See [file](https://github.com/IBM/prompt-declaration-language/blob/main/examples/code/code-eval.pdl): @@ -272,7 +297,7 @@ defs: read: ./ground_truth.txt text: - "\n${ CODE.source_code }\n" -- model: watsonx/ibm/granite-34b-code-instruct +- model: replicate/ibm-granite/granite-3.0-8b-instruct def: EXPLANATION input: | Here is some info about the location of the function in the repo. @@ -321,14 +346,23 @@ public static Map deserializeOffsetMap(String lastSourceOffset) return offsetMap; } +This Java method, `deserializeOffsetMap`, is designed to convert a JSON string into a `Map`. Here's a breakdown of the code: + +1. The method takes a single argument, `lastSourceOffset`, which is expected to be a JSON string. + +2. It initializes a `Map` called `offsetMap`. -The function `deserializeOffsetMap` takes a string as input and returns a map. It first checks if the input string is null or empty. If it is, it creates a new empty map and returns it. Otherwise, it uses the Jackson library to parse the input string into a map and returns it. +3. If `lastSourceOffset` is either `null` or an empty string, it creates a new `HashMap` and assigns it to `offsetMap`. -The `@SuppressWarnings("unchecked")` annotation is used to suppress the warning that the type of the parsed map is not checked. This is because the Jackson library is used to parse the input string into a map, but the specific type of the map is not known at compile time. Therefore, the warning is suppressed to avoid potential issues. +4. If `lastSourceOffset` is not `null` or empty, it uses the `JSON_MAPPER` object (which is presumably an instance of `ObjectMapper` from the Jackson library) to convert the JSON string into a `Map` and assigns it to `offsetMap`. + +5. Finally, it returns the `offsetMap`. + +The `@SuppressWarnings("unchecked")` annotation is used to suppress a potential unchecked warning that might occur if the JSON string does not match the expected `Map` type. EVALUATION: The similarity (Levenshtein) between this answer and the ground truth is: -0.9967637540453075 +0.30199115044247793 ``` PDL allows rapid prototyping of prompts by allowing the user to change prompts and see their immediate effects on metrics. Try it! @@ -344,7 +378,7 @@ defs: TRUTH: read: ./ground_truth.txt text: -- model: watsonx/ibm/granite-34b-code-instruct +- model: replicate/ibm-granite/granite-3.0-8b-instruct def: EXPLANATION contribute: [] input: @@ -406,8 +440,8 @@ This is similar to a spreadsheet for tabular data, where data is in the forefron ## Additional Notes -When using Granite models on watsonx, we use the following defaults for model parameters (except `granite-20b-code-instruct-r1.1`): - - `decoding_method`: `greedy` +When using Granite models, we use the following defaults for model parameters (except `granite-20b-code-instruct-r1.1`): + - `decoding_method`: `greedy`, (`temperature`: 0) - `max_new_tokens`: 1024 - `min_new_tokens`: 1 - `repetition_penalty`: 1.05 diff --git a/docs/tutorial.md b/docs/tutorial.md index e9d15007d..973824137 100644 --- a/docs/tutorial.md +++ b/docs/tutorial.md @@ -36,8 +36,8 @@ Hello, world! --8<-- "./examples/tutorial/calling_llm.pdl" ``` -In this program ([file](https://github.com/IBM/prompt-declaration-language//blob/main/examples/tutorial/calling_llm.pdl)), the `text` starts with the word `Hello,`, and we call a model (`watsonx/ibm/granite-34b-code-instruct`) with this as input prompt. Notice the watsonx model id on LiteLLM. -The model is passed some parameters including the `decoding_method` and `stop`, which corresponds to the `stop_sequences` parameter in watsonx. The stop sequences are to be included in the output. +In this program ([file](https://github.com/IBM/prompt-declaration-language//blob/main/examples/tutorial/calling_llm.pdl)), the `text` starts with the word `"Hello\n"`, and we call a model (`replicate/ibm-granite/granite-3.0-8b-instruct`) with this as input prompt. +The model is passed a parameter `stop_sequences`. A PDL program computes 2 data structures. The first is a JSON corresponding to the result of the overall program, obtained by aggregating the results of each block. This is what is printed by default when we run the interpreter. The second is a conversational background context, which is a list of role/content pairs, where we implicitly keep track of roles and content for the purpose of communicating with models that support chat APIs. The contents in the latter correspond to the results of each block. The conversational background context is what is used to make calls to LLMs via LiteLLM. @@ -47,23 +47,24 @@ default role `user`. When we execute this program using the interpreter, we obtain: ``` -Hello, world! +Hello +Hello ``` -where the portion ` world!` has been generated by Granite. +where the second `Hello` has been generated by Granite. -Here's another of model call that includes an `input` field ([file](https://github.com/IBM/prompt-declaration-language//blob/main/examples/tutorial/calling_llm_with_input.pdl)): +Here's another example of model call that includes an `input` field ([file](https://github.com/IBM/prompt-declaration-language//blob/main/examples/tutorial/calling_llm_with_input.pdl)): ```yaml --8<-- "./examples/tutorial/calling_llm_with_input.pdl" ``` -In this case, we make a call to the granite multilingual model, and the input passed to the model is the sentence: `Translate the word 'world' to French` and nothing else from the surrounding document. When we execute this program, we obtain: +In this case, the input passed to the model is the sentence: `Translate the word 'Hello' to French` and nothing else from the surrounding document. When we execute this program, we obtain: ``` -Hello, : -Le mot 'world' en français est 'monde'. +Hello +The word 'Hello' translates to 'Bonjour' in French. ``` -where everything after the `:` including it were generated by the model. +where the second line is generated by the model. Using the `input` field, we can also give a directly an array of messages (`role`/`content`) to the model ([file](https://github.com/IBM/prompt-declaration-language//blob/main/examples/tutorial/calling_llm_with_input_messages.pdl)): @@ -71,15 +72,17 @@ Using the `input` field, we can also give a directly an array of messages (`role --8<-- "./examples/tutorial/calling_llm_with_input_messages.pdl" ``` +This has the same output as the previous program. + ### Parameter defaults for watsonx Granite models -PDL provides the following defaults for watsonx Granite models, when the following parameters are missing: - - `decoding_method`: `greedy` +When using Granite models, we use the following defaults for model parameters (except `granite-20b-code-instruct-r1.1`): + - `decoding_method`: `greedy`, (`temperature`: 0) - `max_new_tokens`: 1024 - `min_new_tokens`: 1 - `repetition_penalty`: 1.05 - -Also if the `decoding_method` is `sample`, then the following defaults are used: + + Also if the `decoding_method` is `sample`, then the following defaults are used: - `temperature`: 0.7 - `top_p`: 0.85 - `top_k`: 50 @@ -98,12 +101,14 @@ Consider the following example ([file](https://github.com/IBM/prompt-declaration --8<-- "./examples/tutorial/variable_def_use.pdl" ``` -Here we assign the output of the model to variable `GEN` using the `def` field. The last line of the program prints out the value of `GEN`. Notice the notation `${ }` for accessing the value of a variable. +Here we assign the output of the model to variable `GEN` using the `def` field. The last line of the program prints out the value of `GEN`. Notice the notation `${ }` for accessing the value of a variable. Any [Jinja](https://jinja.palletsprojects.com/en/3.1.x/) expression is allowed to be used inside these braces. These expressions +are also used to specify conditions for loops and conditionals. See for example this [file](https://github.com/IBM/prompt-declaration-language//blob/main/examples/tutorial/conditionals_loops.pdl). When we execute this program, we obtain: ``` -Hello, world! -GEN is equal to: world! +Hello +Hello +GEN is equal to: Hello ``` ## Model Chaining @@ -119,9 +124,10 @@ In this program, the first call is to a granite model to complete the sentence ` When we execute this program, we obtain: ``` -Hello, World! -Translate this to French -Bonjour le monde ! +Hello +Hello +Did you just say Hello? +Yes, I did. How can I assist you today? ``` @@ -131,27 +137,9 @@ Bonjour le monde ! PDL also supports function definitions to make it easier to reuse code. Suppose we want to define a translation function that takes a string and calls a falcon model for the translation. This would be written in PDL as follows ([file](https://github.com/IBM/prompt-declaration-language//blob/main/examples/tutorial/function_definition.pdl)): + ```yaml -description: function def and call -text: -- def: translate - function: - sentence: str - language: str - return: - - "\nTranslate the sentence '${ sentence }' to ${ language }.\n" - - model: watsonx/ibm/granite-20b-multilingual - parameters: - stop: ["\n"] -- call: translate - args: - sentence: I love Paris! - language: French -- "\n" -- call: translate - args: - sentence: I love Madrid! - language: Spanish +--8<-- "./examples/tutorial/function_definition.pdl" ``` In this program, the first block defines a function `translate` that takes as parameters `sentence` and `language`, both of which are of type string. The body of the function is defined by its `return` field. In this case, we formulate a translation prompt using the parameters and send it to a Granite multilingual model. @@ -165,6 +153,8 @@ The translation of 'I love Madrid!' to Spanish is 'Me encanta Madrid!'. A function only contributes to the output document when it is called. So the definition itself results in `""`. When we call a function, we implicitly pass the current background context, and this is used as input to model calls inside the function body. In the above example, since the `input` field is omitted, the entire document produced at that point is passed as input to the granite model. +To reset the context when calling a function, we can pass the special argument: `pdl_context: []`. + Notice that the arguments of function calls are expressions and cannot be arbitrary PDL blocks. ## Grouping Variable Definitions in Defs @@ -172,29 +162,10 @@ Notice that the arguments of function calls are expressions and cannot be arbitr In PDL, the above program can be written more neatly by grouping certain variable definitions into a `defs` section, as follows ([file](https://github.com/IBM/prompt-declaration-language//blob/main/examples/tutorial/grouping_definitions.pdl)): ```yaml -description: function def and call -defs: - translate: - function: - sentence: str - language: str - return: - - "\nTranslate the sentence '${ sentence }' to ${ language }.\n" - - model: watsonx/ibm/granite-20b-multilingual - parameters: - stop: ["\n"] -text: -- call: translate - args: - sentence: I love Paris! - language: French -- "\n" -- call: translate - args: - sentence: I love Madrid! - language: Spanish +--8<-- "./examples/tutorial/grouping_definitions.pdl" ``` + This program has the same output has the one from the previous section. @@ -203,30 +174,13 @@ This program has the same output has the one from the previous section. By default, when a PDL block is executed it produces a result that is contributed to the overall result, and it also contributes to the background context. It is possible to mute both contributions by setting `contribute` to `[]` for any block. This feature allows the computation of intermediate values that are not necessarily output in the document. The value of the variable specified in `def` is still set to the result of the block. -Consider the same example as above, but with `contribute` set to `[]` ([file](https://github.com/IBM/prompt-declaration-language//blob/main/examples/tutorial/muting_block_output.pdl)): +Consider the similar example as above, but with `contribute` set to `[]` ([file](https://github.com/IBM/prompt-declaration-language//blob/main/examples/tutorial/muting_block_output.pdl)): ```yaml -description: function def and call -defs: - translate: - function: - sentence: str - language: str - return: - - "\nTranslate the sentence '${ sentence }' to ${ language }.\n" - - model: watsonx/ibm/granite-20b-multilingual - parameters: - stop: ["\n"] -text: -- call: translate - contribute: [] - def: FRENCH - args: - sentence: I love Paris! - language: French -- "The french sentence was: ${ FRENCH }" +--8<-- "./examples/tutorial/muting_block_output.pdl" ``` + The call to the translator with French as language does not produce an output. However, we save the result in variable `FRENCH` and use it in the last sentence of the document. When we execute this program, we obtain: ``` @@ -307,63 +261,39 @@ You could also turn the list into a text or an array by surrounding it with a `t PDL can accept textual input from a file or stdin. In the following example ([file](https://github.com/IBM/prompt-declaration-language//blob/main/examples/tutorial/input_file.pdl)), the contents of this [file](https://github.com/IBM/prompt-declaration-language//blob/main/examples/tutorial/data.txt) are read by PDL and incorporated in the document. The result is also assigned to a variable `HELLO`. - ```yaml -description: PDL code with input block -text: -- read: ./data.txt - def: HELLO +--8<-- "./examples/tutorial/input_file.pdl" ``` + In the next example, prompts are obtained from stdin ([file](https://github.com/IBM/prompt-declaration-language//blob/main/examples/tutorial/input_stdin.pdl)). This is indicated by assigning the value `null` to the `read` field. ```yaml -description: PDL code with input block -text: -- "The following will prompt the user on stdin.\n" -- read: - message: "Please provide an input: " - def: STDIN +--8<-- "./examples/tutorial/input_stdin.pdl" ``` + If the `message` field is omitted then one is provided for you. -The following example shows a multiline stdin input ([file](https://github.com/IBM/prompt-declaration-language//blob/main/examples/tutorial/input_stdin_multiline.pdl)). When executing this code and to exit from the multiline input simply press control D (macos). +The following example shows a multiline stdin input ([file](https://github.com/IBM/prompt-declaration-language//blob/main/examples/tutorial/input_stdin_multiline.pdl)). When executing this code and to exit from the multiline input simply press control D (on macOS). + ```yaml -description: PDL code with input block -text: -- "A multiline stdin input.\n" -- read: - multiline: true +--8<-- "./examples/tutorial/input_stdin_multiline.pdl" ``` + Finally, the following example shows reading content in JSON format. Consider the JSON content in this [file](https://github.com/IBM/prompt-declaration-language//blob/main/examples/tutorial/input.json): -```json -{ - "name": "Bob", - "address": { - "number": 87, - "street": "Smith Road", - "town": "Armonk", - "state": "NY", - "zip": 10504 - } -} + +```yaml +--8<-- "./examples/tutorial/input.json" ``` The following PDL program reads this content and assigns it to variable `PERSON` in JSON format ([file](https://github.com/IBM/prompt-declaration-language//blob/main/examples/tutorial/input_file_json.pdl)). The reference `PERSON.address.street` then refers to that field inside the JSON object. ```yaml -description: Input block example with json input -defs: - PERSON: - read: ./input.json - parser: json -text: | - ${ PERSON.name } lives at the following address: - ${ PERSON.address.number } ${ PERSON.address.street } in the town of ${ PERSON.address.town }, ${ PERSON.address.state } +--8<-- "./examples/tutorial/input_file_json.pdl" ``` When we execute this program, we obtain: @@ -375,25 +305,17 @@ Bob lives at the following address: ## Calling code -The following script shows how to execute python code ([file](https://github.com/IBM/prompt-declaration-language//blob/main/examples/tutorial/calling_code.pdl)). Currently, the python code is executed locally. In principle, PDL is agnostic of any specific programming language, but we currently only support Python. Variables defined in PDL are copied into the global scope of the Python code, so those variables can be used directly in the code. However, mutating variables in Python has no effect on the variables in the PDL program. The result of the code must be assigned to the variable `result` internally to be propagated to the result of the block. A variable `def` on the code block will then be set to this result. +The following script shows how to execute python code ([file](https://github.com/IBM/prompt-declaration-language//blob/main/examples/tutorial/calling_code.pdl)). The python code is executed locally (or in a containerized way if using `pdl --sandbox`). In principle, PDL is agnostic of any specific programming language, but we currently only support Python. Variables defined in PDL are copied into the global scope of the Python code, so those variables can be used directly in the code. However, mutating variables in Python has no effect on the variables in the PDL program. The result of the code must be assigned to the variable `result` internally to be propagated to the result of the block. A variable `def` on the code block will then be set to this result. In order to define variables that are carried over to the next Python code block, a special variable `PDL_SESSION` can be used, and variables assigned to it as fields. See for example: ([file](https://github.com/IBM/prompt-declaration-language//blob/main/examples/rag/rag.pdl)). ```yaml -description: Hello world showing call to python code -text: -- "Hello, " -- lang: python - code: - | - import random - import string - result = random.choice(string.ascii_lowercase) +--8<-- "./examples/tutorial/calling_code.pdl" ``` -This results in the following output: +This results in the following output (for example): ``` Hello, r! ``` @@ -403,42 +325,7 @@ Hello, r! PDL programs can contain calls to REST APIs with Python code. Consider a simple weather app ([file](https://github.com/IBM/prompt-declaration-language//blob/main/examples/tutorial/calling_apis.pdl)): ```yaml -description: Using a weather API and LLM to make a small weather app -text: -- read: - def: QUERY - message: "Ask a query: " - contribute: [] -- model: watsonx/ibm/granite-34b-code-instruct - input: | - Question: What is the weather in London? - London - Question: What's the weather in Paris? - Paris - Question: Tell me the weather in Lagos? - Lagos - Question: ${ QUERY } - parameters: - stop: - - Question - - What - - '!' - include_stop_sequence: false - def: LOCATION - contribute: [] -- lang: python - code: | - import requests - response = requests.get('https://api.weatherapi.com/v1/current.json?key=cf601276764642cb96224947230712&q=${ LOCATION }') - result = response.content - def: WEATHER - parser: json - contribute: [] - -- model: watsonx/ibm/granite-34b-code-instruct - input: | - Explain the weather from the following JSON: - ${ WEATHER } +--8<-- "./examples/tutorial/calling_apis.pdl" ``` In this program, we first prompt the user to enter a query about the weather in some location (assigned to variable `QUERY`). The next block is a call to a granite model with few-shot examples to extract the location, which we assign to variable `LOCATION`. The next block makes an API call with Python. Here the `LOCATION` is appended to the `url`. The result is a JSON object, which may be hard to interpret for a human user. So we make a final call to an LLM to interpret the JSON in terms of weather. Notice that many blocks have `contribute` set to `[]` to hide intermediate results. @@ -446,65 +333,23 @@ In this program, we first prompt the user to enter a query about the weather in ## Data Block -PDL offers the ability to create JSON data as illustrated by the following example (described in detail in the [Overview](https://github.com/IBM/prompt-declaration-language//blob/main/README.md#overview) section). The `data` block can gather previously defined variables into a JSON structure. This feature is useful for data generation. Programs such as this one can be bootstrapped with a bash or Python script to generate data en masse. ([file](https://github.com/IBM/prompt-declaration-language//blob/main/examples/tutorial/data_block.pdl)) +PDL offers the ability to create JSON data as illustrated by the following example (described in detail in the [Overview](https://github.com/IBM/prompt-declaration-language/blob/main/README.md#overview) section). The `data` block can gather previously defined variables into a JSON structure. This feature is useful for data generation. Programs such as this one can be bootstrapped with a bash or Python script to generate data en masse ([file](https://github.com/IBM/prompt-declaration-language/blob/main/examples/tutorial/data_block.pdl)). ```yaml -description: Code explanation example -defs: - CODE: - read: ./data.yaml - parser: yaml - TRUTH: - read: ./ground_truth.txt -text: -- model: watsonx/ibm/granite-34b-code-instruct - def: EXPLANATION - contribute: [] - input: - | - Here is some info about the location of the function in the repo. - repo: - ${ CODE.repo_info.repo } - path: ${ CODE.repo_info.path } - Function_name: ${ CODE.repo_info.function_name } - - - Explain the following code: - ``` - ${ CODE.source_code }``` -- def: EVAL - contribute: [] - lang: python - code: - | - import textdistance - expl = """ - ${ EXPLANATION } - """ - truth = """ - ${ TRUTH } - """ - result = textdistance.levenshtein.normalized_similarity(expl, truth) -- data: - input: ${ CODE } - output: ${ EXPLANATION } - metric: ${ EVAL } -``` - -Notice that in the `data` block the values are interpreted as Jinja2 expressions. If values need to be PDL programs to be interpreted, then you need to use +--8<-- "./examples/tutorial/data_block.pdl" +``` + +Notice that in the `data` block the values are interpreted as Jinja expressions. If values need to be PDL programs to be interpreted, then you need to use the `object` block instead (see [this section](#specifying-data)). In the example above, the expressions inside the `data` block are interpreted. In some cases, it may be useful not to interpret the values in a `data` block. The `raw` field can be used to turn off the interpreter inside a `data` block. For example, consider the ([file](https://github.com/IBM/prompt-declaration-language//blob/main/examples/tutorial/data_block_raw.pdl)): -``` -description: raw data block -data: - name: ${ name } - phone: ${ phone } -raw: True +```yaml +--8<-- "./examples/tutorial/data_block_raw.pdl" ``` + The result of this program is the JSON object: ``` @@ -521,33 +366,14 @@ where the values of `name` and `phone` have been left uninterpreted. PDL allows programs to be defined over multiple files. The `include` block allows one file to incorporate another, as shown in the following [example](https://github.com/IBM/prompt-declaration-language//blob/main/examples/tutorial/include.pdl): -``` -description: Granite Multi-Round Chat -text: -- include: ../granite/granite_defs.pdl -- read: ../granite/multi-prompts.json - parser: json - def: prompts - spec: {prompts: [str]} - contribute: [] -- ${ SYSTEM_CONTENT_CHAT } -- for: - prompt: ${ prompts.prompts } - repeat: - text: - - | - ${ prompt } - - model: watsonx/ibm/granite-13b-chat-v2 - parameters: - decoding_method: sample - max_new_tokens: 512 -role: user +```yaml +--8<-- "./examples/tutorial/include.pdl" ``` which includes the following [file](https://github.com/IBM/prompt-declaration-language//blob/main/examples/granite/granite_defs.pdl): -The `include` block means that the PDL code at that file is executed and its output is included at the point where the `include` block appears in a document. In this example, the file contains definitions, which are used in the program. This feature allows reuse of common templates and patterns and to build libraries. Notice that relative paths are relative to the containing file. +The `include` block means that the PDL code at that file is executed and its output is included at the point where the `include` block appears in the program. In this example, the file contains definitions, which are used in the program. This feature allows reuse of common templates and patterns and to build libraries. Notice that relative paths are relative to the containing file. ## Conditionals and Loops @@ -556,24 +382,7 @@ PDL supports conditionals and loops as illustrated in the following example ([fi ```yaml -description: chatbot -text: -- read: - message: "What is your query?\n" - contribute: [context] -- repeat: - text: - - model: watsonx/ibm/granite-13b-chat-v2 - - read: - def: eval - message: "\nIs this a good answer[yes/no]?\n" - contribute: [] - - if: ${ eval == 'no' } - then: - text: - - read: - message: "Why not?\n" - until: ${ eval == 'yes'} +--8<-- "./examples/tutorial/conditionals_loops.pdl" ``` The first block prompts the user for a query, and this is contributed to the background context. The next @@ -587,9 +396,9 @@ the list is interpreted to be a `lastOf` block. So again the blocks in the list of the last block. The chatbot keeps looping by making a call to a model, asking the user if the generated text is a good answer, -and asking `why not?` if the answer (stored in variable `eval`) is `no`. The loop ends when `eval` becomes `yes`. +and asking `why not?` if the answer (stored in variable `eval`) is `no`. The loop ends when `eval` becomes `yes`. This is specified with a Jinja expression on line 18. -Notice that the `repeat` and `then` blocks are followed by `text`. This is because of the semantics of lists in PDL. If we want to aggregate the result by stringifying every element in the list and collating them together (which is the case of top-level programs in general), then we need the keyword `text` to precede a list. If this is omitted then the list is treated as a programmatic sequence where all the blocks are executed in sequence but result of the overall list is the result of the {\em last} block in the sequence. This behavior can be marked explicitly with a `lastOf` block. +Notice that the `repeat` and `then` blocks are followed by `text`. This is because of the semantics of lists in PDL. If we want to aggregate the result by stringifying every element in the list and collating them together, then we need the keyword `text` to precede a list. If this is omitted then the list is treated as a programmatic sequence where all the blocks are executed in sequence but result of the overall list is the result of the {\em last} block in the sequence. This behavior can be marked explicitly with a `lastOf` block. Another form of iteration in PDL is `repeat` followed by `num_iterations`, which repeats the body `num_iterations` times. @@ -601,12 +410,8 @@ the `join` feature (see the following section). PDL also offers `for` loops over lists. The following [example](https://github.com/IBM/prompt-declaration-language//blob/main/examples/tutorial/for.pdl) stringifies and outputs each number. -``` -description: for loop -for: - i: [1, 2, 3, 4] -repeat: - ${ i } +```yaml +--8<-- "./examples/tutorial/for.pdl" ``` This program outputs: @@ -673,7 +478,9 @@ join: ``` meaning that result of each iteration is stringified and concatenated with that of other iterations. When using `with`, -`as: text` can be elided. +`as: text` can be elided. + +Note that `join` can be added to any looping construct (`repeat`) not just `for` loops. The `for` loop constructs also allows iterating over 2 or more lists of the same length simultaneously: @@ -715,7 +522,7 @@ text: contribute: [context] - repeat: text: - - model: watsonx/ibm/granite-13b-chat-v2 + - model: replicate/ibm-granite/granite-3.0-8b-instruct role: assistant - read: def: eval @@ -731,7 +538,11 @@ role: user ``` In PDL, any block can be adorned with a `role` field indicating the role for that block. These are high-level annotations -that help to make programs more portable accross different models. PDL takes care of applying appropriate chat templates. +that help to make programs more portable accross different models. If the role of a block is not specified (except for model blocks that have `assistant` role), +then the role is inherited from the surrounding block. So in the above example, we only need to specify `role: user` at the top level (this is the default, so it doesn't +need to be specified explicitly). + +PDL takes care of applying appropriate chat templates. The prompt that is actually submitted to the first model call (with query `What is APR?`) is the following: ``` @@ -742,7 +553,7 @@ The prompt that is actually submitted to the first model call (with query `What To change the template that is applied, you can specify it as a parameter of the model call: ```yaml -model: watsonx/ibm/granite-13b-chat-v2 +model: replicate/ibm-granite/granite-3.0-8b-instruct parameters: roles: system: @@ -759,37 +570,11 @@ parameters: ## Type Checking Consider the following PDL program ([file](https://github.com/IBM/prompt-declaration-language//blob/main/examples/tutorial/gen-data.pdl)). It first reads the data -found [here](https://github.com/IBM/prompt-declaration-language//blob/main/examples/tutorial/data.pdl) to form few-shot examples. These demonstrations show how to create +found [here](https://github.com/IBM/prompt-declaration-language//blob/main/examples/tutorial/data.yaml) to form few-shot examples. These demonstrations show how to create some JSON data. -``` -description: Creating JSON Data -defs: - data: - read: ./data.yaml - parser: yaml - spec: { questions: [str], answers: [obj] } -text: - - model: watsonx/ibm/granite-20b-code-instruct - def: model_output - spec: {name: str, age: int} - input: - text: - - for: - question: ${ data.questions } - answer: ${ data.answers } - repeat: - - | - ${ question } - ${ answer } - - > - Question: Create a JSON object with fields 'name' and 'age' - and set them appropriately. Write the age in letters. - parser: yaml - parameters: - stop: - - '}' - include_stop_sequence: true +```yaml +--8<-- "./examples/tutorial/gen-data.pdl" ``` Upon reading the data we use a parser to parse it into a YAML. The `spec` field indicates the expected type for the @@ -801,13 +586,12 @@ Similarly, the output of the model call is parsed as YAML, and the `spec` indica When we run this program, we obtain the output: ``` -{ "name" : "John", "age" : "twenty five" } -examples/tutorial/gen-data.pdl:8 - Type errors during spec checking -examples/tutorial/gen-data.pdl:8 - twenty five should be of type +gen-data.pdl:8 - Type errors during spec checking: +gen-data.pdl:8 - 30 should be of type +{'name': 'John', 'age': '30'} ``` -Notice that since we asked the age to be produced in letters, this causes a type error indicated above. -If we modify the prompt not to contain this directive, the type error goes away. +Notice that since we asked the age to be produced in letters, we got a string back and this causes a type error indicated above. In general, `spec` definitions can be a subset of JSON schema, or use a shorthand notation as illustrated by the examples below: @@ -834,7 +618,9 @@ the examples below: ## Python SDK See examples of PDL being called programmatically in Python -([here](https://github.com/IBM/prompt-declaration-language//blob/main/examples/sdk)). +[here](https://github.com/IBM/prompt-declaration-language/blob/main/examples/sdk). + +For a more sophisticated example, see [here](https://github.com/IBM/prompt-declaration-language/blob/main/examples/callback). ## Debugging PDL Programs @@ -863,9 +649,39 @@ in the right pane. This is similar to a spreadsheet for tabular data, where data is in the forefront and the user can inspect the formula that generates the data in each cell. In the Live Document, cells are not uniform but can take arbitrary extents. Clicking on them similarly reveals the part of the code that produced them. +## Using Ollama models + +1. Install Ollama e.g., `brew install --cask ollama` +2. Run a model e.g., `ollama run granite-code:34b-instruct-q5_K_M`. See [the Ollama library for more models](https://ollama.com/library/granite-code/tags) +3. An OpenAI style server is running locally at [http://localhost:11434/](http://localhost:11434/), see [the Ollama blog](https://ollama.com/blog/openai-compatibility) for more details. + + +Example: + +``` +text: +- Hello, +- model: ollama_chat/granite-code:34b-instruct-q5_K_M + parameters: + stop: + - '!' + decoding_method: greedy +``` + + +Alternatively, one could also use Ollama's OpenAI-style endpoint using the `openai/` prefix instead of `ollama_chat/`. In this case, set the `OPENAI_API_BASE`, `OPENAI_API_KEY`, and `OPENAI_ORGANIZATION` (if necessary) environment variables. If you were using the official OpenAI API, you would only have to set the api key and possibly the organization. For local use e.g., using Ollama, this could look like so: + +```bash +export OPENAI_API_BASE=http://localhost:11434/v1 +export OPENAI_API_KEY=ollama # required, but unused +export OPENAI_ORGANIZATION=ollama # not required + +pdl <...> +``` + ## Strings In Yaml -Multiline strings are commonly used when writing PDL programs. However, their exact semantics are not intuitive. There are two types of formats that YAML supports for strings: block scalar and flow scalar formats. Scalars are what YAML calls basic values like numbers or strings, as opposed to complex types like arrays or objects. Block scalars have more control over how they are interpreted, whereas flow scalars have more limited escaping support. (Explanation here is thanks to [Wolfgang Faust](https://yaml-multiline.info/)) +Multiline strings are commonly used when writing PDL programs. There are two types of formats that YAML supports for strings: block scalar and flow scalar formats. Scalars are what YAML calls basic values like numbers or strings, as opposed to complex types like arrays or objects. Block scalars have more control over how they are interpreted, whereas flow scalars have more limited escaping support. (Explanation here is thanks to [Wolfgang Faust](https://yaml-multiline.info/)) ### Block Scalars @@ -1115,35 +931,7 @@ Several lines of text, with some "quotes" of various 'types'. Escapes (like \n) Newlines can be added by leaving a blank line. Additional leading whitespace is ignored. ``` -## Using Ollama models - -1. Install Ollama e.g., `brew install --cask ollama` -2. Run a model e.g., `ollama run granite-code:34b-instruct-q5_K_M`. See [the Ollama library for more models](https://ollama.com/library/granite-code/tags) -3. An OpenAI style server is running locally at [http://localhost:11434/](http://localhost:11434/), see [the Ollama blog](https://ollama.com/blog/openai-compatibility) for more details. - - -Example: - -``` -text: -- Hello, -- model: ollama_chat/granite-code:34b-instruct-q5_K_M - parameters: - stop: - - '!' - decoding_method: greedy -``` - -Alternatively, one could also use Ollama's OpenAI-style endpoint using the `openai/` prefix instead of `ollama_chat/`. In this case, set the `OPENAI_API_BASE`, `OPENAI_API_KEY`, and `OPENAI_ORGANIZATION` (if necessary) environment variables. If you were using the official OpenAI API, you would only have to set the api key and possibly the organization. For local use e.g., using Ollama, this could look like so: - -```bash -export OPENAI_API_BASE=http://localhost:11434/v1 -export OPENAI_API_KEY=ollama # required, but unused -export OPENAI_ORGANIZATION=ollama # not required - -pdl <...> -``` diff --git a/examples/callback/repair_prompt.pdl b/examples/callback/repair_prompt.pdl index 41601a65c..b110be54a 100644 --- a/examples/callback/repair_prompt.pdl +++ b/examples/callback/repair_prompt.pdl @@ -9,9 +9,10 @@ contribute: [context] - def: raw_output - model: watsonx/ibm/granite-34b-code-instruct + model: replicate/ibm-granite/granite-3.0-8b-instruct parameters: - stop: ["\n\n"] + stop_sequences: "\n\n" + temperature: 0 - lang: python def: parsed_output diff --git a/examples/chatbot/chatbot.pdl b/examples/chatbot/chatbot.pdl index 60d20ef2a..9950f7774 100644 --- a/examples/chatbot/chatbot.pdl +++ b/examples/chatbot/chatbot.pdl @@ -5,18 +5,18 @@ text: contribute: [context] - repeat: text: - - model: watsonx/ibm/granite-13b-chat-v2 + - model: replicate/ibm-granite/granite-3.0-8b-instruct - read: def: eval message: "\nIs this a good answer[yes/no]?\n" - contribute: [] + contribute: [context] - if: ${ eval == 'no' } then: text: - read: message: "Why not?\n" + contribute: [] until: ${ eval == 'yes'} - diff --git a/examples/cldk/README.md b/examples/cldk/README.md index 104a41a9f..ce34fd191 100644 --- a/examples/cldk/README.md +++ b/examples/cldk/README.md @@ -12,7 +12,7 @@ wget https://github.com/apache/commons-cli/archive/refs/tags/rel/commons-cli-1.7 To run the CLDK assistant: ``` -pdl examples/cldk/cldk-assistant.pdl +pdl cldk-assistant.pdl ``` Enter the path to the above Java project when prompted. Then you can make queries such as "what are all the classes?". \ No newline at end of file diff --git a/examples/cldk/cldk-assistant.pdl b/examples/cldk/cldk-assistant.pdl index 59524abea..a13ff2f04 100644 --- a/examples/cldk/cldk-assistant.pdl +++ b/examples/cldk/cldk-assistant.pdl @@ -14,11 +14,11 @@ text: from cldk import CLDK from cldk.models.java.models import * - # Initialize the Codellm-DevKit object with the project directory, language, and backend. + # Initialize the Codellm-DevKit object with the project directory, langguage, and backend. cldk = CLDK("java") cldk_state = cldk.analysis( project_path="${ project }", # Change this to the path of the project you want to analyze. - # language="java", # The language of the project. + # langguage="java", # The langguage of the project. # backend="codeanalyzer", # The backend to use for the analysis. # analysis_db="/tmp", # A temporary directory to store the analysis results. # sdg=True, # Generate the System Dependence Graph (SDG) for the project. @@ -27,141 +27,132 @@ text: result = cldk_state - " done!" - repeat: - - read: - def: query - message: "\n\nHow can I help you [Type 'quit' to quit]?\n" - contribute: [] - - "\n***Generating PDL code for your query:\n" - - if: ${ query != 'quit'} - then: - - model: watsonx/ibm/granite-20b-code-instruct - def: PDL - input: - - | - Question: What are all the classes? - Answer: - ``` - text: - - lang: python - code: - - | - classes = PDL_SESSION.cldk_state.get_classes().keys() - result = ", ".join(classes) - ``` + text: + - read: + def: query + message: "\n\nHow can I help you [Type 'quit' to quit]?\n" + contribute: [] + - "\n***Generating PDL code for your query:\n" + - if: ${ query != 'quit'} + then: + text: + - model: replicate/ibm-granite/granite-3.0-8b-instruct + def: PDL + input: + - | + Question: What are all the classes? + Answer: + ``` + text: + - lang: python + code: + - | + classes = PDL_SESSION.cldk_state.get_classes().keys() + result = ", ".join(classes) + ``` - Question: What are all the constructors of class org.ibm.App? - Answer: - ``` - text: - - lang: python - code: - - | - constructors = PDL_SESSION.cldk_state.get_constructors("org.ibm.App") - result = ", ".join(constructors) - ``` + Question: What are all the constructors of class org.ibm.App? + Answer: + ``` + text: + - lang: python + code: + - | + constructors = PDL_SESSION.cldk_state.get_constructors("org.ibm.App") + result = ", ".join(constructors) + ``` - Question: What are all the fields of class org.ibm.App? - Answer: - ``` - text: - - lang: python - code: - - | - fields = PDL_SESSION.cldk_state.get_fields("org.ibm.App") - names = sum([f.variables for f in fields], []) - result = ", ".join(names) - ``` + Question: What are all the fields of class org.ibm.App? + Answer: + ``` + text: + - lang: python + code: + - | + fields = PDL_SESSION.cldk_state.get_fields("org.ibm.App") + names = sum([f.variables for f in fields], []) + result = ", ".join(names) + ``` - Question: What are all the methods of class org.ibm.App? - Answer: - ``` - text: - - lang: python - code: - - | - methods = PDL_SESSION.cldk_state.get_methods_in_class("org.ibm.App") - result = ", ".join(methods) - ``` + Question: What are all the methods of class org.ibm.App? + Answer: + ``` + text: + - lang: python + code: + - | + methods = PDL_SESSION.cldk_state.get_methods_in_class("org.ibm.App") + result = ", ".join(methods) + ``` - Question: Show me the call graph of class "org.ibm.App" - Answer: - ``` - text: - - lang: python - code: - - | - graph = PDL_SESSION.cldk_state.get_class_call_graph("org.ibm.App", method_name=None) - result = graph - ``` + Question: Show me the call graph of class "org.ibm.App" + Answer: + ``` + text: + - lang: python + code: + - | + graph = PDL_SESSION.cldk_state.get_class_call_graph("org.ibm.App", method_name=None) + result = graph + ``` - Question: What is the code of method Foo(string) of class org.ibm.App? - Answer: - ``` - text: - - lang: python - code: - - | - method = PDL_SESSION.cldk_state.get_method("org.ibm.App", "Foo(string)") - result = method.code - ``` + Question: What is the code of method Foo(string) of class org.ibm.App? + Answer: + ``` + text: + - lang: python + code: + - | + method = PDL_SESSION.cldk_state.get_method("org.ibm.App", "Foo(string)") + result = method.code + ``` - Question: Generate a summary for method Foo(string) of class org.ibm.App - Answer: - ``` - text: - - lang: python - code: - - | - method = PDL_SESSION.cldk_state.get_method("org.ibm.App", "Foo(string)") - result = method - - "\n\nGenerate a summary of method Foo\n\n" - - model: watsonx/meta-llama/llama-3-1-70b-instruct - ``` + Question: Generate a summary for method Foo(string) of class org.ibm.App + Answer: + ``` + text: + - lang: python + code: + - | + method = PDL_SESSION.cldk_state.get_method("org.ibm.App", "Foo(string)") + result = method + - "\n\nGenerate a summary of method Foo\n\n" + - model: replicate/ibm-granite/granite-3.0-8b-instruct + ``` - Question: Generate a different comment for method Foo(string) in class org.ibm.App? - Answer: - ``` - text: - - lang: python - code: - - | - method = PDL_SESSION.cldk_state.get_method("org.ibm.App", "Foo(string)") - result = method - - "\n\nGenerate a different comment for method Foo(string)\n\n" - - model: watsonx/meta-llama/llama-3-1-70b-instruct - ``` + Question: Generate a different comment for method Foo(string) in class org.ibm.App? + Answer: + ``` + text: + - lang: python + code: + - | + method = PDL_SESSION.cldk_state.get_method("org.ibm.App", "Foo(string)") + result = method + - "\n\nGenerate a different comment for method Foo(string)\n\n" + - model: replicate/ibm-granite/granite-3.0-8b-instruct + ``` - If the query contains something about a field be sure to call a model. - - Question: ${ query } + If the query contains something about a field be sure to call a model. + + Question: ${ query } - parameters: - stop_sequence: ["Question"] - include_stop_sequence: false - - "\n\n***Executing the above PDL code:\n\n" - - lang: python - contribute: [] - code: | - from pdl import pdl_ast, pdl_interpreter - from pdl.pdl_ast import Program - from pdl.pdl_interpreter import process_prog - from pdl.pdl_interpreter import InterpreterState - from pdl.pdl_interpreter import empty_scope - import re - import yaml - s = """'${ PDL }'""" - print(s) - pdl = s.split("```")[1] - obj = yaml.safe_load(pdl) - state = InterpreterState() - data = Program.model_validate(obj) - - result, _, _, _ = process_prog(state, empty_scope, data) + parameters: + stop_sequences: "Question" + temperature: 0 + - "\n\n***Executing the above PDL code:\n\n" + - lang: python + contribute: [result] + code: | + from pdl.pdl import exec_str + s = """${ PDL }""" + pdl = s.split("```")[1] + result = exec_str(pdl) + until: ${ query == 'quit' } - - - + + \ No newline at end of file diff --git a/examples/code/README.md b/examples/code/README.md index c0c1ecb04..758c4ac8e 100644 --- a/examples/code/README.md +++ b/examples/code/README.md @@ -15,7 +15,7 @@ public static Map deserializeOffsetMap(String lastSourceOffset) } ``` -We have some information about the repository for this code stored in a JSON [file](examples/code/data.json), such as repository name, path and filename. Using that data, we wish to formulate a prompt as follows: +We have some information about the repository for this code stored in a JSON [file](./data.yaml), such as repository name, path and filename. Using that data, we wish to formulate a prompt as follows: ``` Here is some info about the location of the function in the repo. @@ -38,5 +38,5 @@ public static Map deserializeOffsetMap(String lastSourceOffset) } ``` -The PDL program for this example can be found [here](examples/code/code.yaml). This example also contains a comparison with some ground truth for evaluation and outputs a text similarity metric ([code-eval.pdl](examples/code/code-eval.yaml)). -The program [code-json.pdl](examples/code/code-json.yaml) outputs its results in JSON format. \ No newline at end of file +The PDL program for this example can be found [here](code.pdl). This example also contains a comparison with some ground truth for evaluation and outputs a text similarity metric ([code-eval.pdl](code-eval.pdl)). +The program [code-json.pdl](code-json.pdl) outputs its results in JSON format. \ No newline at end of file diff --git a/examples/code/code-eval.pdl b/examples/code/code-eval.pdl index 6d0a15c29..fcad0997e 100644 --- a/examples/code/code-eval.pdl +++ b/examples/code/code-eval.pdl @@ -7,7 +7,7 @@ defs: read: ./ground_truth.txt text: - "\n${ CODE.source_code }\n" -- model: watsonx/ibm/granite-34b-code-instruct +- model: replicate/ibm-granite/granite-3.0-8b-instruct def: EXPLANATION input: | Here is some info about the location of the function in the repo. @@ -20,6 +20,8 @@ text: Explain the following code: ``` ${ CODE.source_code }``` + parameters: + temperature: 0 - | diff --git a/examples/code/code-json.pdl b/examples/code/code-json.pdl index 915ff8f56..3caf38d4d 100644 --- a/examples/code/code-json.pdl +++ b/examples/code/code-json.pdl @@ -6,7 +6,7 @@ defs: TRUTH: read: ./ground_truth.txt text: -- model: watsonx/ibm/granite-34b-code-instruct +- model: replicate/ibm-granite/granite-3.0-8b-instruct def: EXPLANATION contribute: [] input: @@ -21,6 +21,8 @@ text: Explain the following code: ``` ${ CODE.source_code }``` + parameters: + temperature: 0 - def: EVAL contribute: [] lang: python diff --git a/examples/code/code.pdl b/examples/code/code.pdl index 1e0900f67..66f86ab79 100644 --- a/examples/code/code.pdl +++ b/examples/code/code.pdl @@ -5,7 +5,7 @@ defs: parser: yaml text: - "\n${ CODE.source_code }\n" -- model: watsonx/ibm/granite-34b-code-instruct +- model: replicate/ibm-granite/granite-3.0-8b-instruct input: - | Here is some info about the location of the function in the repo. @@ -17,4 +17,6 @@ text: Explain the following code: ``` - ${ CODE.source_code }``` \ No newline at end of file + ${ CODE.source_code }``` + parameters: + temperature: 0 \ No newline at end of file diff --git a/examples/demo/1-gen-data.pdl b/examples/demo/1-gen-data.pdl index 8cfd7851c..dfa2ffae0 100644 --- a/examples/demo/1-gen-data.pdl +++ b/examples/demo/1-gen-data.pdl @@ -5,7 +5,7 @@ defs: parser: yaml spec: { questions: [str], answers: [obj] } text: - - model: watsonx/ibm/granite-20b-code-instruct + - model: replicate/ibm-granite/granite-3.0-8b-instruct def: model_output spec: {name: str, age: int} input: @@ -22,7 +22,7 @@ text: and set them appropriately. Write the age in letters. parser: yaml parameters: - stop: - - '}' - include_stop_sequence: true + stop_sequences: "\n" + temperature: 0 + \ No newline at end of file diff --git a/examples/demo/2-teacher.pdl b/examples/demo/2-teacher.pdl index 889602bd0..b31763834 100644 --- a/examples/demo/2-teacher.pdl +++ b/examples/demo/2-teacher.pdl @@ -1,6 +1,6 @@ defs: teacher_sys_prompt: You are a very knowledgeable AI Assistant that will faithfully assist the user with their task. - teacher_model: watsonx/mistralai/mixtral-8x7b-instruct-v01 + teacher_model: replicate/ibm-granite/granite-3.0-8b-instruct teacher_template: function: sys_prompt: str @@ -67,8 +67,8 @@ defs: model: ${teacher_model} input: ${teacher_input} parameters: - stop: ["${teacher_stop_token}"] - include_stop_sequence: false + temperature: 0 + stop_sequences: "${teacher_stop_token}" max_new_tokens: ${prompt_data.max_new_tokens} parser: regex: '### Question [0-9]+:\s*([^#\n]+)' @@ -156,9 +156,9 @@ defs: model: ${teacher_model} input: ${teacher_input} parameters: - stop: ["${teacher_stop_token}"] - include_stop_sequence: false + stop_sequences: "${teacher_stop_token}" max_new_tokens: ${prompt_data.max_new_tokens} + temperature: 0 parser: spec: { "rating": str } # regex: "Rating.*\\[\\[(?P\\d+\\.?\\d*)\\]\\]" @@ -251,9 +251,9 @@ defs: model: ${teacher_model} input: ${teacher_input} parameters: - stop: ${ [teacher_stop_token] + prompt_data.additional_stop_tokens } - include_stop_sequence: false + stop_sequences: ${ ([teacher_stop_token] + prompt_data.additional_stop_tokens) | join(',') } max_new_tokens: ${prompt_data.max_new_tokens} + temperature: 0 parsed_answer: lang: python code: | # parse model output @@ -338,9 +338,9 @@ defs: model: ${teacher_model} input: ${teacher_input} parameters: - stop: ["${teacher_stop_token}"] - include_stop_sequence: false + stop_sequences: "${teacher_stop_token}" max_new_tokens: ${prompt_data.max_new_tokens} + temperature: 0 parser: spec: { "rating": str } regex: 'Rating.*\[\[(?P\d+\.?\d*)\]\]' diff --git a/examples/demo/3-weather.pdl b/examples/demo/3-weather.pdl index a7d59d005..266490b3b 100644 --- a/examples/demo/3-weather.pdl +++ b/examples/demo/3-weather.pdl @@ -4,21 +4,18 @@ text: def: QUERY message: "Ask a query: " contribute: [] -- model: watsonx/ibm/granite-34b-code-instruct +- model: replicate/ibm-granite/granite-3.0-8b-instruct input: | + Extract the location from the question. Question: What is the weather in London? - London + Answer: London Question: What's the weather in Paris? - Paris + Answer: Paris Question: Tell me the weather in Lagos? - Lagos + Answer: Lagos Question: ${ QUERY } parameters: - stop: - - Question - - What - - '!' - include_stop_sequence: false + stop_sequences: "Question,What,!,\n" def: LOCATION contribute: [] - lang: python @@ -30,7 +27,7 @@ text: parser: json contribute: [] -- model: watsonx/ibm/granite-34b-code-instruct +- model: replicate/ibm-granite/granite-3.0-8b-instruct input: | Explain the weather from the following JSON: ${ WEATHER } diff --git a/examples/demo/4-translator.pdl b/examples/demo/4-translator.pdl index 8cd2d2a01..a84af271e 100644 --- a/examples/demo/4-translator.pdl +++ b/examples/demo/4-translator.pdl @@ -1,11 +1,7 @@ description: PDL program text: -- What is APR? -- model: watsonx/ibm/granite-13b-chat-v2 - parameters: - max_new_tokens: 128 - stop: ["."] - include_stop_sequence: true +- "What is APR?\n" +- model: replicate/ibm-granite/granite-3.0-8b-instruct - repeat: text: - read: @@ -14,6 +10,6 @@ text: - if: ${ language != 'stop' } then: text: - - "\n\nTranslate the above to ${ language }" - - model: watsonx/ibm/granite-13b-chat-v2 + - "\n\nTranslate the above to ${ language }\n" + - model: replicate/ibm-granite/granite-3.0-8b-instruct until: ${ language == 'stop' } diff --git a/examples/demo/test.pdl b/examples/demo/test.pdl deleted file mode 100644 index 98e5ce3f2..000000000 --- a/examples/demo/test.pdl +++ /dev/null @@ -1,6 +0,0 @@ -description: Hello world calling a model -text: -- model: watsonx/ibm/granite-20b-multilingual - input: - Translate the word 'world' to French - \ No newline at end of file diff --git a/examples/fibonacci/fib.pdl b/examples/fibonacci/fib.pdl index 1671f5504..394fd46cf 100644 --- a/examples/fibonacci/fib.pdl +++ b/examples/fibonacci/fib.pdl @@ -1,8 +1,10 @@ description: Fibonacci text: - def: CODE - model: watsonx/ibm/granite-20b-code-instruct - input: "Write a Python function to compute the Fibonacci sequence\n\n" + model: replicate/ibm-granite/granite-3.0-8b-instruct + input: "Write a Python function to compute the Fibonacci sequence. Do not include a doc string.\n\n" + parameters: + temperature: 0 - "\nFind a random number between 1 and 20\n" - def: N @@ -10,9 +12,7 @@ text: code: | import random result = random.randint(1, 20) -- "\nNow computing fib(" -- get: N -- ")\n" +- "\nNow computing fibonacci(${ N })\n" - def: EXTRACTED lang: python code: | @@ -22,9 +22,9 @@ text: lang: python code: | ${ EXTRACTED } - result = fib(${ N }) + result = fibonacci(${ N }) contribute: [] -- 'The result is: ' -- get: RESULT +- 'The result is: ${ RESULT }' - "\n\nExplain what the above code does and what the result means\n\n" -- model: watsonx/ibm/granite-20b-code-instruct \ No newline at end of file +- model: replicate/ibm-granite/granite-3.0-8b-instruct + \ No newline at end of file diff --git a/examples/granite/multi_round_chat.pdl b/examples/granite/multi_round_chat.pdl index dcf93864f..e3acd7e07 100644 --- a/examples/granite/multi_round_chat.pdl +++ b/examples/granite/multi_round_chat.pdl @@ -6,15 +6,14 @@ text: def: prompts spec: {prompts: [str]} contribute: [] -- ${ SYSTEM_CONTENT_CHAT } - for: prompt: ${ prompts.prompts } repeat: text: - | + ${ prompt } - - model: watsonx/ibm/granite-13b-chat-v2 + - model: replicate/ibm-granite/granite-3.0-8b-instruct parameters: - decoding_method: sample - max_new_tokens: 512 + temperature: 0 role: user \ No newline at end of file diff --git a/examples/granite/single_round_chat.pdl b/examples/granite/single_round_chat.pdl index 5a5e1ee71..917a258ed 100644 --- a/examples/granite/single_round_chat.pdl +++ b/examples/granite/single_round_chat.pdl @@ -1,11 +1,8 @@ description: Granite Single-Round Chat text: - include: ./granite_defs.pdl -- role: system - content: ${ SYSTEM_CONTENT_CHAT } -- ${ PROMPT } -- model: watsonx/ibm/granite-13b-chat-v2 +- "${ PROMPT }\n" +- model: replicate/ibm-granite/granite-3.0-8b-instruct parameters: - decoding_method: sample - max_new_tokens: 512 + temperature: 0 role: user diff --git a/examples/hello/hello-data.pdl b/examples/hello/hello-data.pdl index 285e86bdd..86d744a9d 100644 --- a/examples/hello/hello-data.pdl +++ b/examples/hello/hello-data.pdl @@ -1,4 +1,4 @@ -# Call with pdl --data '"something": "ABC"' +# Call with pdl --data '"something": "ABC"' hello-data.pdl description: Hello world with data text: - def: stutter diff --git a/examples/hello/hello-def-use.pdl b/examples/hello/hello-def-use.pdl index 5d702d76a..cf3f187ea 100644 --- a/examples/hello/hello-def-use.pdl +++ b/examples/hello/hello-def-use.pdl @@ -1,13 +1,11 @@ description: Hello world with variable use text: -- Hello, -- model: watsonx/ibm/granite-34b-code-instruct +- "Hello\n" +- model: replicate/ibm-granite/granite-3.0-8b-instruct parameters: decoding_method: greedy - stop: - - '!' - include_stop_sequence: false - def: NAME + stop_sequences: '!' + def: GEN - | - Who is${ NAME }? \ No newline at end of file + You said ${ GEN }. \ No newline at end of file diff --git a/examples/hello/hello-model-chaining.pdl b/examples/hello/hello-model-chaining.pdl index 78fd0b335..a0fff6f14 100644 --- a/examples/hello/hello-model-chaining.pdl +++ b/examples/hello/hello-model-chaining.pdl @@ -1,24 +1,14 @@ description: Hello world showing model chaining text: -- Hello, -- model: watsonx/ibm/granite-34b-code-instruct +- "Hello\n" +- model: replicate/ibm-granite/granite-3.0-8b-instruct parameters: decoding_method: greedy - stop: - - '!' - include_stop_sequence: false - def: NAME -- |- - ! - Who is${ NAME }? -- "\n" -- model: watsonx/google/flan-t5-xl + stop_sequences: '!' + def: GEN +- "\nDid you say ${ GEN }?\n" +- model: replicate/ibm-granite/granite-3.0-8b-instruct parameters: decoding_method: greedy - stop: - - '.' - include_stop_sequence: true - roles: - user: - pre_message: "" - post_message: "" + stop_sequences: '.' + \ No newline at end of file diff --git a/examples/hello/hello-model-input.pdl b/examples/hello/hello-model-input.pdl index 22f2e910e..acb7ae367 100644 --- a/examples/hello/hello-model-input.pdl +++ b/examples/hello/hello-model-input.pdl @@ -1,8 +1,6 @@ description: Hello world with model input text: -- model: watsonx/ibm/granite-34b-code-instruct +- model: replicate/ibm-granite/granite-3.0-8b-instruct input: "Hello," parameters: - stop: - - '!' - include_stop_sequence: true + stop_sequences: '!' diff --git a/examples/hello/hello-parser-json.pdl b/examples/hello/hello-parser-json.pdl index b6cbe8952..875b53e6e 100644 --- a/examples/hello/hello-parser-json.pdl +++ b/examples/hello/hello-parser-json.pdl @@ -3,12 +3,12 @@ defs: data: read: ./hello-parser-json-data.yaml parser: yaml - spec: { "questions": ["str"], "answers": ["obj"] } + spec: { questions: [str], answers: [obj] } text: - - model: watsonx/ibm/granite-34b-code-instruct + - model: replicate/ibm-granite/granite-3.0-8b-instruct def: model_output - spec: {"name": str, "age": int} - input: + spec: {name: str, age: int} + input: text: - for: question: ${ data.questions } @@ -19,14 +19,9 @@ text: ${ answer } - > Question: Create a JSON object with fields 'name' and 'age' - and set them appropriately. + and set them appropriately. Write the age in letters. parser: yaml parameters: - decoding_method: greedy - stop: - - '}' - include_stop_sequence: true - - - + stop_sequences: "\n" + temperature: 0 diff --git a/examples/hello/hello-parser-regex.pdl b/examples/hello/hello-parser-regex.pdl index f9a59a825..a62c537fd 100644 --- a/examples/hello/hello-parser-regex.pdl +++ b/examples/hello/hello-parser-regex.pdl @@ -1,12 +1,9 @@ description: Hello world with parser using regex text: -- model: watsonx/ibm/granite-34b-code-instruct +- model: replicate/ibm-granite/granite-3.0-8b-instruct input: "Hello," parameters: - decoding_method: greedy - stop: - - '!' - include_stop_sequence: false + stop_sequences: '!' spec: {"name": str} parser: spec: diff --git a/examples/hello/hello-roles-array.pdl b/examples/hello/hello-roles-array.pdl index 7fa6f9250..82c254a96 100644 --- a/examples/hello/hello-roles-array.pdl +++ b/examples/hello/hello-roles-array.pdl @@ -7,6 +7,6 @@ text: - role: user content: Write a Python function that implement merge sort. contribute: [] -- model: watsonx/ibm/granite-8b-code-instruct +- model: replicate/ibm-granite/granite-3.0-8b-instruct input: ${ prompt } diff --git a/examples/hello/hello-type.pdl b/examples/hello/hello-type.pdl index 61f4d0d0a..dcdffd3af 100644 --- a/examples/hello/hello-type.pdl +++ b/examples/hello/hello-type.pdl @@ -1,12 +1,7 @@ description: Hello world with type specification text: -- Hello, -- model: watsonx/ibm/granite-34b-code-instruct - def: GEN - parameters: - stop: - - '!' - include_stop_sequence: true +- def: GEN + text: "What is the meaning of life" - def: translate function: sentence: str @@ -14,19 +9,16 @@ text: spec: int return: - "\nTranslate the sentence '${ sentence }' to ${ language }\n" - - model: watsonx/ibm/granite-20b-multilingual + - model: replicate/ibm-granite/granite-3.0-8b-instruct parameters: - stop: - - "\n" - include_stop_sequence: true - + stop_sequences: "\n" - call: translate spec: str args: - sentence: Hello,${ GEN } + sentence: ${ GEN } language: French - call: translate args: - sentence: Bye,${ GEN } + sentence: ${ GEN } language: Spanish diff --git a/examples/hello/hello.pdl b/examples/hello/hello.pdl index e687a7548..7441cbafd 100644 --- a/examples/hello/hello.pdl +++ b/examples/hello/hello.pdl @@ -1,9 +1,4 @@ description: Hello world text: -- Hello, -- model: watsonx/ibm/granite-34b-code-instruct - parameters: - decoding_method: greedy - stop: - - '!' - include_stop_sequence: true \ No newline at end of file +- "Hello\n" +- model: replicate/ibm-granite/granite-3.0-8b-instruct diff --git a/examples/input/input_test1.pdl b/examples/input/input_test1.pdl index 637c04aab..ae94e94d9 100644 --- a/examples/input/input_test1.pdl +++ b/examples/input/input_test1.pdl @@ -4,4 +4,6 @@ text: - read: message: "Please provide an input: " def: STDIN -- get: STDIN + contribute: [] +- ${ STDIN } + diff --git a/examples/notebooks/demo.ipynb b/examples/notebooks/demo.ipynb new file mode 100644 index 000000000..da1bd1d9b --- /dev/null +++ b/examples/notebooks/demo.ipynb @@ -0,0 +1,491 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "a29514fc-9383-490c-8a73-63c547c21d95", + "metadata": {}, + "source": [ + "# Prompt Declaration Language\n", + "\n", + "Prompt engineering is difficult: minor variations in prompts have large impacts on the output of LLMs and prompts are model-dependent. In recent years prompt programming languages have emerged to bring discipline to prompt engineering. Many of them are embedded in an imperative language such as Python or TypeScript, making it difficult for users to directly interact with prompts and multi-turn LLM interactions.\n", + "\n", + "The Prompt Declaration Language (PDL) is a YAML-based declarative approach to prompt programming, where prompts are at the forefront. PDL facilitates model chaining and tool use, abstracting away the plumbing necessary for such compositions, enables type checking of the input and output of models, and is based on LiteLLM to support a variety of model providers. PDL has been used with RAG, CoT, ReAct, and an agent for solving SWE-bench. PDL is [open-source](https://github.com/IBM/prompt-declaration-language) and works well with watsonx.ai and Granite models.\n", + "\n", + "All examples in this notebook use the new ibm/granite-8b-instruct-preview-4k model. You can use PDL stand-alone or from a Python SDK or, as shown here, in a notebook via a notebook extension. In the cell output, model-generated text is rendered in green font, and tool-generated text is rendered in purple font." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "ca2e7ba6-e0f0-4d88-a083-5ff257ed2c34", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: prompt-declaration-language[examples] in /Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages (0.0.7)\n", + "Requirement already satisfied: pydantic~=2.0 in /Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages (from prompt-declaration-language[examples]) (2.9.1)\n", + "Requirement already satisfied: ibm-generative-ai~=3.0 in /Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages (from prompt-declaration-language[examples]) (3.0.0)\n", + "Requirement already satisfied: requests~=2.0 in /Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages (from prompt-declaration-language[examples]) (2.32.3)\n", + "Requirement already satisfied: python-dotenv~=1.0 in /Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages (from prompt-declaration-language[examples]) (1.0.1)\n", + "Requirement already satisfied: jinja2~=3.0 in /Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages (from prompt-declaration-language[examples]) (3.1.4)\n", + "Requirement already satisfied: PyYAML~=6.0 in /Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages (from prompt-declaration-language[examples]) (6.0.2)\n", + "Requirement already satisfied: jsonschema~=4.0 in /Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages (from prompt-declaration-language[examples]) (4.23.0)\n", + "Requirement already satisfied: litellm~=1.49 in /Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages (from prompt-declaration-language[examples]) (1.49.2)\n", + "Requirement already satisfied: termcolor~=2.0 in /Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages (from prompt-declaration-language[examples]) (2.4.0)\n", + "Requirement already satisfied: ipython~=8.0 in /Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages (from prompt-declaration-language[examples]) (8.27.0)\n", + "Requirement already satisfied: wikipedia~=1.0 in /Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages (from prompt-declaration-language[examples]) (1.4.0)\n", + "Requirement already satisfied: textdistance~=4.0 in /Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages (from prompt-declaration-language[examples]) (4.6.3)\n", + "Requirement already satisfied: faiss-cpu~=1.0 in /Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages (from prompt-declaration-language[examples]) (1.8.0.post1)\n", + "Requirement already satisfied: datasets<4,>2 in /Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages (from prompt-declaration-language[examples]) (3.0.1)\n", + "Requirement already satisfied: sympy~=1.0 in /Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages (from prompt-declaration-language[examples]) (1.13.2)\n", + "Requirement already satisfied: filelock in /Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages (from datasets<4,>2->prompt-declaration-language[examples]) (3.16.0)\n", + "Requirement already satisfied: numpy>=1.17 in /Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages (from datasets<4,>2->prompt-declaration-language[examples]) (1.26.4)\n", + "Requirement already satisfied: pyarrow>=15.0.0 in /Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages (from datasets<4,>2->prompt-declaration-language[examples]) (17.0.0)\n", + "Requirement already satisfied: dill<0.3.9,>=0.3.0 in /Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages (from datasets<4,>2->prompt-declaration-language[examples]) (0.3.8)\n", + "Requirement already satisfied: pandas in /Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages (from datasets<4,>2->prompt-declaration-language[examples]) (2.2.2)\n", + "Requirement already satisfied: tqdm>=4.66.3 in /Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages (from datasets<4,>2->prompt-declaration-language[examples]) (4.66.5)\n", + "Requirement already satisfied: xxhash in /Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages (from datasets<4,>2->prompt-declaration-language[examples]) (3.5.0)\n", + "Requirement already satisfied: multiprocess in /Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages (from datasets<4,>2->prompt-declaration-language[examples]) (0.70.16)\n", + "Requirement already satisfied: fsspec<=2024.6.1,>=2023.1.0 in /Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages (from fsspec[http]<=2024.6.1,>=2023.1.0->datasets<4,>2->prompt-declaration-language[examples]) (2024.6.1)\n", + "Requirement already satisfied: aiohttp in /Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages (from datasets<4,>2->prompt-declaration-language[examples]) (3.10.5)\n", + "Requirement already satisfied: huggingface-hub>=0.22.0 in /Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages (from datasets<4,>2->prompt-declaration-language[examples]) (0.24.7)\n", + "Requirement already satisfied: packaging in /Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages (from datasets<4,>2->prompt-declaration-language[examples]) (24.1)\n", + "Requirement already satisfied: aiolimiter<2.0.0,>=1.1.0 in /Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages (from ibm-generative-ai~=3.0->prompt-declaration-language[examples]) (1.1.0)\n", + "Requirement already satisfied: deprecated<2.0.0,>=1.2.14 in /Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages (from ibm-generative-ai~=3.0->prompt-declaration-language[examples]) (1.2.14)\n", + "Requirement already satisfied: httpx<0.28.0,>=0.27.0 in /Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages (from ibm-generative-ai~=3.0->prompt-declaration-language[examples]) (0.27.2)\n", + "Requirement already satisfied: httpx-sse<0.5.0,>=0.4.0 in /Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages (from ibm-generative-ai~=3.0->prompt-declaration-language[examples]) (0.4.0)\n", + "Requirement already satisfied: decorator in /Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages (from ipython~=8.0->prompt-declaration-language[examples]) (5.1.1)\n", + "Requirement already satisfied: jedi>=0.16 in /Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages (from ipython~=8.0->prompt-declaration-language[examples]) (0.19.1)\n", + "Requirement already satisfied: matplotlib-inline in /Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages (from ipython~=8.0->prompt-declaration-language[examples]) (0.1.7)\n", + "Requirement already satisfied: prompt-toolkit<3.1.0,>=3.0.41 in /Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages (from ipython~=8.0->prompt-declaration-language[examples]) (3.0.47)\n", + "Requirement already satisfied: pygments>=2.4.0 in /Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages (from ipython~=8.0->prompt-declaration-language[examples]) (2.18.0)\n", + "Requirement already satisfied: stack-data in /Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages (from ipython~=8.0->prompt-declaration-language[examples]) (0.6.3)\n", + "Requirement already satisfied: traitlets>=5.13.0 in /Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages (from ipython~=8.0->prompt-declaration-language[examples]) (5.14.3)\n", + "Requirement already satisfied: pexpect>4.3 in /Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages (from ipython~=8.0->prompt-declaration-language[examples]) (4.9.0)\n", + "Requirement already satisfied: MarkupSafe>=2.0 in /Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages (from jinja2~=3.0->prompt-declaration-language[examples]) (2.1.5)\n", + "Requirement already satisfied: attrs>=22.2.0 in /Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages (from jsonschema~=4.0->prompt-declaration-language[examples]) (24.2.0)\n", + "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages (from jsonschema~=4.0->prompt-declaration-language[examples]) (2023.12.1)\n", + "Requirement already satisfied: referencing>=0.28.4 in /Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages (from jsonschema~=4.0->prompt-declaration-language[examples]) (0.35.1)\n", + "Requirement already satisfied: rpds-py>=0.7.1 in /Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages (from jsonschema~=4.0->prompt-declaration-language[examples]) (0.20.0)\n", + "Requirement already satisfied: click in /Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages (from litellm~=1.49->prompt-declaration-language[examples]) (8.1.7)\n", + "Requirement already satisfied: importlib-metadata>=6.8.0 in /Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages (from litellm~=1.49->prompt-declaration-language[examples]) (8.4.0)\n", + "Requirement already satisfied: openai>=1.51.0 in /Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages (from litellm~=1.49->prompt-declaration-language[examples]) (1.51.0)\n", + "Requirement already satisfied: tiktoken>=0.7.0 in /Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages (from litellm~=1.49->prompt-declaration-language[examples]) (0.7.0)\n", + "Requirement already satisfied: tokenizers in /Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages (from litellm~=1.49->prompt-declaration-language[examples]) (0.19.1)\n", + "Requirement already satisfied: annotated-types>=0.6.0 in /Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages (from pydantic~=2.0->prompt-declaration-language[examples]) (0.7.0)\n", + "Requirement already satisfied: pydantic-core==2.23.3 in /Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages (from pydantic~=2.0->prompt-declaration-language[examples]) (2.23.3)\n", + "Requirement already satisfied: typing-extensions>=4.6.1 in /Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages (from pydantic~=2.0->prompt-declaration-language[examples]) (4.12.2)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages (from requests~=2.0->prompt-declaration-language[examples]) (3.3.2)\n", + "Requirement already satisfied: idna<4,>=2.5 in /Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages (from requests~=2.0->prompt-declaration-language[examples]) (3.10)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in /Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages (from requests~=2.0->prompt-declaration-language[examples]) (1.26.20)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages (from requests~=2.0->prompt-declaration-language[examples]) (2024.8.30)\n", + "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages (from sympy~=1.0->prompt-declaration-language[examples]) (1.3.0)\n", + "Requirement already satisfied: beautifulsoup4 in /Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages (from wikipedia~=1.0->prompt-declaration-language[examples]) (4.12.3)\n", + "Requirement already satisfied: wrapt<2,>=1.10 in /Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages (from deprecated<2.0.0,>=1.2.14->ibm-generative-ai~=3.0->prompt-declaration-language[examples]) (1.16.0)\n", + "Requirement already satisfied: aiohappyeyeballs>=2.3.0 in /Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages (from aiohttp->datasets<4,>2->prompt-declaration-language[examples]) (2.4.0)\n", + "Requirement already satisfied: aiosignal>=1.1.2 in /Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages (from aiohttp->datasets<4,>2->prompt-declaration-language[examples]) (1.3.1)\n", + "Requirement already satisfied: frozenlist>=1.1.1 in /Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages (from aiohttp->datasets<4,>2->prompt-declaration-language[examples]) (1.4.1)\n", + "Requirement already satisfied: multidict<7.0,>=4.5 in /Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages (from aiohttp->datasets<4,>2->prompt-declaration-language[examples]) (6.1.0)\n", + "Requirement already satisfied: yarl<2.0,>=1.0 in /Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages (from aiohttp->datasets<4,>2->prompt-declaration-language[examples]) (1.11.1)\n", + "Requirement already satisfied: anyio in /Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages (from httpx<0.28.0,>=0.27.0->ibm-generative-ai~=3.0->prompt-declaration-language[examples]) (4.4.0)\n", + "Requirement already satisfied: httpcore==1.* in /Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages (from httpx<0.28.0,>=0.27.0->ibm-generative-ai~=3.0->prompt-declaration-language[examples]) (1.0.5)\n", + "Requirement already satisfied: sniffio in /Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages (from httpx<0.28.0,>=0.27.0->ibm-generative-ai~=3.0->prompt-declaration-language[examples]) (1.3.1)\n", + "Requirement already satisfied: h11<0.15,>=0.13 in /Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages (from httpcore==1.*->httpx<0.28.0,>=0.27.0->ibm-generative-ai~=3.0->prompt-declaration-language[examples]) (0.14.0)\n", + "Requirement already satisfied: zipp>=0.5 in /Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages (from importlib-metadata>=6.8.0->litellm~=1.49->prompt-declaration-language[examples]) (3.20.1)\n", + "Requirement already satisfied: parso<0.9.0,>=0.8.3 in /Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages (from jedi>=0.16->ipython~=8.0->prompt-declaration-language[examples]) (0.8.4)\n", + "Requirement already satisfied: distro<2,>=1.7.0 in /Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages (from openai>=1.51.0->litellm~=1.49->prompt-declaration-language[examples]) (1.9.0)\n", + "Requirement already satisfied: jiter<1,>=0.4.0 in /Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages (from openai>=1.51.0->litellm~=1.49->prompt-declaration-language[examples]) (0.5.0)\n", + "Requirement already satisfied: ptyprocess>=0.5 in /Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages (from pexpect>4.3->ipython~=8.0->prompt-declaration-language[examples]) (0.7.0)\n", + "Requirement already satisfied: wcwidth in /Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages (from prompt-toolkit<3.1.0,>=3.0.41->ipython~=8.0->prompt-declaration-language[examples]) (0.2.13)\n", + "Requirement already satisfied: regex>=2022.1.18 in /Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages (from tiktoken>=0.7.0->litellm~=1.49->prompt-declaration-language[examples]) (2024.7.24)\n", + "Requirement already satisfied: soupsieve>1.2 in /Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages (from beautifulsoup4->wikipedia~=1.0->prompt-declaration-language[examples]) (2.6)\n", + "Requirement already satisfied: python-dateutil>=2.8.2 in /Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages (from pandas->datasets<4,>2->prompt-declaration-language[examples]) (2.9.0.post0)\n", + "Requirement already satisfied: pytz>=2020.1 in /Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages (from pandas->datasets<4,>2->prompt-declaration-language[examples]) (2024.2)\n", + "Requirement already satisfied: tzdata>=2022.7 in /Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages (from pandas->datasets<4,>2->prompt-declaration-language[examples]) (2024.1)\n", + "Requirement already satisfied: executing>=1.2.0 in /Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages (from stack-data->ipython~=8.0->prompt-declaration-language[examples]) (2.1.0)\n", + "Requirement already satisfied: asttokens>=2.1.0 in /Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages (from stack-data->ipython~=8.0->prompt-declaration-language[examples]) (2.4.1)\n", + "Requirement already satisfied: pure-eval in /Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages (from stack-data->ipython~=8.0->prompt-declaration-language[examples]) (0.2.3)\n", + "Requirement already satisfied: six>=1.12.0 in /Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages (from asttokens>=2.1.0->stack-data->ipython~=8.0->prompt-declaration-language[examples]) (1.16.0)\n" + ] + } + ], + "source": [ + "! pip install 'prompt-declaration-language[examples]'" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "e25a6874-54d9-4167-82ed-ab2f4fdc0a6f", + "metadata": {}, + "outputs": [], + "source": [ + "%load_ext pdl.pdl_notebook_ext" + ] + }, + { + "cell_type": "markdown", + "id": "0f4a750b-5765-4e2a-9dc1-9c9af2eab940", + "metadata": {}, + "source": [ + "## Model call\n", + "\n", + "In PDL, the user specifies step-by-step the shape of data they want to generate. In the following, the `text` construct indicates a text block containing a prompt and a model call. Implicitly, PDL builds a background conversational context (list of role/content) which is used to make model calls. Each model call uses the context built so far as its input prompt." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "f3c62df1-0347-4711-acd7-3892cfd5df30", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "What is the meaning of life?\n", + "\u001b[32mThe meaning of life is a philosophical and metaphysical question related to the purpose or significance of life or existence in general. This question has been asked for centuries and does not have a definitive answer. Some people find meaning through personal growth, relationships, love, or through contributing to the betterment of humanity. Others may find\u001b[0m\u001b[32m it through spirituality or religious beliefs. Ultimately, the meaning of life may be something personal and subjective.\u001b[0m" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n" + ] + } + ], + "source": [ + "%%pdl --reset-context\n", + "text: \n", + "- \"What is the meaning of life?\\n\"\n", + "- model: \"replicate/ibm-granite/granite-3.0-8b-instruct\"" + ] + }, + { + "cell_type": "markdown", + "id": "1e41eb56-e278-4024-9979-7c3410e9ccf5", + "metadata": {}, + "source": [ + "## Model chaining\n", + "Model chaining can be done by simply adding to the list of models to call declaratively. Since this cell has the `%%pdl` cell magic without `--reset-context`, it executes in the context created by the previous cell." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "7f6c323b-ad1a-4434-8732-bc19c5c47883", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Say it like a poem\n", + "\u001b[32mLife's meaning, a question vast,\n", + "In philosophy, it's often cast.\n", + "Personal growth, love, or a cause,\n", + "Some find meaning in life's applause.\n", + "\n", + "Spirituality, faith, or belief,\n", + "Can guide us, like a gentle relief.\n", + "Meaning may be subjective,\n", + "\u001b[0m\u001b[32mYet, in our hearts, it's subjective.\u001b[0m\n", + "\n", + "Translate it to French\n", + "\u001b[32mLa signification de la vie, une question vaste,\n", + "Dans la philosophie, elle est souvent lancée.\n", + "La croissance personnelle, l'amour, ou une cause,\n", + "Certains trouvent une signification dans la vie's applaudissement.\u001b[0m\u001b[32m\n", + "\n", + "Spiritualité, foi, ou croyance,\n", + "Pouvant nous guider, comme une douce répit.\n", + "La signification peut être subjective,\n", + "Yet, dans nos cœurs, elle est subjective.\u001b[0m" + ] + } + ], + "source": [ + "%%pdl\n", + "text:\n", + "- \"\\nSay it like a poem\\n\"\n", + "- model: \"replicate/ibm-granite/granite-3.0-8b-instruct\"\n", + "- \"\\n\\nTranslate it to French\\n\"\n", + "- model: \"replicate/ibm-granite/granite-3.0-8b-instruct\"" + ] + }, + { + "cell_type": "markdown", + "id": "fe9b1959-e49b-48d3-b722-19ace9b981d2", + "metadata": {}, + "source": [ + "## Chat templates\n", + "\n", + "The second call to the model in the above program submits the following prompt. PDL takes care of applying the appropriate chat templates and tags, and builds the background context implicitly. Chat templates make your program easier to port across models, since you do not need to specify control tokens by hand. All the user has to do is list the models they want to chain, PDL takes care of the rest.\n", + "\n", + "```\n", + "<|start_of_role|>user<|end_of_role|>What is the meaning of life?\n", + "<|end_of_text|>\n", + "The meaning of life is a philosophical and metaphysical question related to the purpose or significance of life or existence in general. This concept has been approached by many perspectives including philosophy, religion, and science. Some people find meaning through personal growth, relationships, love, and through helping others. Others seek meaning through spirituality or religious beliefs. Ultimately, the meaning of life may be a personal and subjective experience.\n", + "\n", + "<|start_of_role|>user<|end_of_role|>Say it like a poem<|end_of_text|>\n", + "Life's meaning, a question vast,\n", + "In philosophy, religion, and science cast.\n", + "Some find purpose in personal growth,\n", + "In love and relationships, they find their troth.\n", + "Others seek meaning through spirituality,\n", + "In faith and belief, they find their reality.\n", + "Ultimately, meaning is a personal quest,\n", + "In life's journey, we are put to the test.\n", + "\n", + "<|start_of_role|>user<|end_of_role|>Translate it to French\n", + "<|end_of_text|>\n", + "<|start_of_role|>assistant<|end_of_role|>\n", + "```" + ] + }, + { + "cell_type": "markdown", + "id": "99681db4-8a43-4b06-92a6-8d140989f2ea", + "metadata": {}, + "source": [ + "## Data pipeline\n", + "\n", + "The following program shows a common prompting pattern: read some data, formulate a prompt using that data, submit to a model, and evaluate. In this program, we formulate a prompt for code explanation. The program first defines two variables: `code`, which holds the data we read, and `truth` for the ground truth. It then prints out the source code, formulates a prompts with the data, and calls a model to get an explanation. Finally, a Python code block uses the Levenshtein text distance metric and evaluate the explanation against the ground truth. This pipeline can similarly be applied to an entire data set to produce a jsonl file." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "b61b2e25-72a4-4f70-ae83-40d77bed3f4f", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "@SuppressWarnings(\"unchecked\")\n", + "public static Map deserializeOffsetMap(String lastSourceOffset) throws IOException {\n", + " Map offsetMap;\n", + " if (lastSourceOffset == null || lastSourceOffset.isEmpty()) { \n", + " offsetMap = new HashMap<>(); \n", + " } else {\n", + " offsetMap = JSON_MAPPER.readValue(lastSourceOffset, Map.class); \n", + " }\n", + " return offsetMap;\n", + "}\n", + "\n", + "\u001b[32mThis Java function, `deserializeOffsetMap`, is designed to convert a JSON string into a `Map`. Here's a breakdown of what it does:\n", + "\n", + "1. It takes a single argument, `lastSourceOffset`, which is expected to be a JSON string.\n", + "2. It initializes a `Map` called `offsetMap`.\n", + "3. If `lastSourceOffset` is either `null` or an empty string, it creates a new `HashMap` and assigns it to `offsetMap`.\n", + "4. If `lastSourceOffset` is not `null` or empty, it uses Jackson's `JSON_MAPPER` to convert the JSON string into a `Map\u001b[0m\u001b[32m` and assigns it to `offsetMap`.\n", + "5. Finally, it returns the `offsetMap`.\n", + "\n", + "The `@SuppressWarnings(\"unchecked\")` annotation is used to suppress a compile-time warning about the raw use of the `Map` type. This is because the `JSON_MAPPER.readValue` method returns a `Map` of `Object` and\u001b[0m\u001b[32m `Object`, which is then cast to `Map`.\u001b[0m\n", + "Evaluation:\n", + "The similarity (Levenshtein) between this answer and the ground truth is:\n", + "\u001b[35m0.3113839285714286\u001b[0m" + ] + } + ], + "source": [ + "%%pdl\n", + "defs:\n", + " code:\n", + " read: ./data.yaml\n", + " parser: yaml\n", + " truth:\n", + " read: ./ground_truth.txt\n", + "text:\n", + "- \"\\n${ code.source_code }\\n\"\n", + "- model: \"replicate/ibm-granite/granite-3.0-8b-instruct\"\n", + " def: explanation\n", + " input: |\n", + " Here is some info about the location of the function in the repo.\n", + " repo: \n", + " ${ code.repo_info.repo }\n", + " path: ${ code.repo_info.path }\n", + " Function_name: ${ code.repo_info.function_name }\n", + "\n", + "\n", + " Explain the following code:\n", + " ```\n", + " ${ code.source_code }```\n", + "- |\n", + "\n", + " Evaluation:\n", + " The similarity (Levenshtein) between this answer and the ground truth is:\n", + "- def: EVAL\n", + " lang: python\n", + " code: |\n", + " import textdistance\n", + " expl = \"\"\"\n", + " ${ explanation }\n", + " \"\"\"\n", + " truth = \"\"\"\n", + " ${ truth }\n", + " \"\"\"\n", + " result = textdistance.levenshtein.normalized_similarity(expl, truth)\n" + ] + }, + { + "cell_type": "markdown", + "id": "41a0dd93-febb-408b-ae22-e829e02906e9", + "metadata": {}, + "source": [ + "## Agentic Flow\n", + "\n", + "The following PDL program shows an agentic flow with a ReAct prompt pattern. It first reads some demonstrations to be used as few-shots. The ReAct pattern is captured with PDL control structures (repeat-until and if-then-else), and consists of cycling through thoughts, actions, and observations. The tools available are Wikipedia search, and calculator (as Python code). The agent decides when to search and when to calculate. The `spec` indicates a type for the output of the model when actions are produced, it is used to dynamically check outputs of models and fail when they don't conform to the expectation. " + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "dfef7096-b7a6-4966-8356-a306e701974b", + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "How many years ago was the discoverer of the Hudson River born? Keep in mind we are in 2024.\n", + "\u001b[32mTho: I need to search Henry Hudson, find out when he was born, and then calculate how many years ago that was.\n", + "\u001b[0m" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[32mAct: {\"name\": \"Search\", \"arguments\": {\"topic\": \"Henry Hudson\"}}\u001b[0m\n", + "Obs: \u001b[35mHenry Hudson (c. 1565 – disappeared 23 June 1611) was an English sea explorer and navigator during the early 17th century, best known for his explorations of present-day Canada and parts of the Northeastern United States.\n", + "In 1607 and 1608, Hudson made two attempts on behalf of English merchants to find a rumoured Northeast Passage to Cathay via a route above the Arctic Circle. In 1609, he landed in North America on behalf of the Dutch East India Company and explored the region around the modern New York metropolitan area. Looking for a Northwest Passage to Asia on his ship Halve Maen (\"Half Moon\"), he sailed up the Hudson River, which was later named after him, and thereby laid the foundation for Dutch colonization of the region. His contributions to the exploration of the New World were significant and lasting. His voyages helped to establish European contact with the native peoples of North America and contributed to the development of trade and commerce. \n", + "On his final expedition, while still searching for the Northwest Passage, Hudson became the first European to see Hudson Strait and the immense Hudson Bay. In 1611, after wintering on the shore of James Bay, Hudson wanted to press on to the west, but most of his crew mutinied. The mutineers cast Hudson, his son, and six others adrift; the Hudsons and their companions were never seen again.\u001b[0m\n", + "\u001b[32mTho: Henry Hudson was born around 1565.\n", + "\u001b[0m\u001b[32mAct: {\"name\": \"Calc\", \"arguments\": {\"expr\": \"2024 - 1565\"}}\u001b[0m\n", + "Obs: \u001b[35m459\u001b[0m\n" + ] + } + ], + "source": [ + "%%pdl --reset-context\n", + "text:\n", + "- read: demonstrations.txt\n", + " contribute: [context]\n", + "- \"How many years ago was the discoverer of the Hudson River born? Keep in mind we are in 2024.\\n\"\n", + "- repeat:\n", + " text:\n", + " - def: thought\n", + " model: replicate/ibm-granite/granite-3.0-8b-instruct\n", + " parameters:\n", + " stop_sequences: \"Act:\"\n", + " temperature: 0\n", + " - def: rawAction\n", + " model: replicate/ibm-granite/granite-3.0-8b-instruct\n", + " parameters:\n", + " stop_sequences: \"\\n\"\n", + " temperature: 0\n", + " - def: action\n", + " lang: python\n", + " parser: json\n", + " spec: {name: str, arguments: obj}\n", + " contribute: [context]\n", + " code:\n", + " |\n", + " result = '${ rawAction }'.replace(\"Act: \", \"\")\n", + " - def: observation\n", + " if: ${ action.name == \"Search\" }\n", + " then:\n", + " text:\n", + " - \"\\nObs: \"\n", + " - lang: python\n", + " code: |\n", + " import warnings, wikipedia\n", + " warnings.simplefilter(\"ignore\")\n", + " try:\n", + " result = wikipedia.summary(\"${ action.arguments.topic }\")\n", + " except wikipedia.WikipediaException as e:\n", + " result = str(e)\n", + " - \"\\n\"\n", + " else:\n", + " - if: ${ action.name == \"Calc\" }\n", + " then:\n", + " text:\n", + " - \"\\nObs: \"\n", + " - lang: python\n", + " code: result = ${ action.arguments.expr }\n", + " - \"\\n\"\n", + " until: ${ action.name != \"Search\" }\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9b919506-79ed-4a13-ac1d-1fdfc3e4f9d9", + "metadata": {}, + "outputs": [], + "source": [ + "! cat log.txt" + ] + }, + { + "cell_type": "markdown", + "id": "89438b62-29e4-472e-89ec-57c1626ffd44", + "metadata": {}, + "source": [ + "## Conclusion\n", + "\n", + "Since prompts are at the forefront, PDL makes users more productive in their trial-and-error with LLMs. Try it!\n", + "\n", + "https://github.com/IBM/prompt-declaration-language" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b7576532-aee3-4580-85fd-0b97bc503621", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.5" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/notebooks/demonstrations.txt b/examples/notebooks/demonstrations.txt new file mode 100644 index 000000000..7503f04b9 --- /dev/null +++ b/examples/notebooks/demonstrations.txt @@ -0,0 +1,31 @@ + +What is the elevation range for the area that the eastern sector of the Colorado orogeny extends into? +Tho: I need to search Colorado orogeny, find the area that the eastern sector of the Colorado ... +Act: {"name": "Search", "arguments": {"topic": "Colorado orogeny"}} +Obs: The Colorado orogeny was an episode of mountain building (an orogeny) ... +Tho: It does not mention the eastern sector. So I need to look up eastern sector. +Tho: High Plains rise in elevation from around 1,800 to 7,000 ft, so the answer is 1,800 to 7,000 ft. +Act: {"name": "Finish", "arguments": {"topic": "1,800 to 7,000 ft"}} + +What profession does Nicholas Ray and Elia Kazan have in common? +Tho: I need to search Nicholas Ray and Elia Kazan, find their professions, then find the profession they have in common. +Act: {"name": "Search", "arguments": {"topic": "Nicholas Ray"}} +Obs: Nicholas Ray (born Raymond Nicholas Kienzle Jr., August 7, 1911 - June 16, 1979) was an American film director, screenwriter, and actor best known for the 1955 film Rebel Without a Cause. +Tho: Professions of Nicholas Ray are director, screenwriter, and actor. I need to search Elia Kazan next and find his professions. +Act: {"name": "Search", "arguments": {"topic": "Elia Kazan"}} +Obs: Elia Kazan was an American film and theatre director, producer, screenwriter and actor. +Tho: Professions of Elia Kazan are director, producer, screenwriter, and actor. So profession Nicholas Ray and Elia Kazan have in common is director, screenwriter, and actor. +Act: {"name": "Finish", "arguments": {"topic": "director, screenwriter, actor"}} + +What is 18 + 12 x 3? +Tho: I need to call a calculator. +Act: {"name": "Calc", "arguments": {"expr": "18 + 12 * 3"}} +Obs: 54 +Act: {"name": "Finish", "arguments": {"topic": "54"}} + +A total of 252 qualifying matches were played, and 723 goals were scored. What was the average number of goals per match? +Tho: I need to call a calculator. +Act: {"name": "Calc", "arguments": {"expr": "723 / 252"}} +Obs: 2.869047619047619 +Act: {"name": "Finish", "arguments": {"topic": "2.869047619047619"}} + diff --git a/examples/notebooks/notebook.ipynb b/examples/notebooks/notebook.ipynb index 9d8c5a411..eeba55899 100644 --- a/examples/notebooks/notebook.ipynb +++ b/examples/notebooks/notebook.ipynb @@ -2,26 +2,17 @@ "cells": [ { "cell_type": "code", - "execution_count": 12, + "execution_count": 2, "id": "e25a6874-54d9-4167-82ed-ab2f4fdc0a6f", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "The pdl.pdl_notebook_ext extension is already loaded. To reload it, use:\n", - " %reload_ext pdl.pdl_notebook_ext\n" - ] - } - ], + "outputs": [], "source": [ "%load_ext pdl.pdl_notebook_ext" ] }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 4, "id": "f3c62df1-0347-4711-acd7-3892cfd5df30", "metadata": {}, "outputs": [ @@ -29,7 +20,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "Hello,\u001b[32m world\u001b[0m\u001b[32m!\u001b[0m" + "Hello\n", + "\u001b[32mHello\u001b[0m" ] } ], @@ -37,16 +29,16 @@ "%%pdl\n", "description: Model call\n", "text: \n", - "- Hello,\n", - "- model: watsonx/ibm/granite-34b-code-instruct\n", + "- \"Hello\\n\"\n", + "- model: \"replicate/ibm-granite/granite-3.0-8b-instruct\"\n", " parameters:\n", - " stop: [\"!\"]\n", - " include_stop_sequence: true" + " stop_sequences: \"!\"\n", + " " ] }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 5, "id": "7f6c323b-ad1a-4434-8732-bc19c5c47883", "metadata": {}, "outputs": [ @@ -66,14 +58,23 @@ " return offsetMap;\n", "}\n", "\n", - "\u001b[32m\n", - "\u001b[0m\u001b[32mThe\u001b[0m\u001b[32m function\u001b[0m\u001b[32m `\u001b[0m\u001b[32mdeserialize\u001b[0m\u001b[32mOffset\u001b[0m\u001b[32mMap\u001b[0m\u001b[32m`\u001b[0m\u001b[32m takes\u001b[0m\u001b[32m a\u001b[0m\u001b[32m string\u001b[0m\u001b[32m as\u001b[0m\u001b[32m input\u001b[0m\u001b[32m and\u001b[0m\u001b[32m returns\u001b[0m\u001b[32m a\u001b[0m\u001b[32m map\u001b[0m\u001b[32m.\u001b[0m\u001b[32m It\u001b[0m\u001b[32m first\u001b[0m\u001b[32m checks\u001b[0m\u001b[32m if\u001b[0m\u001b[32m the\u001b[0m\u001b[32m input\u001b[0m\u001b[32m string\u001b[0m\u001b[32m is\u001b[0m\u001b[32m null\u001b[0m\u001b[32m or\u001b[0m\u001b[32m empty\u001b[0m\u001b[32m.\u001b[0m\u001b[32m If\u001b[0m\u001b[32m it\u001b[0m\u001b[32m is\u001b[0m\u001b[32m,\u001b[0m\u001b[32m it\u001b[0m\u001b[32m creates\u001b[0m\u001b[32m a\u001b[0m\u001b[32m new\u001b[0m\u001b[32m empty\u001b[0m\u001b[32m map\u001b[0m\u001b[32m and\u001b[0m\u001b[32m returns\u001b[0m\u001b[32m it\u001b[0m\u001b[32m.\u001b[0m\u001b[32m Otherwise\u001b[0m\u001b[32m,\u001b[0m\u001b[32m it\u001b[0m\u001b[32m uses\u001b[0m\u001b[32m the\u001b[0m\u001b[32m Jack\u001b[0m\u001b[32mson\u001b[0m\u001b[32m library\u001b[0m\u001b[32m to\u001b[0m\u001b[32m parse\u001b[0m\u001b[32m the\u001b[0m\u001b[32m input\u001b[0m\u001b[32m string\u001b[0m\u001b[32m into\u001b[0m\u001b[32m a\u001b[0m\u001b[32m map\u001b[0m\u001b[32m and\u001b[0m\u001b[32m returns\u001b[0m\u001b[32m it\u001b[0m\u001b[32m.\u001b[0m\u001b[32m\n", - "\u001b[0m\u001b[32m\n", - "\u001b[0m\u001b[32mThe\u001b[0m\u001b[32m `@\u001b[0m\u001b[32mSuppressWarnings\u001b[0m\u001b[32m(\"\u001b[0m\u001b[32munchecked\u001b[0m\u001b[32m\")\u001b[0m\u001b[32m`\u001b[0m\u001b[32m annotation\u001b[0m\u001b[32m is\u001b[0m\u001b[32m used\u001b[0m\u001b[32m to\u001b[0m\u001b[32m suppress\u001b[0m\u001b[32m the\u001b[0m\u001b[32m warning\u001b[0m\u001b[32m that\u001b[0m\u001b[32m the\u001b[0m\u001b[32m type\u001b[0m\u001b[32m of\u001b[0m\u001b[32m the\u001b[0m\u001b[32m parsed\u001b[0m\u001b[32m map\u001b[0m\u001b[32m is\u001b[0m\u001b[32m not\u001b[0m\u001b[32m checked\u001b[0m\u001b[32m.\u001b[0m\u001b[32m This\u001b[0m\u001b[32m is\u001b[0m\u001b[32m because\u001b[0m\u001b[32m the\u001b[0m\u001b[32m Jack\u001b[0m\u001b[32mson\u001b[0m\u001b[32m library\u001b[0m\u001b[32m is\u001b[0m\u001b[32m used\u001b[0m\u001b[32m to\u001b[0m\u001b[32m parse\u001b[0m\u001b[32m the\u001b[0m\u001b[32m input\u001b[0m\u001b[32m string\u001b[0m\u001b[32m into\u001b[0m\u001b[32m a\u001b[0m\u001b[32m map\u001b[0m\u001b[32m,\u001b[0m\u001b[32m but\u001b[0m\u001b[32m the\u001b[0m\u001b[32m specific\u001b[0m\u001b[32m type\u001b[0m\u001b[32m of\u001b[0m\u001b[32m the\u001b[0m\u001b[32m map\u001b[0m\u001b[32m is\u001b[0m\u001b[32m not\u001b[0m\u001b[32m known\u001b[0m\u001b[32m at\u001b[0m\u001b[32m compile\u001b[0m\u001b[32m time\u001b[0m\u001b[32m.\u001b[0m\u001b[32m Therefore\u001b[0m\u001b[32m,\u001b[0m\u001b[32m the\u001b[0m\u001b[32m warning\u001b[0m\u001b[32m is\u001b[0m\u001b[32m supp\u001b[0m\u001b[32mressed\u001b[0m\u001b[32m to\u001b[0m\u001b[32m avoid\u001b[0m\u001b[32m potential\u001b[0m\u001b[32m issues\u001b[0m\u001b[32m.\u001b[0m\n", + "\u001b[32mThis Java function, `deserializeOffsetMap`, is designed to convert a JSON string into a `Map`. Here's a breakdown of what it does:\n", + "\n", + "1. It takes a single argument, `lastSourceOffset`, which is expected to be a JSON string.\n", + "\n", + "2. It initializes a `Map` called `offsetMap`.\n", + "\n", + "3. If `lastSourceOffset` is either `null` or an empty string, it creates a new `HashMap` and assigns it to `offsetMap`.\n", + "\n", + "4. If `lastSourceOffset` is not `null` or an empty string, it uses the `JSON_MAPPER` object (presumably an\u001b[0m\u001b[32m instance of `ObjectMapper` from the Jackson library) to convert the JSON string into a `Map` and assigns it to `offsetMap`.\n", + "\n", + "5. Finally, it returns the `offsetMap`.\n", + "\n", + "The `@SuppressWarnings(\"unchecked\")` annotation is used to suppress a compile-time warning about the raw use of the `Map` type. This is\u001b[0m\u001b[32m because the `JSON_MAPPER.readValue` method returns a `Map`, but the compiler doesn't know that this `Map` will be a `Map`. The `unchecked` warning is suppressed to avoid cluttering the output with this warning.\u001b[0m\n", "\n", "EVALUATION:\n", "The similarity (Levenshtein) between this answer and the ground truth is:\n", - "\u001b[35m0.9983818770226537\u001b[0m" + "\u001b[35m0.3671003717472119\u001b[0m" ] } ], @@ -88,7 +89,7 @@ " read: ./ground_truth.txt\n", "text:\n", "- \"\\n${ CODE.source_code }\\n\"\n", - "- model: watsonx/ibm/granite-34b-code-instruct\n", + "- model: \"replicate/ibm-granite/granite-3.0-8b-instruct\"\n", " def: EXPLANATION\n", " input: |\n", " Here is some info about the location of the function in the repo.\n", @@ -144,7 +145,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.3" + "version": "3.12.5" } }, "nbformat": 4, diff --git a/examples/notebooks/notebook_debug.ipynb b/examples/notebooks/notebook_debug.ipynb index e72b50d2d..744ef498a 100644 --- a/examples/notebooks/notebook_debug.ipynb +++ b/examples/notebooks/notebook_debug.ipynb @@ -164,10 +164,9 @@ "description: Model call\n", "text: \n", "- Hello,\n", - "- model: watsonx/ibm/granite-34b-code-instruct\n", + "- model: \"replicate/ibm-granite/granite-3.0-8b-instruct\"\n", " parameters:\n", - " stop: [\"!\"]\n", - " include_stop_sequence: true" + " stop_sequences: \"!\"" ] }, { diff --git a/examples/notebooks/pdl.ipynb b/examples/notebooks/pdl.ipynb index 2269339bf..e6d43298f 100644 --- a/examples/notebooks/pdl.ipynb +++ b/examples/notebooks/pdl.ipynb @@ -21,13 +21,12 @@ "metadata": {}, "outputs": [], "source": [ - "! pip install prompt-declaration-language\n", "! pip install 'prompt-declaration-language[examples]'" ] }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 1, "id": "e25a6874-54d9-4167-82ed-ab2f4fdc0a6f", "metadata": {}, "outputs": [], @@ -47,7 +46,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 2, "id": "f3c62df1-0347-4711-acd7-3892cfd5df30", "metadata": {}, "outputs": [ @@ -56,9 +55,17 @@ "output_type": "stream", "text": [ "What is the meaning of life?\n", - "\u001b[32mThe meaning of life is a philosophical question that has been debated by many thinkers throughout history. There is no one definitive answer, as the answer may vary depending on one's personal beliefs, values, and experiences.\n", + "\u001b[32mThe meaning of life is a philosophical question that has been debated by many thinkers throughout history and is still a subject of discussion today. The answer to this question can vary greatly depending on one's personal beliefs, values, and experiences.\n", "\u001b[0m" ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/mvaziri/.pyenv/versions/3.12.5/lib/python3.12/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n" + ] } ], "source": [ @@ -83,7 +90,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 3, "id": "d7149b3f", "metadata": {}, "outputs": [ @@ -93,60 +100,22 @@ "text": [ "\n", "Say it like a poem\n", - "\u001b[32mThe meaning of life, a question so profound,\n", - "A mystery that has puzzled men and women for so long,\n", - "A path that we must tread, a goal to reach,\n", - "A journey that will bring us joy and pain,\n", - "\n", - "A road that twists and turns, a fork in the road,\u001b[0m\u001b[32m\n", - "Where we must choose, which way to go,\n", - "A decision that we must make, with our souls at stake,\n", - "A choice that will shape our destiny,\n", - "\n", - "The meaning of life, a question so grand,\n", - "A goal that we must strive for, to find,\n", - "A purpose that gives our hearts meaning,\n", - "A reason to live, a\u001b[0m\u001b[32m reason to die,\n", - "\n", - "A journey that will take us far, a journey that will bring,\n", - "A new understanding of the world we live in,\n", - "A new perspective on life, a new way of thinking,\n", - "A new path to follow, a new way to live,\n", - "\n", - "The meaning of life, a question so deep,\n", - "A mystery that will never be solved,\n", - "A journey that will\u001b[0m\u001b[32m take us far, a journey that will bring,\n", - "A new understanding of the world we live in,\n", - "\n", - "A road that twists and turns, a fork in the road,\n", - "Where we must choose, which way to go,\n", - "A decision that we must make, with our souls at stake,\n", - "A choice that will shape our destiny\u001b[0m\u001b[32m,\n", - "\n", - "The meaning of life, a question so grand,\n", - "A goal that we must strive for, to find,\n", - "A purpose that gives our hearts meaning,\n", - "A reason to live, a reason to die,\n", - "\n", - "A journey that will take us far, a journey that will bring,\n", - "A new understanding of the world we live in,\n", - "A new perspective on life, a new way of thinking,\n", - "A new\u001b[0m\u001b[32m path to follow, a new way to live,\n", - "\n", - "The meaning of life, a question so deep,\n", - "A mystery that will never be solved,\n", - "A journey that will take us far, a journey that will bring,\n", - "A new understanding of the world we live in,\n", - "\n", - "A road that twists and turns, a fork in the road,\n", - "Where we must choose, which way to go\u001b[0m\u001b[32m,\n", - "A decision that we must make, with our souls at stake,\n", - "A choice that will shape our destiny,\n", - "\n", - "The meaning of life, a question\u001b[0m\n", + "\u001b[32mThe meaning of life,\n", + "A question that we all ask,\n", + "A mystery that still defies,\n", + "The answer that we all seek.\n", + "It's a question that's been asked,\n", + "Throughout history, by many,\n", + "And yet, it remains,\n", + "A mystery that we'll never know.\n", + "But, we can try to\u001b[0m\u001b[32m find,\n", + "The meaning of life,\n", + "And in doing so,\n", + "We can find our own way.\n", + "\u001b[0m\n", "\n", "What is the most important verse in this poem?\n", - "\u001b[32mThe most important verse in this poem is the first one: \"The meaning of life, a question so profound.\" This line sets the tone for the entire poem and emphasizes the central theme of the question of what gives life meaning. It also highlights the idea that the answer to this question is not straightforward\u001b[0m\u001b[32m and may vary depending on one's personal beliefs and experiences.\n", + "\u001b[32mThe most important verse in this poem is \"But, we can try to find, The meaning of life.\" This verse emphasizes the importance of trying to find the meaning of life, even though it may still be a mystery. It suggests that the search for meaning is an ongoing process, and that we can each find our own\u001b[0m\u001b[32m way to discover it.\n", "\u001b[0m" ] } @@ -172,7 +141,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 4, "id": "455b2dbc-69fb-4164-9b8b-5817b3f33e9b", "metadata": {}, "outputs": [ @@ -194,7 +163,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "\u001b[32mAPR stands for Annual Percentage Rate. It is a measure of the total cost of borrowing money, including interest and fees, expressed as a yearly rate. It is commonly used in the lending industry to compare the cost of different loans and credit products.\n", + "\u001b[32mAPR stands for Annual Percentage Rate, which is a measure of the total cost of borrowing money. It is the total amount of interest that a borrower pays over the life of a loan, expressed as a percentage of the loan amount. APR includes both the interest rate and any fees or charges associated with the loan, such as orig\u001b[0m\u001b[32mination fees or prepayment penalties. It is a useful tool for comparing the cost of different loans and for understanding the true cost of borrowing money.\n", "\u001b[0m\n", "\n" ] @@ -203,14 +172,29 @@ "name": "stdin", "output_type": "stream", "text": [ - ">>> Say it like I'm 5 years old\n" + ">>> say it like a poem\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "\u001b[32mThe meaning of life is like a big, big, big question mark. It's a question that has been asked for as long as people can remember, and it's still a question that people don't always know the answer to. Some people think the answer is to have fun and make friends, while others think the answer is to work hard\u001b[0m\u001b[32m and be smart. But no matter what the answer is, the question of what gives life meaning is a question that will always be with us.\n", + "\u001b[32mAPR stands for Annual Percentage Rate,\n", + "A measure of the total cost of borrowing,\n", + "It's the total amount of interest you pay,\n", + "Over the life of your loan, expressed in percentage.\n", + "APR includes fees and charges,\n", + " origination fees, prepayment penalties, and more,\n", + "It's a useful\u001b[0m\u001b[32m tool for comparing loans,\n", + "And for understanding the true cost of borrowing.\n", + "But remember, not all loans are created equal,\n", + "APR doesn't tell the whole story,\n", + "It's important to read the fine print,\n", + "And understand all the terms and conditions.\n", + "So, before you sign on the dotted line,\n", + "Make sure you understand what you're getting,\n", + "APR is just\u001b[0m\u001b[32m one of many factors,\n", + "To consider when choosing a loan.\n", "\u001b[0m\n", "\n" ] @@ -226,7 +210,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "\u001b[32mThank you for chatting with me! If you have any more questions or need further assistance, feel free to ask.\n", + "\u001b[32mThank you for using my services. If you have any further questions or need additional assistance, please don't hesitate to ask.\n", "\u001b[0m" ] } @@ -278,7 +262,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 5, "id": "7f6c323b-ad1a-4434-8732-bc19c5c47883", "metadata": {}, "outputs": [ @@ -298,18 +282,18 @@ " return offsetMap;\n", "}\n", "\n", - "\u001b[32mThe function `deserializeOffsetMap` is a method that takes a string `lastSourceOffset` as input and returns a `Map` of `String` keys and `String` values. The function is used to deserialize a JSON string into a `Map` object.\n", + "\u001b[32mThe code is a method in the `OffsetUtil` class that takes a string `lastSourceOffset` as input and returns a `Map` of `String` keys and `String` values. The method is annotated with `@SuppressWarnings(\"unchecked\")` to suppress a warning about the unchecked conversion of the JSON string to a `Map` object.\n", " \n", "\n", - " The function first checks if the `lastSourceOffset` is null or empty.\u001b[0m\u001b[32m If it is, it creates a new `HashMap` object and assigns it to the `offsetMap` variable. If the `lastSourceOffset` is not null or empty, the function uses the `JSON_MAPPER` object to read the JSON string and deserialize it into a `Map` object. The `JSON_MAPPER` object is assumed to be a pre-defined\u001b[0m\u001b[32m object that is used for JSON serialization and deserialization.\n", + " The method first checks if `\u001b[0m\u001b[32mlastSourceOffset` is `null` or empty. If it is, it creates a new empty `HashMap` and assigns it to the `offsetMap` variable. If `lastSourceOffset` is not `null` or empty, it uses the `JSON_MAPPER` object to deserialize the JSON string into a `Map` object. The `JSON_MAPPER` object\u001b[0m\u001b[32m is assumed to be a pre-defined object that can deserialize JSON strings into Java objects.\n", " \n", "\n", - " Finally, the function returns the `offsetMap` object.\n", + " Finally, the method returns the `offsetMap` object.\n", "\u001b[0m\n", "\n", "EVALUATION:\n", "The similarity (Levenshtein) between this answer and the ground truth is:\n", - "\u001b[35m0.31163434903047094\u001b[0m" + "\u001b[35m0.3046875\u001b[0m" ] } ], diff --git a/examples/rag/README.md b/examples/rag/README.md new file mode 100644 index 000000000..add819ee1 --- /dev/null +++ b/examples/rag/README.md @@ -0,0 +1,4 @@ +This example requires you to install: +``` +pip install scikit-learn +``` \ No newline at end of file diff --git a/examples/rag/rag.pdl b/examples/rag/rag.pdl index ea6e92462..d5b67093d 100644 --- a/examples/rag/rag.pdl +++ b/examples/rag/rag.pdl @@ -42,7 +42,5 @@ text: - |- Q: ${ TEST_PROMPT } - A: ``` -- model: watsonx/ibm/granite-34b-code-instruct - parameters: - stop: ["```"] + A: +- model: replicate/ibm-granite/granite-3.0-8b-instruct diff --git a/examples/react/demo.pdl b/examples/react/demo.pdl new file mode 100644 index 000000000..0116ac672 --- /dev/null +++ b/examples/react/demo.pdl @@ -0,0 +1,78 @@ +text: +- | + What is the elevation range for the area that the eastern sector of the Colorado orogeny extends into? + Tho: I need to search Colorado orogeny, find the area that the eastern sector of the Colorado ... + Act: {"name": "Search", "arguments": {"topic": "Colorado orogeny"}} + Obs: The Colorado orogeny was an episode of mountain building (an orogeny) ... + Tho: It does not mention the eastern sector. So I need to look up eastern sector. + Tho: High Plains rise in elevation from around 1,800 to 7,000 ft, so the answer is 1,800 to 7,000 ft. + Act: {"name": "Finish", "arguments": {"topic": "1,800 to 7,000 ft"}} + + What profession does Nicholas Ray and Elia Kazan have in common? + Tho: I need to search Nicholas Ray and Elia Kazan, find their professions, then find the profession they have in common. + Act: {"name": "Search", "arguments": {"topic": "Nicholas Ray"}} + Obs: Nicholas Ray (born Raymond Nicholas Kienzle Jr., August 7, 1911 - June 16, 1979) was an American film director, screenwriter, and actor best known for the 1955 film Rebel Without a Cause. + Tho: Professions of Nicholas Ray are director, screenwriter, and actor. I need to search Elia Kazan next and find his professions. + Act: {"name": "Search", "arguments": {"topic": "Elia Kazan"}} + Obs: Elia Kazan was an American film and theatre director, producer, screenwriter and actor. + Tho: Professions of Elia Kazan are director, producer, screenwriter, and actor. So profession Nicholas Ray and Elia Kazan have in common is director, screenwriter, and actor. + Act: {"name": "Finish", "arguments": {"topic": "director, screenwriter, actor"}} + + What is 18 + 12 x 3? + Tho: I need to call a calculator. + Act: {"name": "Calc", "arguments": {"expr": "18 + 12 * 3"}} + Obs: 54 + Act: {"name": "Finish", "arguments": {"topic": "54"}} + + A total of 252 qualifying matches were played, and 723 goals were scored. What was the average number of goals per match? + Tho: I need to call a calculator. + Act: {"name": "Calc", "arguments": {"expr": "723 / 252"}} + Obs: 2.869047619047619 + Act: {"name": "Finish", "arguments": {"topic": "2.869047619047619"}} + + + How many years ago was the discoverer of the Hudson River born? Keep in mind we are in 2024. +- repeat: + text: + - def: thought + model: replicate/ibm-granite/granite-3.0-8b-instruct + parameters: + stop_sequences: "Act:" + temperature: 0 + - def: rawAction + model: replicate/ibm-granite/granite-3.0-8b-instruct + parameters: + stop_sequences: "\n" + temperature: 0 + - def: action + lang: python + parser: json + spec: {name: str, arguments: obj} + contribute: [context] + code: + | + result = '${ rawAction }'.replace("Act: ", "") + - def: observation + if: ${ action.name == "Search" } + then: + text: + - "\nObs: " + - lang: python + code: | + import warnings, wikipedia + warnings.simplefilter("ignore") + try: + result = wikipedia.summary("${ action.arguments.topic }") + except wikipedia.WikipediaException as e: + result = str(e) + - "\n" + else: + - if: ${ action.name == "Calc" } + then: + text: + - "\nObs: " + - lang: python + code: result = ${ action.arguments.expr } + - "\n" + until: ${ action.name != "Search" } + diff --git a/examples/react/demonstrations.txt b/examples/react/demonstrations.txt new file mode 100644 index 000000000..14ca31ad8 --- /dev/null +++ b/examples/react/demonstrations.txt @@ -0,0 +1,33 @@ + +What is the elevation range for the area that the eastern sector of the Colorado orogeny extends into? +Tho: I need to search Colorado orogeny, find the area that the eastern sector of the Colorado ... +Act: {"name": "Search", "arguments": {"topic": "Colorado orogeny"}} +Obs: The Colorado orogeny was an episode of mountain building (an orogeny) ... +Tho: It does not mention the eastern sector. So I need to look up eastern sector. +Tho: High Plains rise in elevation from around 1,800 to 7,000 ft, so the answer is 1,800 to 7,000 ft. +Act: {"name": "Finish", "arguments": {"topic": "1,800 to 7,000 ft"}} + +What profession does Nicholas Ray and Elia Kazan have in common? +Tho: I need to search Nicholas Ray and Elia Kazan, find their professions, then find the profession they have in common. +Act: {"name": "Search", "arguments": {"topic": "Nicholas Ray"}} +Obs: Nicholas Ray (born Raymond Nicholas Kienzle Jr., August 7, 1911 - June 16, 1979) was an American film director, screenwriter, and actor best known for the 1955 film Rebel Without a Cause. +Tho: Professions of Nicholas Ray are director, screenwriter, and actor. I need to search Elia Kazan next and find his professions. +Act: {"name": "Search", "arguments": {"topic": "Elia Kazan"}} +Obs: Elia Kazan was an American film and theatre director, producer, screenwriter and actor. +Tho: Professions of Elia Kazan are director, producer, screenwriter, and actor. So profession Nicholas Ray and Elia Kazan have in common is director, screenwriter, and actor. +Act: {"name": "Finish", "arguments": {"topic": "director, screenwriter, actor"}} + +What is 18 + 12 x 3? +Tho: I need to call a calculator. +Act: {"name": "Calc", "arguments": {"expr": "18 + 12 * 3"}} +Obs: 54 +Act: {"name": "Finish", "arguments": {"topic": "54"}} + +A total of 252 qualifying matches were played, and 723 goals were scored. What was the average number of goals per match? +Tho: I need to call a calculator. +Act: {"name": "Calc", "arguments": {"expr": "723 / 252"}} +Obs: 2.869047619047619 +Act: {"name": "Finish", "arguments": {"topic": "2.869047619047619"}} + + + diff --git a/examples/react/multi-agent.pdl b/examples/react/multi-agent.pdl deleted file mode 100644 index d30fc40aa..000000000 --- a/examples/react/multi-agent.pdl +++ /dev/null @@ -1,22 +0,0 @@ -text: -- include: ./react_fun.pdl -- defs: - question: How many years ago was the discoverer of the Hudson River born? -- def: proposed - call: react - args: - question: ${ question } - model: watsonx/ibm/granite-20b-code-instruct - -- "\n\n----- Verifying answer... -----\n\n" -- def: verified - call: react - args: - question: ${ question } - model: watsonx/ibm/granite-34b-code-instruct -- "\n" -- if: ${ proposed.split('Act:') | last == verified.split('Act:') | last } - then: - Answer is correct! - else: - Answer is incorrect! diff --git a/examples/react/react_call.pdl b/examples/react/react_call.pdl index b310cbd78..2ad7043a8 100644 --- a/examples/react/react_call.pdl +++ b/examples/react/react_call.pdl @@ -3,5 +3,7 @@ text: - include: ./react_fun.pdl - call: react args: - question: How many years ago was the discoverer of the Hudson River born? - model: watsonx/ibm/granite-34b-code-instruct + question: How many years ago was the discoverer of the Hudson River born? Keep in mind we are in 2024. + model: replicate/ibm-granite/granite-3.0-8b-instruct + + diff --git a/examples/react/react_fun.pdl b/examples/react/react_fun.pdl index 947e13ad0..ef0bdd883 100644 --- a/examples/react/react_fun.pdl +++ b/examples/react/react_fun.pdl @@ -25,23 +25,28 @@ defs: - def: thought model: ${ model } parameters: - stop: - - "Act:" - include_stop_sequence: true - - def: action + stop_sequences: "Act:" + temperature: 0 + - def: rawAction model: ${ model } parameters: - stop: - - "\n" + stop_sequences: "\n" + temperature: 0 + - def: action + lang: python parser: json spec: {name: str, arguments: obj} - - if: ${ action != prev_action} + contribute: [context] + code: + | + result = '${ rawAction }'.replace("Act: ", "") + - if: ${ action != prev_action} then: - def: observation if: ${ action.name == "Search" } then: text: - - "Obs: " + - "\nObs: " - lang: python code: | import warnings, wikipedia @@ -50,13 +55,15 @@ defs: result = wikipedia.summary("${ action.arguments.topic }") except wikipedia.WikipediaException as e: result = str(e) + - "\n" else: - if: ${ action.name == "Calc" } then: text: - - "Obs: " + - "\nObs: " - lang: python code: result = ${ action.arguments.expr } + - "\n" else: def: exit contribute: [] @@ -84,6 +91,7 @@ defs: Tho: High Plains rise in elevation from around 1,800 to 7,000 ft, so the answer is 1,800 to 7,000 ft. Act: {"name": "Finish", "arguments": {"topic": "1,800 to 7,000 ft"}} + - text: | What profession does Nicholas Ray and Elia Kazan have in common? @@ -96,19 +104,25 @@ defs: Tho: Professions of Elia Kazan are director, producer, screenwriter, and actor. So profession Nicholas Ray and Elia Kazan have in common is director, screenwriter, and actor. Act: {"name": "Finish", "arguments": {"topic": "director, screenwriter, actor"}} + - text: | What is 18 + 12 x 3? + Tho: I need to call a calculator. Act: {"name": "Calc", "arguments": {"expr": "18 + 12 * 3"}} Obs: 54 Act: {"name": "Finish", "arguments": {"topic": "54"}} + - text: | A total of 252 qualifying matches were played, and 723 goals were scored. What was the average number of goals per match? + Tho: I need to call a calculator. Act: {"name": "Calc", "arguments": {"expr": "723 / 252"}} Obs: 2.869047619047619 Act: {"name": "Finish", "arguments": {"topic": "2.869047619047619"}} + + - call: react_inner args: pdl_context: [] @@ -116,3 +130,4 @@ defs: question: ${ question } model: ${ model } + diff --git a/examples/react/wikipedia.pdl b/examples/react/wikipedia.pdl index 05061e84e..114b3932e 100644 --- a/examples/react/wikipedia.pdl +++ b/examples/react/wikipedia.pdl @@ -18,27 +18,57 @@ text: Tho: Professions of Elia Kazan are director, producer, screenwriter, and actor. So profession Nicholas Ray and Elia Kazan have in common is director, screenwriter, and actor. Act: {"name": "Finish", "arguments": {"topic": "director, screenwriter, actor"}} - When was the discoverer of the Hudson River born? + What is 18 + 12 x 3? + Tho: I need to call a calculator. + Act: {"name": "Calc", "arguments": {"expr": "18 + 12 * 3"}} + Obs: 54 + Act: {"name": "Finish", "arguments": {"topic": "54"}} + + A total of 252 qualifying matches were played, and 723 goals were scored. What was the average number of goals per match? + Tho: I need to call a calculator. + Act: {"name": "Calc", "arguments": {"expr": "723 / 252"}} + Obs: 2.869047619047619 + Act: {"name": "Finish", "arguments": {"topic": "2.869047619047619"}} + + What is 18 + 12 x 3? + Tho: I need to call a calculator. + Act: {"name": "Calc", "arguments": {"expr": "18 + 12 * 3"}} + Obs: 54 + Act: {"name": "Finish", "arguments": {"topic": "54"}} + + A total of 252 qualifying matches were played, and 723 goals were scored. What was the average number of goals per match? + Tho: I need to call a calculator. + Act: {"name": "Calc", "arguments": {"expr": "723 / 252"}} + Obs: 2.869047619047619 + Act: {"name": "Finish", "arguments": {"topic": "2.869047619047619"}} + + + when was the discoverer of the Hudson River born? - repeat: text: - def: thought - model: watsonx/ibm/granite-34b-code-instruct + model: replicate/ibm-granite/granite-3.0-8b-instruct parameters: - stop: - - "Act:" - include_stop_sequence: true - - def: action - model: watsonx/ibm/granite-34b-code-instruct + stop_sequences: "Act:" + temperature: 0 + - def: rawAction + model: replicate/ibm-granite/granite-3.0-8b-instruct parameters: - stop: - - "\n" + stop_sequences: "\n" + temperature: 0 + - def: action + lang: python parser: json - spec: {name: str, arguments: {topic: str}} + spec: {name: str, arguments: obj} + contribute: [context] + code: + | + result = '${ rawAction }'.replace("Act: ", "") - def: observation if: ${ action.name == "Search" } then: text: - - "Obs: " + - "\nObs: " - lang: python code: | import warnings, wikipedia @@ -47,5 +77,6 @@ text: result = wikipedia.summary("${ action.arguments.topic }") except wikipedia.WikipediaException as e: result = str(e) + - "\n" until: ${ action.name != "Search" } diff --git a/examples/sdk/hello.pdl b/examples/sdk/hello.pdl index 4e93b0ef8..28ef61974 100644 --- a/examples/sdk/hello.pdl +++ b/examples/sdk/hello.pdl @@ -1,8 +1,5 @@ text: -- Hello, -- model: watsonx/ibm/granite-34b-code-instruct +- "Hello\n" +- model: replicate/ibm-granite/granite-3.0-8b-instruct parameters: - stop: - - '!' - include_stop_sequence: true - + stop_sequences: '!' diff --git a/examples/sdk/hello_dict.py b/examples/sdk/hello_dict.py index d68ffd6db..d7cd2b031 100644 --- a/examples/sdk/hello_dict.py +++ b/examples/sdk/hello_dict.py @@ -2,10 +2,10 @@ hello = { "text": [ - "Hello,", + "Hello\n", { - "model": "watsonx/ibm/granite-34b-code-instruct", - "parameters": {"stop": ["!"], "include_stop_sequence": True}, + "model": "replicate/ibm-granite/granite-3.0-8b-instruct", + "parameters": {"stop_sequences": "!"}, }, ] } diff --git a/examples/sdk/hello_prog.py b/examples/sdk/hello_prog.py index 73c4542ff..a458ce182 100644 --- a/examples/sdk/hello_prog.py +++ b/examples/sdk/hello_prog.py @@ -4,11 +4,11 @@ hello = Program( TextBlock( text=[ - "Hello,", + "Hello\n", LitellmModelBlock( - model="watsonx/ibm/granite-34b-code-instruct", + model="replicate/ibm-granite/granite-3.0-8b-instruct", parameters=LitellmParameters( - stop=["!"], include_stop_sequence=True # pyright: ignore + stop_sequences="!" # pyright: ignore ), ), ] diff --git a/examples/sdk/hello_str.py b/examples/sdk/hello_str.py index 4d521076f..48d8b2ba7 100644 --- a/examples/sdk/hello_str.py +++ b/examples/sdk/hello_str.py @@ -2,12 +2,10 @@ HELLO = """ text: -- Hello, -- model: watsonx/ibm/granite-34b-code-instruct +- "Hello\n" +- model: replicate/ibm-granite/granite-3.0-8b-instruct parameters: - stop: - - '!' - include_stop_sequence: true + stop_sequences: '!' """ diff --git a/examples/talk/1-hello.pdl b/examples/talk/1-hello.pdl index 01ae84e44..2bd0a8d62 100644 --- a/examples/talk/1-hello.pdl +++ b/examples/talk/1-hello.pdl @@ -1,8 +1,8 @@ description: Model call text: -- Hello, -- model: watsonx/ibm/granite-34b-code-instruct +- "Hello\n" +- model: replicate/ibm-granite/granite-3.0-8b-instruct parameters: - stop: ["!"] + stop_sequences: "!" include_stop_sequence: true diff --git a/examples/talk/10-multi-agent.pdl b/examples/talk/10-multi-agent.pdl deleted file mode 100644 index de878e1e3..000000000 --- a/examples/talk/10-multi-agent.pdl +++ /dev/null @@ -1,22 +0,0 @@ -text: -- include: ./react_fun.pdl -- defs: - question: When was the discoverer of the Hudson River born? -- def: proposed - call: react - args: - question: ${ question } - model: watsonx/ibm/granite-20b-code-instruct - -- "\n\n----- Verifying answer... -----\n\n" -- def: verified - call: react - args: - question: ${ question } - model: watsonx/ibm/granite-34b-code-instruct -- "\n" -- if: ${ proposed.split('Act:') | last == verified.split('Act:') | last } - then: - Answer is correct! - else: - Answer is incorrect! diff --git a/examples/talk/11-sdg.pdl b/examples/talk/10-sdg.pdl similarity index 96% rename from examples/talk/11-sdg.pdl rename to examples/talk/10-sdg.pdl index 7d946f96a..b31763834 100644 --- a/examples/talk/11-sdg.pdl +++ b/examples/talk/10-sdg.pdl @@ -1,7 +1,6 @@ -description: Real-world Synthetic Data Generation (SDG) pipeline defs: teacher_sys_prompt: You are a very knowledgeable AI Assistant that will faithfully assist the user with their task. - teacher_model: watsonx/mistralai/mixtral-8x7b-instruct-v01 + teacher_model: replicate/ibm-granite/granite-3.0-8b-instruct teacher_template: function: sys_prompt: str @@ -68,8 +67,8 @@ defs: model: ${teacher_model} input: ${teacher_input} parameters: - stop: ["${teacher_stop_token}"] - include_stop_sequence: false + temperature: 0 + stop_sequences: "${teacher_stop_token}" max_new_tokens: ${prompt_data.max_new_tokens} parser: regex: '### Question [0-9]+:\s*([^#\n]+)' @@ -157,9 +156,9 @@ defs: model: ${teacher_model} input: ${teacher_input} parameters: - stop: ["${teacher_stop_token}"] - include_stop_sequence: false + stop_sequences: "${teacher_stop_token}" max_new_tokens: ${prompt_data.max_new_tokens} + temperature: 0 parser: spec: { "rating": str } # regex: "Rating.*\\[\\[(?P\\d+\\.?\\d*)\\]\\]" @@ -191,7 +190,7 @@ defs: filtered: lang: python code: | # keep only if "keep" column is non-zero - result = [p["question"] for p in ${list_of_pairs} if p["keep"]] + result = [p["question"] for p in ${ list_of_pairs } if p["keep"]] - ${filtered} @@ -252,9 +251,9 @@ defs: model: ${teacher_model} input: ${teacher_input} parameters: - stop: ${ [teacher_stop_token] + prompt_data.additional_stop_tokens } - include_stop_sequence: false + stop_sequences: ${ ([teacher_stop_token] + prompt_data.additional_stop_tokens) | join(',') } max_new_tokens: ${prompt_data.max_new_tokens} + temperature: 0 parsed_answer: lang: python code: | # parse model output @@ -339,9 +338,9 @@ defs: model: ${teacher_model} input: ${teacher_input} parameters: - stop: ["${teacher_stop_token}"] - include_stop_sequence: false + stop_sequences: "${teacher_stop_token}" max_new_tokens: ${prompt_data.max_new_tokens} + temperature: 0 parser: spec: { "rating": str } regex: 'Rating.*\[\[(?P\d+\.?\d*)\]\]' diff --git a/examples/talk/12-repeat.pdl b/examples/talk/11-repeat.pdl similarity index 100% rename from examples/talk/12-repeat.pdl rename to examples/talk/11-repeat.pdl diff --git a/examples/talk/2-model-chaining.pdl b/examples/talk/2-model-chaining.pdl index 24279ec12..d3fdd6334 100644 --- a/examples/talk/2-model-chaining.pdl +++ b/examples/talk/2-model-chaining.pdl @@ -1,12 +1,11 @@ description: Model chaining text: -- Hello, -- model: watsonx/ibm/granite-34b-code-instruct +- "Hello\n" +- model: replicate/ibm-granite/granite-3.0-8b-instruct parameters: - stop: ["!"] - include_stop_sequence: true -- "\nTranslate this to French\n" -- model: watsonx/ibm/granite-20b-multilingual + stop_sequences: "!" +- "\nDid you just say Hello?\n" +- model: replicate/ibm-granite/granite-3.0-8b-instruct parameters: - stop: ["!"] - include_stop_sequence: true + stop_sequences: "!" + \ No newline at end of file diff --git a/examples/talk/3-def-use.pdl b/examples/talk/3-def-use.pdl index 8032d4856..325668cca 100644 --- a/examples/talk/3-def-use.pdl +++ b/examples/talk/3-def-use.pdl @@ -1,12 +1,14 @@ description: Variable def and use, model input text: -- Hello, -- model: watsonx/ibm/granite-34b-code-instruct +- "Hello\n" +- model: replicate/ibm-granite/granite-3.0-8b-instruct def: name parameters: - stop: ["!"] - include_stop_sequence: false + stop_sequences: "!" - "\n" -- model: watsonx/ibm/granite-20b-multilingual +- model: replicate/ibm-granite/granite-3.0-8b-instruct input: - "Translate the word ${ name | trim } to French\n" + parameters: + stop_sequences: "\n" + temperature: 0 diff --git a/examples/talk/4-function.pdl b/examples/talk/4-function.pdl index 3a75fb2db..15751c436 100644 --- a/examples/talk/4-function.pdl +++ b/examples/talk/4-function.pdl @@ -6,9 +6,10 @@ text: language: str return: - "\nTranslate the sentence '${ sentence }' to ${ language }.\n" - - model: watsonx/ibm/granite-20b-multilingual + - model: replicate/ibm-granite/granite-3.0-8b-instruct parameters: - stop: ["\n"] + stop_sequences: "\n" + temperature: 0 - call: translate args: sentence: I love Paris! diff --git a/examples/talk/5-code-eval.pdl b/examples/talk/5-code-eval.pdl index 6d0a15c29..b61e6b7ae 100644 --- a/examples/talk/5-code-eval.pdl +++ b/examples/talk/5-code-eval.pdl @@ -7,7 +7,7 @@ defs: read: ./ground_truth.txt text: - "\n${ CODE.source_code }\n" -- model: watsonx/ibm/granite-34b-code-instruct +- model: replicate/ibm-granite/granite-3.0-8b-instruct def: EXPLANATION input: | Here is some info about the location of the function in the repo. diff --git a/examples/talk/6-code-json.pdl b/examples/talk/6-code-json.pdl index 2b72fc495..cfb8561f9 100644 --- a/examples/talk/6-code-json.pdl +++ b/examples/talk/6-code-json.pdl @@ -6,7 +6,7 @@ defs: TRUTH: read: ./ground_truth.txt text: -- model: watsonx/ibm/granite-34b-code-instruct +- model: replicate/ibm-granite/granite-3.0-8b-instruct def: EXPLANATION contribute: [] input: diff --git a/examples/talk/7-chatbot-roles.pdl b/examples/talk/7-chatbot-roles.pdl index f27439d44..5833970bb 100644 --- a/examples/talk/7-chatbot-roles.pdl +++ b/examples/talk/7-chatbot-roles.pdl @@ -1,15 +1,17 @@ description: chatbot, control structures, roles, contribute text: -- include: ../granite/granite_defs.pdl - role: system contribute: [context] - text: ${ SYSTEM_CONTENT_CHAT } + text: You are Granite, an AI language model developed by IBM in 2024. - "Type `quit` to exit this chatbot.\n" - repeat: - - read: - message: ">>> " - def: query - - model: watsonx/ibm/granite-13b-chat-v2 + text: + - read: + message: ">>> " + def: query + contribute: [context] + - model: replicate/ibm-granite/granite-3.0-8b-instruct + - "\n\n" until: ${ query == 'quit'} join: with: "\n\n" diff --git a/examples/talk/8-tools.pdl b/examples/talk/8-tools.pdl index d3deede91..c90faba59 100644 --- a/examples/talk/8-tools.pdl +++ b/examples/talk/8-tools.pdl @@ -5,18 +5,30 @@ text: Act: {"name": "Calc", "arguments": {"expr": "18 + 12 * 3"}} Obs: 54 + What is 9 + 12? + Act: {"name": "Calc", "arguments": {"expr": "9 + 12"}} + Obs: 23 + + A total of 252 qualifying matches were played, and 723 goals were scored. What was the average number of goals per match? Act: {"name": "Calc", "arguments": {"expr": "723 / 252"}} Obs: 2.869047619047619 Out of 1400 participants, 400 passed the test. What percentage is that? - Act: -- def: action - model: watsonx/ibm/granite-8b-code-instruct +- "\n" +- def: rawAction + model: replicate/ibm-granite/granite-3.0-8b-instruct parameters: - stop: ["\n"] + stop_sequences: "\n" +- def: action + lang: python parser: json - spec: {name: str, arguments: {expr: str}} + spec: {name: str, arguments: obj} + contribute: [context] + code: + | + result = '${ rawAction }'.replace("Act: ", "") +- "\n" - if: ${ action.name == "Calc" } then: - "Obs: " diff --git a/examples/talk/9-react.pdl b/examples/talk/9-react.pdl index 476ae6842..114b3932e 100644 --- a/examples/talk/9-react.pdl +++ b/examples/talk/9-react.pdl @@ -1,4 +1,3 @@ -description: react agentic flow text: - | What is the elevation range for the area that the eastern sector of the Colorado orogeny extends into? @@ -19,27 +18,57 @@ text: Tho: Professions of Elia Kazan are director, producer, screenwriter, and actor. So profession Nicholas Ray and Elia Kazan have in common is director, screenwriter, and actor. Act: {"name": "Finish", "arguments": {"topic": "director, screenwriter, actor"}} - When was the discoverer of the Hudson River born? + What is 18 + 12 x 3? + Tho: I need to call a calculator. + Act: {"name": "Calc", "arguments": {"expr": "18 + 12 * 3"}} + Obs: 54 + Act: {"name": "Finish", "arguments": {"topic": "54"}} + + A total of 252 qualifying matches were played, and 723 goals were scored. What was the average number of goals per match? + Tho: I need to call a calculator. + Act: {"name": "Calc", "arguments": {"expr": "723 / 252"}} + Obs: 2.869047619047619 + Act: {"name": "Finish", "arguments": {"topic": "2.869047619047619"}} + + What is 18 + 12 x 3? + Tho: I need to call a calculator. + Act: {"name": "Calc", "arguments": {"expr": "18 + 12 * 3"}} + Obs: 54 + Act: {"name": "Finish", "arguments": {"topic": "54"}} + + A total of 252 qualifying matches were played, and 723 goals were scored. What was the average number of goals per match? + Tho: I need to call a calculator. + Act: {"name": "Calc", "arguments": {"expr": "723 / 252"}} + Obs: 2.869047619047619 + Act: {"name": "Finish", "arguments": {"topic": "2.869047619047619"}} + + + when was the discoverer of the Hudson River born? - repeat: text: - def: thought - model: watsonx/ibm/granite-34b-code-instruct + model: replicate/ibm-granite/granite-3.0-8b-instruct parameters: - stop: - - "Act:" - include_stop_sequence: true - - def: action - model: watsonx/ibm/granite-34b-code-instruct + stop_sequences: "Act:" + temperature: 0 + - def: rawAction + model: replicate/ibm-granite/granite-3.0-8b-instruct parameters: - stop: - - "\n" + stop_sequences: "\n" + temperature: 0 + - def: action + lang: python parser: json - spec: {name: str, arguments: {topic: str}} + spec: {name: str, arguments: obj} + contribute: [context] + code: + | + result = '${ rawAction }'.replace("Act: ", "") - def: observation - if: '${ action.name == "Search" }' + if: ${ action.name == "Search" } then: text: - - "Obs: " + - "\nObs: " - lang: python code: | import warnings, wikipedia @@ -48,5 +77,6 @@ text: result = wikipedia.summary("${ action.arguments.topic }") except wikipedia.WikipediaException as e: result = str(e) - until: '${ action.name != "Search" }' + - "\n" + until: ${ action.name != "Search" } diff --git a/examples/talk/out.json b/examples/talk/out.json deleted file mode 100644 index 52f290708..000000000 --- a/examples/talk/out.json +++ /dev/null @@ -1 +0,0 @@ -{"kind": "text", "description": "Code explanation example", "defs": {"CODE": {"kind": "read", "defs": {}, "read": "./data.yaml", "message": null, "multiline": false, "result": "source_code: \n |\n @SuppressWarnings(\"unchecked\")\n public static Map deserializeOffsetMap(String lastSourceOffset) throws IOException {\n Map offsetMap;\n if (lastSourceOffset == null || lastSourceOffset.isEmpty()) { \n offsetMap = new HashMap<>(); \n } else {\n offsetMap = JSON_MAPPER.readValue(lastSourceOffset, Map.class); \n }\n return offsetMap;\n }\nrepo_info: \n repo: streamsets/datacollector\n path: stagesupport/src/main/java/com/.../OffsetUtil.java\n function_name: OffsetUtil.deserializeOffsetMap", "parser": "yaml"}, "TRUTH": {"kind": "read", "defs": {}, "read": "./ground_truth.txt", "message": null, "multiline": false, "result": "The function `deserializeOffsetMap` takes a string as input and returns a map. It first checks if the input string is null or empty. If it is, it creates a new empty map and returns it. Otherwise, it uses the Jackson library to parse the input string into a map and returns it.\n\nThe `@SuppressWarnings(\"unchecked\")` annotation is used to suppress the warning that the type of the parsed map is not checked. This is because the Jackson library is used to parse the input string into a map, but the specific type of the map is not known at compile time. Therefore, the warning is suppressed to avoid potential issues."}}, "text": ["\n@SuppressWarnings(\"unchecked\")\npublic static Map deserializeOffsetMap(String lastSourceOffset) throws IOException {\n Map offsetMap;\n if (lastSourceOffset == null || lastSourceOffset.isEmpty()) { \n offsetMap = new HashMap<>(); \n } else {\n offsetMap = JSON_MAPPER.readValue(lastSourceOffset, Map.class); \n }\n return offsetMap;\n}\n\n", {"kind": "model", "defs": {}, "platform": "litellm", "model": "watsonx/ibm/granite-34b-code-instruct", "input": "Here is some info about the location of the function in the repo.\nrepo: \n${ CODE.repo_info.repo }\npath: ${ CODE.repo_info.path }\nFunction_name: ${ CODE.repo_info.function_name }\n\n\nExplain the following code:\n```\n${ CODE.source_code }```\n", "def": "EXPLANATION", "result": "\nThe function `deserializeOffsetMap` takes a string as input and returns a map. It first checks if the input string is null or empty. If it is, it creates a new empty map and returns it. Otherwise, it uses the Jackson library to parse the input string into a map and returns it.\n\nThe `@SuppressWarnings(\"unchecked\")` annotation is used to suppress the warning that the type of the parsed map is not checked. This is because the Jackson library is used to parse the input string into a map, but the specific type of the map is not known at compile time. Therefore, the warning is suppressed to avoid potential issues."}, "\n\nEVALUATION:\nThe similarity (Levenshtein) between this answer and the ground truth is:\n", {"kind": "code", "defs": {}, "lang": "python", "code": "import textdistance\nexpl = \"\"\"\n\nThe function `deserializeOffsetMap` takes a string as input and returns a map. It first checks if the input string is null or empty. If it is, it creates a new empty map and returns it. Otherwise, it uses the Jackson library to parse the input string into a map and returns it.\n\nThe `@SuppressWarnings(\"unchecked\")` annotation is used to suppress the warning that the type of the parsed map is not checked. This is because the Jackson library is used to parse the input string into a map, but the specific type of the map is not known at compile time. Therefore, the warning is suppressed to avoid potential issues.\n\"\"\"\ntruth = \"\"\"\nThe function `deserializeOffsetMap` takes a string as input and returns a map. It first checks if the input string is null or empty. If it is, it creates a new empty map and returns it. Otherwise, it uses the Jackson library to parse the input string into a map and returns it.\n\nThe `@SuppressWarnings(\"unchecked\")` annotation is used to suppress the warning that the type of the parsed map is not checked. This is because the Jackson library is used to parse the input string into a map, but the specific type of the map is not known at compile time. Therefore, the warning is suppressed to avoid potential issues.\n\"\"\"\nresult = textdistance.levenshtein.normalized_similarity(expl, truth)\n", "def": "EVAL", "result": 0.9983818770226537}], "result": "\n@SuppressWarnings(\"unchecked\")\npublic static Map deserializeOffsetMap(String lastSourceOffset) throws IOException {\n Map offsetMap;\n if (lastSourceOffset == null || lastSourceOffset.isEmpty()) { \n offsetMap = new HashMap<>(); \n } else {\n offsetMap = JSON_MAPPER.readValue(lastSourceOffset, Map.class); \n }\n return offsetMap;\n}\n\n\nThe function `deserializeOffsetMap` takes a string as input and returns a map. It first checks if the input string is null or empty. If it is, it creates a new empty map and returns it. Otherwise, it uses the Jackson library to parse the input string into a map and returns it.\n\nThe `@SuppressWarnings(\"unchecked\")` annotation is used to suppress the warning that the type of the parsed map is not checked. This is because the Jackson library is used to parse the input string into a map, but the specific type of the map is not known at compile time. Therefore, the warning is suppressed to avoid potential issues.\n\nEVALUATION:\nThe similarity (Levenshtein) between this answer and the ground truth is:\n0.9983818770226537"} \ No newline at end of file diff --git a/examples/talk/react_fun.pdl b/examples/talk/react_fun.pdl deleted file mode 100644 index 947e13ad0..000000000 --- a/examples/talk/react_fun.pdl +++ /dev/null @@ -1,118 +0,0 @@ -description: Function definition for react -defs: - react_inner: - function: - examples: [str] - question: str - model: str - return: - text: - - for: - ex: ${ examples } - repeat: - "${ ex }\n" - - "\n" - - ${ question } - - "\n" - - def: prev_action - contribute: [] - data: none - - def: exit - contribute: [] - data: False - - repeat: - text: - - def: thought - model: ${ model } - parameters: - stop: - - "Act:" - include_stop_sequence: true - - def: action - model: ${ model } - parameters: - stop: - - "\n" - parser: json - spec: {name: str, arguments: obj} - - if: ${ action != prev_action} - then: - - def: observation - if: ${ action.name == "Search" } - then: - text: - - "Obs: " - - lang: python - code: | - import warnings, wikipedia - warnings.simplefilter("ignore") - try: - result = wikipedia.summary("${ action.arguments.topic }") - except wikipedia.WikipediaException as e: - result = str(e) - else: - - if: ${ action.name == "Calc" } - then: - text: - - "Obs: " - - lang: python - code: result = ${ action.arguments.expr } - else: - def: exit - contribute: [] - data: True - - def: prev_action - contribute: [] - data: ${ action } - until: ${ action.name == "Finish" or exit } - - react: - function: - question: str - model: str - return: - - defs: - examples: - array: - - text: - | - What is the elevation range for the area that the eastern sector of the Colorado orogeny extends into? - Tho: I need to search Colorado orogeny, find the area that the eastern sector of the Colorado ... - Act: {"name": "Search", "arguments": {"topic": "Colorado orogeny"}} - Obs: The Colorado orogeny was an episode of mountain building (an orogeny) ... - Tho: It does not mention the eastern sector. So I need to look up eastern sector. - Tho: High Plains rise in elevation from around 1,800 to 7,000 ft, so the answer is 1,800 to 7,000 ft. - Act: {"name": "Finish", "arguments": {"topic": "1,800 to 7,000 ft"}} - - - text: - | - What profession does Nicholas Ray and Elia Kazan have in common? - Tho: I need to search Nicholas Ray and Elia Kazan, find their professions, then find the profession they have in common. - Act: {"name": "Search", "arguments": {"topic": "Nicholas Ray"}} - Obs: Nicholas Ray (born Raymond Nicholas Kienzle Jr., August 7, 1911 - June 16, 1979) was an American film director, screenwriter, and actor best known for the 1955 film Rebel Without a Cause. - Tho: Professions of Nicholas Ray are director, screenwriter, and actor. I need to search Elia Kazan next and find his professions. - Act: {"name": "Search", "arguments": {"topic": "Elia Kazan"}} - Obs: Elia Kazan was an American film and theatre director, producer, screenwriter and actor. - Tho: Professions of Elia Kazan are director, producer, screenwriter, and actor. So profession Nicholas Ray and Elia Kazan have in common is director, screenwriter, and actor. - Act: {"name": "Finish", "arguments": {"topic": "director, screenwriter, actor"}} - - - text: - | - What is 18 + 12 x 3? - Act: {"name": "Calc", "arguments": {"expr": "18 + 12 * 3"}} - Obs: 54 - Act: {"name": "Finish", "arguments": {"topic": "54"}} - - - text: - | - A total of 252 qualifying matches were played, and 723 goals were scored. What was the average number of goals per match? - Act: {"name": "Calc", "arguments": {"expr": "723 / 252"}} - Obs: 2.869047619047619 - Act: {"name": "Finish", "arguments": {"topic": "2.869047619047619"}} - - call: react_inner - args: - pdl_context: [] - examples: ${ examples } - question: ${ question } - model: ${ model } - diff --git a/examples/teacher/teacher.pdl b/examples/teacher/teacher.pdl index d518b3a99..b31763834 100644 --- a/examples/teacher/teacher.pdl +++ b/examples/teacher/teacher.pdl @@ -1,6 +1,6 @@ defs: teacher_sys_prompt: You are a very knowledgeable AI Assistant that will faithfully assist the user with their task. - teacher_model: watsonx/mistralai/mixtral-8x7b-instruct-v01 + teacher_model: replicate/ibm-granite/granite-3.0-8b-instruct teacher_template: function: sys_prompt: str @@ -67,8 +67,8 @@ defs: model: ${teacher_model} input: ${teacher_input} parameters: - stop: ["${teacher_stop_token}"] - include_stop_sequence: false + temperature: 0 + stop_sequences: "${teacher_stop_token}" max_new_tokens: ${prompt_data.max_new_tokens} parser: regex: '### Question [0-9]+:\s*([^#\n]+)' @@ -156,9 +156,9 @@ defs: model: ${teacher_model} input: ${teacher_input} parameters: - stop: ["${teacher_stop_token}"] - include_stop_sequence: false + stop_sequences: "${teacher_stop_token}" max_new_tokens: ${prompt_data.max_new_tokens} + temperature: 0 parser: spec: { "rating": str } # regex: "Rating.*\\[\\[(?P\\d+\\.?\\d*)\\]\\]" @@ -190,7 +190,7 @@ defs: filtered: lang: python code: | # keep only if "keep" column is non-zero - result = [p["question"] for p in ${list_of_pairs} if p["keep"]] + result = [p["question"] for p in ${ list_of_pairs } if p["keep"]] - ${filtered} @@ -251,9 +251,9 @@ defs: model: ${teacher_model} input: ${teacher_input} parameters: - stop: ${ [teacher_stop_token] + prompt_data.additional_stop_tokens } - include_stop_sequence: false + stop_sequences: ${ ([teacher_stop_token] + prompt_data.additional_stop_tokens) | join(',') } max_new_tokens: ${prompt_data.max_new_tokens} + temperature: 0 parsed_answer: lang: python code: | # parse model output @@ -338,9 +338,9 @@ defs: model: ${teacher_model} input: ${teacher_input} parameters: - stop: ["${teacher_stop_token}"] - include_stop_sequence: false + stop_sequences: "${teacher_stop_token}" max_new_tokens: ${prompt_data.max_new_tokens} + temperature: 0 parser: spec: { "rating": str } regex: 'Rating.*\[\[(?P\d+\.?\d*)\]\]' @@ -404,4 +404,4 @@ text: - call: filter_question_answer_pair args: qa_pairs: ${qa_pairs} -- "\n" + diff --git a/examples/tools/calc.pdl b/examples/tools/calc.pdl index ffb90768d..c90faba59 100644 --- a/examples/tools/calc.pdl +++ b/examples/tools/calc.pdl @@ -1,25 +1,36 @@ -description: Calculator tool use using Granite models via watsonx.ai. +description: tool use text: - |- What is 18 + 12 x 3? Act: {"name": "Calc", "arguments": {"expr": "18 + 12 * 3"}} Obs: 54 + What is 9 + 12? + Act: {"name": "Calc", "arguments": {"expr": "9 + 12"}} + Obs: 23 + + A total of 252 qualifying matches were played, and 723 goals were scored. What was the average number of goals per match? Act: {"name": "Calc", "arguments": {"expr": "723 / 252"}} Obs: 2.869047619047619 Out of 1400 participants, 400 passed the test. What percentage is that? - Act: -- def: action - model: watsonx/ibm/granite-8b-code-instruct +- "\n" +- def: rawAction + model: replicate/ibm-granite/granite-3.0-8b-instruct parameters: - STOP_SEQUENCES: ["\n"] + stop_sequences: "\n" +- def: action + lang: python parser: json - spec: {name: str, arguments: {expr: str}} + spec: {name: str, arguments: obj} + contribute: [context] + code: + | + result = '${ rawAction }'.replace("Act: ", "") +- "\n" - if: ${ action.name == "Calc" } then: - "Obs: " - lang: python code: result = ${ action.arguments.expr } -- "\n" diff --git a/examples/tutorial/calling_apis.pdl b/examples/tutorial/calling_apis.pdl index c3db1ff86..266490b3b 100644 --- a/examples/tutorial/calling_apis.pdl +++ b/examples/tutorial/calling_apis.pdl @@ -4,21 +4,18 @@ text: def: QUERY message: "Ask a query: " contribute: [] -- model: watsonx/ibm/granite-34b-code-instruct +- model: replicate/ibm-granite/granite-3.0-8b-instruct input: | + Extract the location from the question. Question: What is the weather in London? - London + Answer: London Question: What's the weather in Paris? - Paris + Answer: Paris Question: Tell me the weather in Lagos? - Lagos + Answer: Lagos Question: ${ QUERY } parameters: - stop: - - Question - - What - - '!' - include_stop_sequence: false + stop_sequences: "Question,What,!,\n" def: LOCATION contribute: [] - lang: python @@ -30,7 +27,8 @@ text: parser: json contribute: [] -- model: watsonx/ibm/granite-34b-code-instruct +- model: replicate/ibm-granite/granite-3.0-8b-instruct input: | Explain the weather from the following JSON: ${ WEATHER } + diff --git a/examples/tutorial/calling_llm.pdl b/examples/tutorial/calling_llm.pdl index dcecd7c44..bdf3180e7 100644 --- a/examples/tutorial/calling_llm.pdl +++ b/examples/tutorial/calling_llm.pdl @@ -1,9 +1,6 @@ description: Hello world calling a model text: -- Hello, -- model: watsonx/ibm/granite-34b-code-instruct +- "Hello\n" +- model: replicate/ibm-granite/granite-3.0-8b-instruct parameters: - decoding_method: greedy - stop: - - '!' - include_stop_sequence: true \ No newline at end of file + stop_sequences: '!' \ No newline at end of file diff --git a/examples/tutorial/calling_llm_with_input.pdl b/examples/tutorial/calling_llm_with_input.pdl index f9134b9ed..d03e870c5 100644 --- a/examples/tutorial/calling_llm_with_input.pdl +++ b/examples/tutorial/calling_llm_with_input.pdl @@ -1,6 +1,6 @@ description: Hello world calling a model text: -- "Hello, " -- model: watsonx/ibm/granite-20b-multilingual +- "Hello\n" +- model: replicate/ibm-granite/granite-3.0-8b-instruct input: - Translate the word 'world' to French + Translate the word 'Hello' to French diff --git a/examples/tutorial/calling_llm_with_input_messages.pdl b/examples/tutorial/calling_llm_with_input_messages.pdl index 8fb02679c..afee7a37e 100644 --- a/examples/tutorial/calling_llm_with_input_messages.pdl +++ b/examples/tutorial/calling_llm_with_input_messages.pdl @@ -1,10 +1,10 @@ description: Hello world calling a model text: -- "Hello, " -- model: watsonx/ibm/granite-20b-multilingual +- "Hello\n" +- model: replicate/ibm-granite/granite-3.0-8b-instruct input: array: - role: system content: You are a helpful assistant that is fluent in French. - role: user - content: Translate the word 'world' to French + content: Translate the word 'Hello' to French diff --git a/examples/tutorial/conditionals_loops.pdl b/examples/tutorial/conditionals_loops.pdl index 0ac35cbd5..2ae7078d9 100644 --- a/examples/tutorial/conditionals_loops.pdl +++ b/examples/tutorial/conditionals_loops.pdl @@ -5,7 +5,7 @@ text: contribute: [context] - repeat: text: - - model: watsonx/ibm/granite-13b-chat-v2 + - model: replicate/ibm-granite/granite-3.0-8b-instruct - read: def: eval message: "\nIs this a good answer[yes/no]?\n" @@ -15,4 +15,4 @@ text: text: - read: message: "Why not?\n" - until: ${ eval == 'yes'} \ No newline at end of file + until: ${ eval == 'yes'} diff --git a/examples/tutorial/data_block.pdl b/examples/tutorial/data_block.pdl index a81c2d267..1dd78d532 100644 --- a/examples/tutorial/data_block.pdl +++ b/examples/tutorial/data_block.pdl @@ -6,7 +6,7 @@ defs: TRUTH: read: ./ground_truth.txt lastOf: -- model: watsonx/ibm/granite-34b-code-instruct +- model: replicate/ibm-granite/granite-3.0-8b-instruct def: EXPLANATION input: | diff --git a/examples/tutorial/function_definition.pdl b/examples/tutorial/function_definition.pdl index f104b6778..15751c436 100644 --- a/examples/tutorial/function_definition.pdl +++ b/examples/tutorial/function_definition.pdl @@ -6,9 +6,10 @@ text: language: str return: - "\nTranslate the sentence '${ sentence }' to ${ language }.\n" - - model: watsonx/ibm/granite-20b-multilingual + - model: replicate/ibm-granite/granite-3.0-8b-instruct parameters: - stop: ["\n"] + stop_sequences: "\n" + temperature: 0 - call: translate args: sentence: I love Paris! @@ -17,4 +18,4 @@ text: - call: translate args: sentence: I love Madrid! - language: Spanish \ No newline at end of file + language: Spanish diff --git a/examples/tutorial/gen-data.pdl b/examples/tutorial/gen-data.pdl index 0d7a7c121..a22c2d376 100644 --- a/examples/tutorial/gen-data.pdl +++ b/examples/tutorial/gen-data.pdl @@ -5,7 +5,7 @@ defs: parser: yaml spec: { questions: [str], answers: [obj] } text: - - model: watsonx/ibm/granite-20b-code-instruct + - model: replicate/ibm-granite/granite-3.0-8b-instruct def: model_output spec: {name: str, age: int} input: @@ -22,7 +22,5 @@ text: and set them appropriately. Write the age in letters. parser: yaml parameters: - stop: - - '}' - include_stop_sequence: true - \ No newline at end of file + stop_sequences: "\n" + temperature: 0 diff --git a/examples/tutorial/grouping_definitions.pdl b/examples/tutorial/grouping_definitions.pdl index 49623a280..88cf3e05b 100644 --- a/examples/tutorial/grouping_definitions.pdl +++ b/examples/tutorial/grouping_definitions.pdl @@ -6,9 +6,9 @@ defs: language: str return: - "\nTranslate the sentence '${ sentence }' to ${ language }.\n" - - model: watsonx/ibm/granite-20b-multilingual + - model: replicate/ibm-granite/granite-3.0-8b-instruct parameters: - stop: ["\n"] + stop_sequences: "\n" text: - call: translate args: diff --git a/examples/tutorial/include.pdl b/examples/tutorial/include.pdl index 733ccc080..5f95e5e36 100644 --- a/examples/tutorial/include.pdl +++ b/examples/tutorial/include.pdl @@ -6,14 +6,14 @@ text: def: prompts spec: {prompts: [str]} contribute: [] -- ${ SYSTEM_CONTENT_CHAT } - for: prompt: ${ prompts.prompts } repeat: text: - | + ${ prompt } - - model: watsonx/ibm/granite-13b-chat-v2 + - model: replicate/ibm-granite/granite-3.0-8b-instruct parameters: decoding_method: sample max_new_tokens: 512 diff --git a/examples/tutorial/model_chaining.pdl b/examples/tutorial/model_chaining.pdl index 24279ec12..ac52de8c1 100644 --- a/examples/tutorial/model_chaining.pdl +++ b/examples/tutorial/model_chaining.pdl @@ -1,12 +1,10 @@ description: Model chaining text: -- Hello, -- model: watsonx/ibm/granite-34b-code-instruct +- "Hello\n" +- model: replicate/ibm-granite/granite-3.0-8b-instruct parameters: - stop: ["!"] - include_stop_sequence: true -- "\nTranslate this to French\n" -- model: watsonx/ibm/granite-20b-multilingual + stop_sequences: "!" +- "\nDid you just say Hello?\n" +- model: replicate/ibm-granite/granite-3.0-8b-instruct parameters: - stop: ["!"] - include_stop_sequence: true + stop_sequences: "!" diff --git a/examples/tutorial/muting_block_output.pdl b/examples/tutorial/muting_block_output.pdl index 99a7ef4b2..ce1b0bf80 100644 --- a/examples/tutorial/muting_block_output.pdl +++ b/examples/tutorial/muting_block_output.pdl @@ -6,9 +6,9 @@ defs: language: str return: - "\nTranslate the sentence '${ sentence }' to ${ language }.\n" - - model: watsonx/ibm/granite-20b-multilingual + - model: replicate/ibm-granite/granite-3.0-8b-instruct parameters: - stop: ["\n"] + stop_sequences: "\n" text: - call: translate contribute: [] diff --git a/examples/tutorial/variable_def_use.pdl b/examples/tutorial/variable_def_use.pdl index 2ab30afdb..112ec6db7 100644 --- a/examples/tutorial/variable_def_use.pdl +++ b/examples/tutorial/variable_def_use.pdl @@ -1,11 +1,8 @@ description: Hello world with variable def and use text: -- Hello, -- model: watsonx/ibm/granite-34b-code-instruct +- "Hello\n" +- model: replicate/ibm-granite/granite-3.0-8b-instruct def: GEN parameters: - decoding_method: greedy - stop: - - '!' - include_stop_sequence: true + stop_sequences: '!' - "\nGEN is equal to: ${ GEN }" \ No newline at end of file diff --git a/examples/weather/weather.pdl b/examples/weather/weather.pdl index d8f7f6ada..266490b3b 100644 --- a/examples/weather/weather.pdl +++ b/examples/weather/weather.pdl @@ -4,21 +4,18 @@ text: def: QUERY message: "Ask a query: " contribute: [] -- model: watsonx/ibm/granite-34b-code-instruct +- model: replicate/ibm-granite/granite-3.0-8b-instruct input: | + Extract the location from the question. Question: What is the weather in London? - London + Answer: London Question: What's the weather in Paris? - Paris + Answer: Paris Question: Tell me the weather in Lagos? - Lagos + Answer: Lagos Question: ${ QUERY } parameters: - stop: - - Question - - What - - '!' - include_stop_sequence: false + stop_sequences: "Question,What,!,\n" def: LOCATION contribute: [] - lang: python @@ -30,9 +27,8 @@ text: parser: json contribute: [] -- model: watsonx/ibm/granite-34b-code-instruct +- model: replicate/ibm-granite/granite-3.0-8b-instruct input: | Explain the weather from the following JSON: ${ WEATHER } - diff --git a/out.json b/out.json deleted file mode 100644 index 52f290708..000000000 --- a/out.json +++ /dev/null @@ -1 +0,0 @@ -{"kind": "text", "description": "Code explanation example", "defs": {"CODE": {"kind": "read", "defs": {}, "read": "./data.yaml", "message": null, "multiline": false, "result": "source_code: \n |\n @SuppressWarnings(\"unchecked\")\n public static Map deserializeOffsetMap(String lastSourceOffset) throws IOException {\n Map offsetMap;\n if (lastSourceOffset == null || lastSourceOffset.isEmpty()) { \n offsetMap = new HashMap<>(); \n } else {\n offsetMap = JSON_MAPPER.readValue(lastSourceOffset, Map.class); \n }\n return offsetMap;\n }\nrepo_info: \n repo: streamsets/datacollector\n path: stagesupport/src/main/java/com/.../OffsetUtil.java\n function_name: OffsetUtil.deserializeOffsetMap", "parser": "yaml"}, "TRUTH": {"kind": "read", "defs": {}, "read": "./ground_truth.txt", "message": null, "multiline": false, "result": "The function `deserializeOffsetMap` takes a string as input and returns a map. It first checks if the input string is null or empty. If it is, it creates a new empty map and returns it. Otherwise, it uses the Jackson library to parse the input string into a map and returns it.\n\nThe `@SuppressWarnings(\"unchecked\")` annotation is used to suppress the warning that the type of the parsed map is not checked. This is because the Jackson library is used to parse the input string into a map, but the specific type of the map is not known at compile time. Therefore, the warning is suppressed to avoid potential issues."}}, "text": ["\n@SuppressWarnings(\"unchecked\")\npublic static Map deserializeOffsetMap(String lastSourceOffset) throws IOException {\n Map offsetMap;\n if (lastSourceOffset == null || lastSourceOffset.isEmpty()) { \n offsetMap = new HashMap<>(); \n } else {\n offsetMap = JSON_MAPPER.readValue(lastSourceOffset, Map.class); \n }\n return offsetMap;\n}\n\n", {"kind": "model", "defs": {}, "platform": "litellm", "model": "watsonx/ibm/granite-34b-code-instruct", "input": "Here is some info about the location of the function in the repo.\nrepo: \n${ CODE.repo_info.repo }\npath: ${ CODE.repo_info.path }\nFunction_name: ${ CODE.repo_info.function_name }\n\n\nExplain the following code:\n```\n${ CODE.source_code }```\n", "def": "EXPLANATION", "result": "\nThe function `deserializeOffsetMap` takes a string as input and returns a map. It first checks if the input string is null or empty. If it is, it creates a new empty map and returns it. Otherwise, it uses the Jackson library to parse the input string into a map and returns it.\n\nThe `@SuppressWarnings(\"unchecked\")` annotation is used to suppress the warning that the type of the parsed map is not checked. This is because the Jackson library is used to parse the input string into a map, but the specific type of the map is not known at compile time. Therefore, the warning is suppressed to avoid potential issues."}, "\n\nEVALUATION:\nThe similarity (Levenshtein) between this answer and the ground truth is:\n", {"kind": "code", "defs": {}, "lang": "python", "code": "import textdistance\nexpl = \"\"\"\n\nThe function `deserializeOffsetMap` takes a string as input and returns a map. It first checks if the input string is null or empty. If it is, it creates a new empty map and returns it. Otherwise, it uses the Jackson library to parse the input string into a map and returns it.\n\nThe `@SuppressWarnings(\"unchecked\")` annotation is used to suppress the warning that the type of the parsed map is not checked. This is because the Jackson library is used to parse the input string into a map, but the specific type of the map is not known at compile time. Therefore, the warning is suppressed to avoid potential issues.\n\"\"\"\ntruth = \"\"\"\nThe function `deserializeOffsetMap` takes a string as input and returns a map. It first checks if the input string is null or empty. If it is, it creates a new empty map and returns it. Otherwise, it uses the Jackson library to parse the input string into a map and returns it.\n\nThe `@SuppressWarnings(\"unchecked\")` annotation is used to suppress the warning that the type of the parsed map is not checked. This is because the Jackson library is used to parse the input string into a map, but the specific type of the map is not known at compile time. Therefore, the warning is suppressed to avoid potential issues.\n\"\"\"\nresult = textdistance.levenshtein.normalized_similarity(expl, truth)\n", "def": "EVAL", "result": 0.9983818770226537}], "result": "\n@SuppressWarnings(\"unchecked\")\npublic static Map deserializeOffsetMap(String lastSourceOffset) throws IOException {\n Map offsetMap;\n if (lastSourceOffset == null || lastSourceOffset.isEmpty()) { \n offsetMap = new HashMap<>(); \n } else {\n offsetMap = JSON_MAPPER.readValue(lastSourceOffset, Map.class); \n }\n return offsetMap;\n}\n\n\nThe function `deserializeOffsetMap` takes a string as input and returns a map. It first checks if the input string is null or empty. If it is, it creates a new empty map and returns it. Otherwise, it uses the Jackson library to parse the input string into a map and returns it.\n\nThe `@SuppressWarnings(\"unchecked\")` annotation is used to suppress the warning that the type of the parsed map is not checked. This is because the Jackson library is used to parse the input string into a map, but the specific type of the map is not known at compile time. Therefore, the warning is suppressed to avoid potential issues.\n\nEVALUATION:\nThe similarity (Levenshtein) between this answer and the ground truth is:\n0.9983818770226537"} \ No newline at end of file diff --git a/src/pdl/pdl_ast.py b/src/pdl/pdl_ast.py index 2a78348bd..6d9c9e55f 100644 --- a/src/pdl/pdl_ast.py +++ b/src/pdl/pdl_ast.py @@ -620,6 +620,7 @@ def set_default_granite_model_parameters( ) -> dict[str, Any]: if parameters is None: parameters = {} + if "decoding_method" not in parameters: parameters["decoding_method"] = ( DECODING_METHOD # pylint: disable=attribute-defined-outside-init @@ -649,7 +650,10 @@ def set_default_granite_model_parameters( parameters["top_p"] = ( TOP_P_SAMPLING # pylint: disable=attribute-defined-outside-init ) - if "granite-3b" in model_id or "granite-8b" in model_id: + if "granite-3.0" in model_id: + if "temperature" not in parameters or parameters["temperature"] is None: + parameters["temperature"] = (0) # setting to decoding greedy + if "roles" not in parameters: parameters["roles"] = { "system": { diff --git a/tests/results/examples/chatbot/chatbot.result b/tests/results/examples/chatbot/chatbot.result index b18e9f544..a76e79640 100644 --- a/tests/results/examples/chatbot/chatbot.result +++ b/tests/results/examples/chatbot/chatbot.result @@ -1,3 +1 @@ -What is APR? -Annual Percentage Rate (APR) is the total cost of borrowing money, expressed as a percentage. It includes the interest rate and any additional fees or charges associated with the loan. APR is used to compare different loan offers and to understand the true cost of borrowing money over time. - +APR stands for Annual Percentage Rate. It is the annual interest rate charged for borrowing or earned through an investment, and it represents the actual yearly cost of funds over the term of a loan. It includes any fees or additional costs associated with the transaction. diff --git a/tests/results/examples/code/code-eval.result b/tests/results/examples/code/code-eval.result index c9e156623..044b0d34f 100644 --- a/tests/results/examples/code/code-eval.result +++ b/tests/results/examples/code/code-eval.result @@ -10,11 +10,15 @@ public static Map deserializeOffsetMap(String lastSourceOffset) return offsetMap; } +This Java method, `deserializeOffsetMap`, is used to deserialize a JSON string into a `Map`. Here's a breakdown of the code: -The function `deserializeOffsetMap` takes a string as input and returns a map. It first checks if the input string is null or empty. If it is, it creates a new empty map and returns it. Otherwise, it uses the Jackson library to parse the input string into a map and returns it. +1. The method takes a single parameter, `lastSourceOffset`, which is expected to be a JSON string. +2. It first checks if `lastSourceOffset` is either `null` or an empty string. If true, it initializes an empty `HashMap` and assigns it to `offsetMap`. +3. If `lastSourceOffset` is not `null` or empty, it uses the `JSON_MAPPER` object (presumably an instance of `ObjectMapper` from the Jackson library) to deserialize the JSON string into a `Map`. The `readValue` method is used for this purpose, with `Map.class` as the second argument, which specifies the expected type of the deserialized object. +4. Finally, the method returns the `offsetMap`, which now contains the deserialized data. -The `@SuppressWarnings("unchecked")` annotation is used to suppress the warning that the type of the parsed map is not checked. This is because the Jackson library is used to parse the input string into a map, but the specific type of the map is not known at compile time. Therefore, the warning is suppressed to avoid potential issues. +The `@SuppressWarnings("unchecked")` annotation is used to suppress a potential unchecked warning that might occur due to the raw `Map` type used in the method signature. EVALUATION: The similarity (Levenshtein) between this answer and the ground truth is: -0.9967637540453075 +0.30421982335623154 diff --git a/tests/results/examples/code/code-json.result b/tests/results/examples/code/code-json.result index 09c640713..4025a6142 100644 --- a/tests/results/examples/code/code-json.result +++ b/tests/results/examples/code/code-json.result @@ -1 +1 @@ -{"input": {"source_code": "@SuppressWarnings(\"unchecked\")\npublic static Map deserializeOffsetMap(String lastSourceOffset) throws IOException {\n Map offsetMap;\n if (lastSourceOffset == null || lastSourceOffset.isEmpty()) { \n offsetMap = new HashMap<>(); \n } else {\n offsetMap = JSON_MAPPER.readValue(lastSourceOffset, Map.class); \n }\n return offsetMap;\n}\n", "repo_info": {"repo": "streamsets/datacollector", "path": "stagesupport/src/main/java/com/.../OffsetUtil.java", "function_name": "OffsetUtil.deserializeOffsetMap"}}, "output": "\nThe function `deserializeOffsetMap` takes a string as input and returns a map. It first checks if the input string is null or empty. If it is, it creates a new empty map and returns it. Otherwise, it uses the Jackson library to parse the input string into a map and returns it.\n\nThe `@SuppressWarnings(\"unchecked\")` annotation is used to suppress the warning that the type of the parsed map is not checked. This is because the Jackson library is used to parse the input string into a map, but the specific type of the map is not known at compile time. Therefore, the warning is suppressed to avoid potential issues.", "metric": 0.9967637540453075} +{"input": {"source_code": "@SuppressWarnings(\"unchecked\")\npublic static Map deserializeOffsetMap(String lastSourceOffset) throws IOException {\n Map offsetMap;\n if (lastSourceOffset == null || lastSourceOffset.isEmpty()) { \n offsetMap = new HashMap<>(); \n } else {\n offsetMap = JSON_MAPPER.readValue(lastSourceOffset, Map.class); \n }\n return offsetMap;\n}\n", "repo_info": {"repo": "streamsets/datacollector", "path": "stagesupport/src/main/java/com/.../OffsetUtil.java", "function_name": "OffsetUtil.deserializeOffsetMap"}}, "output": "This Java method, `deserializeOffsetMap`, is used to deserialize a JSON string into a `Map`. Here's a breakdown of the code:\n\n1. The method takes a single parameter, `lastSourceOffset`, which is expected to be a JSON string.\n2. It first checks if `lastSourceOffset` is either `null` or an empty string. If true, it initializes an empty `HashMap` and assigns it to `offsetMap`.\n3. If `lastSourceOffset` is not `null` or empty, it uses the `JSON_MAPPER` object (presumably an instance of `ObjectMapper` from the Jackson library) to deserialize the JSON string into a `Map`. The `readValue` method is used for this purpose, with `Map.class` as the second argument, which specifies the expected type of the deserialized object.\n4. Finally, the method returns the `offsetMap`, which now contains the deserialized data.\n\nThe `@SuppressWarnings(\"unchecked\")` annotation is used to suppress a potential unchecked warning that might occur due to the raw `Map` type used in the method signature.", "metric": 0.30421982335623154} diff --git a/tests/results/examples/code/code.result b/tests/results/examples/code/code.result index dea410598..b05e46388 100644 --- a/tests/results/examples/code/code.result +++ b/tests/results/examples/code/code.result @@ -10,8 +10,11 @@ public static Map deserializeOffsetMap(String lastSourceOffset) return offsetMap; } +This Java method, `deserializeOffsetMap`, is used to deserialize a JSON string into a `Map`. Here's a breakdown of the code: +1. The method takes a single parameter, `lastSourceOffset`, which is expected to be a JSON string. +2. It first checks if `lastSourceOffset` is either `null` or an empty string. If true, it initializes an empty `HashMap` and assigns it to `offsetMap`. +3. If `lastSourceOffset` is not `null` or empty, it uses the `JSON_MAPPER` object (presumably an instance of `ObjectMapper` from the Jackson library) to deserialize the JSON string into a `Map`. The `readValue` method is used for this purpose, with `Map.class` as the second argument, which specifies the expected type of the deserialized object. +4. Finally, the method returns the `offsetMap`, which now contains the deserialized data. -The function `deserializeOffsetMap` takes a string as input and returns a map. It first checks if the input string is null or empty. If it is, it creates a new empty map and returns it. Otherwise, it uses the Jackson library to parse the input string into a map and returns it. - -The `@SuppressWarnings("unchecked")` annotation is used to suppress the warning that the type of the parsed map is not checked. This is because the Jackson library is used to parse the input string into a map, but the specific type of the map is not known at compile time. Therefore, the warning is suppressed to avoid potential issues. +The `@SuppressWarnings("unchecked")` annotation is used to suppress a potential unchecked warning that might occur due to the raw `Map` type used in the method signature. diff --git a/tests/results/examples/demo/3-weather.result b/tests/results/examples/demo/3-weather.result index 32fc549a7..ba7bbb4fe 100644 --- a/tests/results/examples/demo/3-weather.result +++ b/tests/results/examples/demo/3-weather.result @@ -1,2 +1 @@ - -The weather in Yorktown Heights, New York, USA is currently partly cloudy with a temperature of 15.6°C (60.1°F). The wind speed is 10.1 mph (16.2 kph) from the NNW direction. The humidity is 40%, and the visibility is 16 km (9 miles). The UV index is 3.4, and there is no precipitation. The feels-like temperature is 15.6°C (60.1°F), and the wind chill is 13.7°C (56.6°F). The heat index is 14.4°C (58.0°F), and the dew point is 3.8°C (38.8°F). The pressure is 1017 mb (30.02 in). +The current weather in Yorktown Heights, New York, USA is sunny with a temperature of 13.3°C (55.9°F). The wind is blowing at a speed of 7.6 mph (12.2 kph) from the north (N) direction. The pressure is 1031.0 mb (30.43 inches). The humidity is 42% and the dew point is 3.5°C (38.3°F). The visibility is 16.0 km (9.0 miles). The UV index is 1.3 and the wind gusts are up to 9.3 mph (14.9 kph). diff --git a/tests/results/examples/demo/4-translator.result b/tests/results/examples/demo/4-translator.result index 2c100b9ca..ce1317c5a 100644 --- a/tests/results/examples/demo/4-translator.result +++ b/tests/results/examples/demo/4-translator.result @@ -1,5 +1,5 @@ What is APR? -Annual Percentage Rate (APR) is the total cost of borrowing money, expressed as a percentage.french +APR stands for Annual Percentage Rate. It is the annual interest rate charged for borrowing or earned through an investment, and it represents the actual yearly cost of funds over the term of a loan. It includes any fees or additional costs associated with the transaction.french Translate the above to french -La taux de change annuel (TAC) est le coût total du prêt, exprimé en pourcentage.stop +APR signifie Taux Annuel Équivalent (TAE) en français. Il s'agit du taux d'intérêt annuel chargé pour emprunter ou gagné grâce à une investissement, et il représente le coût réel annuel des fonds sur la durée d'un prêt. Il inclut toutes les frais ou coûts supplémentaires associés à la transaction.stop diff --git a/tests/results/examples/fibonacci/fib.result b/tests/results/examples/fibonacci/fib.result index 303fe94da..fab29f20e 100644 --- a/tests/results/examples/fibonacci/fib.result +++ b/tests/results/examples/fibonacci/fib.result @@ -1,43 +1,43 @@ +Here is a simple Python function to compute the Fibonacci sequence: ```python -def fib(n): - if n == 0: - return 0 +def fibonacci(n): + if n <= 0: + return "Input should be a positive integer." elif n == 1: + return 0 + elif n == 2: return 1 else: - return fib(n-1) + fib(n-2) + a, b = 0, 1 + for _ in range(n - 2): + a, b = b, a + b + return b ``` -The above code is a recursive implementation of the Fibonacci sequence. -However, it is not very efficient since it recalculates the same values multiple times. -To improve the performance, we can use dynamic programming to store the previously calculated values and reuse them. - -Here's an optimized version of the Fibonacci function using dynamic programming: - -```python -def fib_dp(n): - fib_values = [0, 1] - for i in range(2, n+1): - fib_values.append(fib_values[i-1] + fib_values[i-2]) - return fib_values[n] -``` +This function takes an integer `n` as input and returns the `n`th number in the Fibonacci sequence. The Fibonacci sequence is a series of numbers in which each number is the sum of the two preceding ones, usually starting with 0 and 1. Find a random number between 1 and 20 15 -Now computing fib(15) +Now computing fibonacci(15) -def fib(n): - if n == 0: - return 0 +def fibonacci(n): + if n <= 0: + return "Input should be a positive integer." elif n == 1: + return 0 + elif n == 2: return 1 else: - return fib(n-1) + fib(n-2) -The result is: 610 + a, b = 0, 1 + for _ in range(n - 2): + a, b = b, a + b + return b +The result is: 377 Explain what the above code does and what the result means +The provided code is a Python function named `fibonacci(n)` that computes the `n`th number in the Fibonacci sequence. The Fibonacci sequence is a series of numbers where each number is the sum of the two preceding ones, usually starting with 0 and 1. -The code calculates the nth Fibonacci number using dynamic programming. It initializes a list `fib_values` with the first two Fibonacci numbers [0, 1]. Then, it iterates from 2 to n and calculates each Fibonacci number by adding the previous two numbers in the list. Finally, it returns the nth Fibonacci number from the list. +The function takes an integer `n` as input and returns the `n`th number in the Fibonacci sequence. It first checks if the input is a positive integer and returns an error message if it's not. If the input is 1, it returns 0 (the first number in the Fibonacci sequence). If the input is 2, it returns 1 (the second number in the Fibonacci sequence). For any other positive integer input, it uses a loop to calculate the `n`th number in the Fibonacci sequence and returns the result. -In this case, the result is 610, which means that the 15th Fibonacci number is 610. +In the given example, the function is called with the argument `15`, which means it will compute the 15th number in the Fibonacci sequence. The result of this computation is `377`. This number is the 15th number in the Fibonacci sequence, which is the sum of the 14th and 13th numbers in the sequence. diff --git a/tests/results/examples/granite/multi_round_chat.result b/tests/results/examples/granite/multi_round_chat.result index b36a0a765..068d20d0d 100644 --- a/tests/results/examples/granite/multi_round_chat.result +++ b/tests/results/examples/granite/multi_round_chat.result @@ -1,29 +1,15 @@ -You are Granite Chat, an AI language model developed by IBM. You are a cautious assistant. You carefully follow instructions. You are helpful and harmless and you follow ethical guidelines and promote positive behavior. You always respond to greetings (for example, hi, hello, g'day, morning, afternoon, evening, night, what's up, nice to meet you, sup, etc) with "Hello! I am Granite Chat, created by IBM. How can I help you today?". Please do not say anything else and do not start a conversation. -What is APR? - -Annual Percentage Rate (APR) is a measure that represents the annual cost of borrowing money, including interest payments and other fees. It is expressed as a percentage and is used to compare different loan offers and determine the true cost of borrowing. APR takes into account the interest rate, points, and other charges associated with the loan, providing a more accurate picture of the total cost of a loan compared to just the interest rate alone. By considering all these factors, APR helps borrowers make informed decisions about which loans offer the best value.Can you write a poem about APR? - -In the realm of lending, there exists a term, -A figure that dance with numbers, both fair and unfair. -Annual Percentage Rate, or APR, it's called, -A tale of interest, wrapped in a mathematical balm. - -It's not just the Interest rate that it entails, -But also the points, the fees, the hidden tangles. -A comprehensive view, it provides, -Of the overall cost, come what may. -Borrowers, take heed, of this true cost, -Not just the interest rate, but the APR's prolong. -For when comparing loans, make sure to delve, -Into the APR's depths, before you succumb. - -APR, a faithful friend, or a foe unseen, -Helps you determine if a loan is right for you. -So, let's embrace this number, so neat, -And navigate the financial market with stealth. - -In the world of finance, APR's the guiding star, -With patience and wisdom, you'll make the most of a far star.Now explain APR to me like I'm 5 years old - -APR is like a special word that grown-ups use when they borrow money from a place like a bank. It's a way to know how much it will cost them to get that money over a year. APR includes not only the interest rate, which is like the cost of borrowing the money, but also things like fees and extra costs. By adding it all up, APR gives a better idea of how much the loan will really cost. So, when choosing a loan, it's important to look at the APR to make sure it's a good deal. +What is APR? +APR stands for Annual Percentage Rate. It is the annual interest rate charged for borrowing or earned through an investment, and it represents the actual yearly cost of funds over the term of a loan. It includes any fees or additional costs associated with the transaction. +Can you write a poem about APR? +In the world of finance, APR is the key, +A number that tells us what we'll pay, or earn, you see. +It's the annual rate, both simple and clear, +Including all fees, for loans and investments, near and far. + +It's the cost of borrowing, or the return we gain, +A figure that helps us make informed financial gain. +So, when you're comparing loans, or investments to choose, +Remember APR, and make the right move. +Now explain APR to me like I'm 5 years old +Sure! Imagine you have a piggy bank, and you want to borrow some money from your parents to buy a toy. Your parents will ask you to pay them back with some extra money, which is like interest. APR is the special number that tells you how much extra money you'll have to pay back, all in one year. It's like a special rule that helps you understand how much you'll owe your parents. diff --git a/tests/results/examples/granite/single_round_chat.result b/tests/results/examples/granite/single_round_chat.result index 03d4014fc..807524844 100644 --- a/tests/results/examples/granite/single_round_chat.result +++ b/tests/results/examples/granite/single_round_chat.result @@ -1,4 +1,4 @@ -{"role": "system", "content": "You are Granite Chat, an AI language model developed by IBM. You are a cautious assistant. You carefully follow instructions. You are helpful and harmless and you follow ethical guidelines and promote positive behavior. You always respond to greetings (for example, hi, hello, g'day, morning, afternoon, evening, night, what's up, nice to meet you, sup, etc) with \"Hello! I am Granite Chat, created by IBM. How can I help you today?\". Please do not say anything else and do not start a conversation.\n"}What is APR? +What is APR? yes -Annual Percentage Rate (APR) is a measure that represents the yearly cost of borrowing money, expressed as a percentage. It includes the interest rate and other fees related to the loan, if any. APR is used to compare different loan offers and is a more accurate representation of the total cost of borrowing than simply looking at the interest rate alone. It's essential to consider both the interest rate and APR when evaluating loan options to make an informed decision. +APR stands for Annual Percentage Rate. It is the annual interest rate charged for borrowing or earned through an investment, and it represents the actual yearly cost of funds over the term of a loan. It includes any fees or additional costs associated with the transaction. diff --git a/tests/results/examples/hello/hello-def-use.result b/tests/results/examples/hello/hello-def-use.result index c80245ebe..9b863a148 100644 --- a/tests/results/examples/hello/hello-def-use.result +++ b/tests/results/examples/hello/hello-def-use.result @@ -1,2 +1,3 @@ -Hello, World -Who is World? +Hello +Hello +You said Hello. diff --git a/tests/results/examples/hello/hello-model-chaining.result b/tests/results/examples/hello/hello-model-chaining.result index f583d329f..0702a9755 100644 --- a/tests/results/examples/hello/hello-model-chaining.result +++ b/tests/results/examples/hello/hello-model-chaining.result @@ -1,3 +1,4 @@ -Hello, World! -Who is World? -World is a fictional character in the popular science fiction television series "The X-Files +Hello +Hello +Did you say Hello? +Yes, I did diff --git a/tests/results/examples/hello/hello-model-input.result b/tests/results/examples/hello/hello-model-input.result index 9fc31828c..e965047ad 100644 --- a/tests/results/examples/hello/hello-model-input.result +++ b/tests/results/examples/hello/hello-model-input.result @@ -1 +1 @@ - World! +Hello diff --git a/tests/results/examples/hello/hello-parser-regex.result b/tests/results/examples/hello/hello-parser-regex.result index 866e5ddb7..a48fa9858 100644 --- a/tests/results/examples/hello/hello-parser-regex.result +++ b/tests/results/examples/hello/hello-parser-regex.result @@ -1 +1 @@ -{"name": "World"} +{"name": "Hello"} diff --git a/tests/results/examples/hello/hello-roles-array.result b/tests/results/examples/hello/hello-roles-array.result index eac1b89d3..c72e5bff4 100644 --- a/tests/results/examples/hello/hello-roles-array.result +++ b/tests/results/examples/hello/hello-roles-array.result @@ -1,36 +1,42 @@ - - -Here is the pseudocode for merge sort: +Here is a Python function that implements the merge sort algorithm: ```python def merge_sort(arr): - if len(arr) > 1: - mid = len(arr) // 2 - left_half = arr[:mid] - right_half = arr[mid:] - - merge_sort(left_half) - merge_sort(right_half) - - i = j = k = 0 - - while i < len(left_half) and j < len(right_half): - if left_half[i] < right_half[j]: - arr[k] = left_half[i] - i += 1 - else: - arr[k] = right_half[j] - j += 1 - k += 1 - - while i < len(left_half): - arr[k] = left_half[i] - i += 1 - k += 1 - - while j < len(right_half): - arr[k] = right_half[j] - j += 1 - k += 1 + # Base case: if the array has 1 or 0 elements, it's already sorted + if len(arr) <= 1: + return arr + + # Split the array into two halves + mid = len(arr) // 2 + left_half = arr[:mid] + right_half = arr[mid:] + + # Recursively sort both halves + left_sorted = merge_sort(left_half) + right_sorted = merge_sort(right_half) + + # Merge the sorted halves + return merge(left_sorted, right_sorted) + +def merge(left, right): + merged = [] + left_index = 0 + right_index = 0 + + # Merge the two halves, keeping them sorted + while left_index < len(left) and right_index < len(right): + if left[left_index] < right[right_index]: + merged.append(left[left_index]) + left_index += 1 + else: + merged.append(right[right_index]) + right_index += 1 + + # Add any remaining elements from the left and right halves + merged.extend(left[left_index:]) + merged.extend(right[right_index:]) + + return merged ``` +This function first checks if the input array has 1 or 0 elements, in which case it's already sorted. If not, it splits the array into two halves and recursively sorts each half using the merge sort algorithm. Finally, it merges the two sorted halves back together. diff --git a/tests/results/examples/hello/hello.result b/tests/results/examples/hello/hello.result index 8ab686eaf..5d752d231 100644 --- a/tests/results/examples/hello/hello.result +++ b/tests/results/examples/hello/hello.result @@ -1 +1,2 @@ -Hello, World! +Hello +Hello! How can I assist you today? diff --git a/tests/results/examples/input/input_test1.result b/tests/results/examples/input/input_test1.result index 8c7d795e5..18d0688d7 100644 --- a/tests/results/examples/input/input_test1.result +++ b/tests/results/examples/input/input_test1.result @@ -1,2 +1,2 @@ The following will prompt the user on stdin. -HelloHello +Hello diff --git a/tests/results/examples/react/demo.result b/tests/results/examples/react/demo.result new file mode 100644 index 000000000..6e2e92a08 --- /dev/null +++ b/tests/results/examples/react/demo.result @@ -0,0 +1,41 @@ +What is the elevation range for the area that the eastern sector of the Colorado orogeny extends into? +Tho: I need to search Colorado orogeny, find the area that the eastern sector of the Colorado ... +Act: {"name": "Search", "arguments": {"topic": "Colorado orogeny"}} +Obs: The Colorado orogeny was an episode of mountain building (an orogeny) ... +Tho: It does not mention the eastern sector. So I need to look up eastern sector. +Tho: High Plains rise in elevation from around 1,800 to 7,000 ft, so the answer is 1,800 to 7,000 ft. +Act: {"name": "Finish", "arguments": {"topic": "1,800 to 7,000 ft"}} + +What profession does Nicholas Ray and Elia Kazan have in common? +Tho: I need to search Nicholas Ray and Elia Kazan, find their professions, then find the profession they have in common. +Act: {"name": "Search", "arguments": {"topic": "Nicholas Ray"}} +Obs: Nicholas Ray (born Raymond Nicholas Kienzle Jr., August 7, 1911 - June 16, 1979) was an American film director, screenwriter, and actor best known for the 1955 film Rebel Without a Cause. +Tho: Professions of Nicholas Ray are director, screenwriter, and actor. I need to search Elia Kazan next and find his professions. +Act: {"name": "Search", "arguments": {"topic": "Elia Kazan"}} +Obs: Elia Kazan was an American film and theatre director, producer, screenwriter and actor. +Tho: Professions of Elia Kazan are director, producer, screenwriter, and actor. So profession Nicholas Ray and Elia Kazan have in common is director, screenwriter, and actor. +Act: {"name": "Finish", "arguments": {"topic": "director, screenwriter, actor"}} + +What is 18 + 12 x 3? +Tho: I need to call a calculator. +Act: {"name": "Calc", "arguments": {"expr": "18 + 12 * 3"}} +Obs: 54 +Act: {"name": "Finish", "arguments": {"topic": "54"}} + +A total of 252 qualifying matches were played, and 723 goals were scored. What was the average number of goals per match? +Tho: I need to call a calculator. +Act: {"name": "Calc", "arguments": {"expr": "723 / 252"}} +Obs: 2.869047619047619 +Act: {"name": "Finish", "arguments": {"topic": "2.869047619047619"}} + + +How many years ago was the discoverer of the Hudson River born? Keep in mind we are in 2024. +Tho: I need to search Henry Hudson, find out when he was born, and then calculate how many years ago that was. +Act: {"name": "Search", "arguments": {"topic": "Henry Hudson"}} +Obs: Henry Hudson (c. 1565 – disappeared 23 June 1611) was an English sea explorer and navigator during the early 17th century, best known for his explorations of present-day Canada and parts of the Northeastern United States. +In 1607 and 1608, Hudson made two attempts on behalf of English merchants to find a rumoured Northeast Passage to Cathay via a route above the Arctic Circle. In 1609, he landed in North America on behalf of the Dutch East India Company and explored the region around the modern New York metropolitan area. Looking for a Northwest Passage to Asia on his ship Halve Maen ("Half Moon"), he sailed up the Hudson River, which was later named after him, and thereby laid the foundation for Dutch colonization of the region. His contributions to the exploration of the New World were significant and lasting. His voyages helped to establish European contact with the native peoples of North America and contributed to the development of trade and commerce. +On his final expedition, while still searching for the Northwest Passage, Hudson became the first European to see Hudson Strait and the immense Hudson Bay. In 1611, after wintering on the shore of James Bay, Hudson wanted to press on to the west, but most of his crew mutinied. The mutineers cast Hudson, his son, and six others adrift; the Hudsons and their companions were never seen again. +Tho: Henry Hudson was born around 1565. To find out how many years ago that was, I need to subtract his birth year from the current year, which is 2024. +Act: {"name": "Calc", "arguments": {"expr": "2024 - 1565"}} +Obs: 459 + diff --git a/tests/results/examples/react/wikipedia.result b/tests/results/examples/react/wikipedia.result index 84d9e0f95..4620b2aa6 100644 --- a/tests/results/examples/react/wikipedia.result +++ b/tests/results/examples/react/wikipedia.result @@ -16,15 +16,36 @@ Obs: Elia Kazan was an American film and theatre director, producer, screenwrite Tho: Professions of Elia Kazan are director, producer, screenwriter, and actor. So profession Nicholas Ray and Elia Kazan have in common is director, screenwriter, and actor. Act: {"name": "Finish", "arguments": {"topic": "director, screenwriter, actor"}} -When was the discoverer of the Hudson River born? -Tho: I need to search the discoverer of the Hudson River, find when he was born. -Act:{"name": "Search", "arguments": {"topic": "discoverer of the Hudson River"}}Obs: The Hudson River is a 315-mile (507 km) river that flows from north to south primarily through eastern New York, United States. It originates in the Adirondack Mountains of upstate New York at Henderson Lake in the town of Newcomb, and flows southward through the Hudson Valley to the New York Harbor between New York City and Jersey City, eventually draining into the Atlantic Ocean at Upper New York Bay. The river serves as a physical boundary between the states of New Jersey and New York at its southern end. Farther north, it marks local boundaries between several New York counties. The lower half of the river is a tidal estuary, deeper than the body of water into which it flows, occupying the Hudson Fjord, an inlet that formed during the most recent period of North American glaciation, estimated at 26,000 to 13,300 years ago. Even as far north as the city of Troy, the flow of the river changes direction with the tides. -The Hudson River runs through the Munsee, Lenape, Mohican, Mohawk, and Haudenosaunee homelands. Prior to European exploration, the river was known as the Mahicannittuk by the Mohicans, Ka'nón:no by the Mohawks, and Muhheakantuck by the Lenape. The river was subsequently named after Henry Hudson, an Englishman sailing for the Dutch East India Company who explored it in 1609, and after whom Hudson Bay in Canada is also named. It had previously been observed by Italian explorer Giovanni da Verrazzano sailing for King Francis I of France in 1524, as he became the first European known to have entered the Upper New York Bay, but he considered the river to be an estuary. The Dutch called the river the North River, and they called the present-day Delaware River the South River, which formed the spine of the Dutch colony of New Netherland. Settlements of the colony clustered around the Hudson, and its strategic importance as the gateway to the American interior led to years of competition between the English and the Dutch over control of the river and colony. -During the 18th century, the river valley and its inhabitants were the subject and inspiration of Washington Irving, the first internationally acclaimed American author. In the nineteenth century, the area inspired the Hudson River School of landscape painting, an American pastoral style, as well as the concepts of environmentalism and wilderness. The Hudson River was also the eastern outlet for the Erie Canal, which, when completed in 1825, became an important transportation artery for the early 19th century United States. -Pollution in the Hudson River increased in the 20th century, more acutely by mid-century, particularly with industrial contamination from polychlorinated biphenyls, also known by their acronym PCBs. Pollution control regulations, enforcement actions and restoration projects initiated in the latter 20th century have begun to improve water quality, and restoration work has continued in the 21st century. -Tho: The discoverer of the Hudson River is Henry Hudson. I need to search Henry Hudson, find when he was born. -Act:{"name": "Search", "arguments": {"topic": "Henry Hudson"}}Obs: Henry Hudson (c. 1565 – disappeared 23 June 1611) was an English sea explorer and navigator during the early 17th century, best known for his explorations of present-day Canada and parts of the Northeastern United States. +What is 18 + 12 x 3? +Tho: I need to call a calculator. +Act: {"name": "Calc", "arguments": {"expr": "18 + 12 * 3"}} +Obs: 54 +Act: {"name": "Finish", "arguments": {"topic": "54"}} + +A total of 252 qualifying matches were played, and 723 goals were scored. What was the average number of goals per match? +Tho: I need to call a calculator. +Act: {"name": "Calc", "arguments": {"expr": "723 / 252"}} +Obs: 2.869047619047619 +Act: {"name": "Finish", "arguments": {"topic": "2.869047619047619"}} + +What is 18 + 12 x 3? +Tho: I need to call a calculator. +Act: {"name": "Calc", "arguments": {"expr": "18 + 12 * 3"}} +Obs: 54 +Act: {"name": "Finish", "arguments": {"topic": "54"}} + +A total of 252 qualifying matches were played, and 723 goals were scored. What was the average number of goals per match? +Tho: I need to call a calculator. +Act: {"name": "Calc", "arguments": {"expr": "723 / 252"}} +Obs: 2.869047619047619 +Act: {"name": "Finish", "arguments": {"topic": "2.869047619047619"}} + + +when was the discoverer of the Hudson River born? +Tho: I need to search Henry Hudson, find out when he was born. +Act: {"name": "Search", "arguments": {"topic": "Henry Hudson"}} +Obs: Henry Hudson (c. 1565 – disappeared 23 June 1611) was an English sea explorer and navigator during the early 17th century, best known for his explorations of present-day Canada and parts of the Northeastern United States. In 1607 and 1608, Hudson made two attempts on behalf of English merchants to find a rumoured Northeast Passage to Cathay via a route above the Arctic Circle. In 1609, he landed in North America on behalf of the Dutch East India Company and explored the region around the modern New York metropolitan area. Looking for a Northwest Passage to Asia on his ship Halve Maen ("Half Moon"), he sailed up the Hudson River, which was later named after him, and thereby laid the foundation for Dutch colonization of the region. His contributions to the exploration of the New World were significant and lasting. His voyages helped to establish European contact with the native peoples of North America and contributed to the development of trade and commerce. On his final expedition, while still searching for the Northwest Passage, Hudson became the first European to see Hudson Strait and the immense Hudson Bay. In 1611, after wintering on the shore of James Bay, Hudson wanted to press on to the west, but most of his crew mutinied. The mutineers cast Hudson, his son, and six others adrift; the Hudsons and their companions were never seen again. -Tho: Henry Hudson was born in 1565. -Act:{"name": "Finish", "arguments": {"topic": "1565"}} +Tho: Henry Hudson was born around 1565. +Act: {"name": "Finish", "arguments": {"topic": "around 1565"}} diff --git a/tests/results/examples/sdk/hello.result b/tests/results/examples/sdk/hello.result index 8ab686eaf..af17d2f87 100644 --- a/tests/results/examples/sdk/hello.result +++ b/tests/results/examples/sdk/hello.result @@ -1 +1,2 @@ -Hello, World! +Hello +Hello diff --git a/tests/results/examples/talk/1-hello.result b/tests/results/examples/talk/1-hello.result index 8ab686eaf..af17d2f87 100644 --- a/tests/results/examples/talk/1-hello.result +++ b/tests/results/examples/talk/1-hello.result @@ -1 +1,2 @@ -Hello, World! +Hello +Hello diff --git a/tests/results/examples/talk/11-repeat.result b/tests/results/examples/talk/11-repeat.result new file mode 100644 index 000000000..e75e22558 --- /dev/null +++ b/tests/results/examples/talk/11-repeat.result @@ -0,0 +1,8 @@ +Bob's number is 1 +{"name": "Bob", "number": 1} +Carol's number is 2 +{"name": "Carol", "number": 2} +David's number is 3 +{"name": "David", "number": 3} +Ernest's number is 4 +{"name": "Ernest", "number": 4} diff --git a/tests/results/examples/talk/2-model-chaining.result b/tests/results/examples/talk/2-model-chaining.result index 511abb028..7e2cf3dc9 100644 --- a/tests/results/examples/talk/2-model-chaining.result +++ b/tests/results/examples/talk/2-model-chaining.result @@ -1,3 +1,4 @@ -Hello, World! -Translate this to French -Bonjour le monde ! +Hello +Hello +Did you just say Hello? +Yes, I did. How can I assist you today? diff --git a/tests/results/examples/talk/3-def-use.result b/tests/results/examples/talk/3-def-use.result index 65f258688..ebf1038bd 100644 --- a/tests/results/examples/talk/3-def-use.result +++ b/tests/results/examples/talk/3-def-use.result @@ -1,2 +1,3 @@ -Hello, World -The French translation of the word "World" is "Monde". This term is used to refer to the entire planet Earth, including all its countries, continents, and oceans. It can also be used metaphorically to describe a vast or extensive system or organization. +Hello +Hello +The word "Hello" translates to "Bonjour" in French. diff --git a/tests/results/examples/talk/4-function.result b/tests/results/examples/talk/4-function.result index 4d6fd2d7b..bb60b1210 100644 --- a/tests/results/examples/talk/4-function.result +++ b/tests/results/examples/talk/4-function.result @@ -1,2 +1,2 @@ -The translation of 'I love Paris!' to French is 'J'aime Paris!'. -The translation of 'I love Madrid!' to Spanish is 'Me encanta Madrid!'. +The translation of 'I love Paris!' in French is 'Je t'aime, Paris!'. +The translation of 'I love Madrid!' in Spanish is 'Me encanta Madrid!'. diff --git a/tests/results/examples/talk/5-code-eval.result b/tests/results/examples/talk/5-code-eval.result index 912ad1476..9693c956b 100644 --- a/tests/results/examples/talk/5-code-eval.result +++ b/tests/results/examples/talk/5-code-eval.result @@ -10,11 +10,15 @@ public static Map deserializeOffsetMap(String lastSourceOffset) return offsetMap; } +This Java method, `deserializeOffsetMap`, is used to deserialize a JSON string into a `Map`. Here's a breakdown of the code: -The function `deserializeOffsetMap` takes a string as input and returns a map. It first checks if the input string is null or empty. If it is, it creates a new empty map and returns it. Otherwise, it uses the Jackson library to parse the input string into a map and returns it. +1. The method takes a single parameter, `lastSourceOffset`, which is expected to be a JSON string. +2. It first checks if `lastSourceOffset` is either `null` or an empty string. If true, it initializes an empty `HashMap` and assigns it to `offsetMap`. +3. If `lastSourceOffset` is not `null` or empty, it uses the `JSON_MAPPER` object (presumably an instance of `ObjectMapper` from the Jackson library) to deserialize the JSON string into a `Map`. The `readValue` method is used for this purpose, with `Map.class` as the second argument, which specifies the expected type of the deserialized object. +4. Finally, the method returns the `offsetMap`, which now contains the deserialized data. -The `@SuppressWarnings("unchecked")` annotation is used to suppress the warning that the type of the parsed map is not checked. This is because the Jackson library is used to parse the input string into a map, but the specific type of the map is not known at compile time. Therefore, the warning is suppressed to avoid potential issues. +The `@SuppressWarnings("unchecked")` annotation is used to suppress a potential unchecked warning that might occur due to the raw `Map` type used in the method signature. EVALUATION: The similarity (Levenshtein) between this answer and the ground truth is: -0.9983818770226537 +0.30520117762512267 diff --git a/tests/results/examples/talk/6-code-json.result b/tests/results/examples/talk/6-code-json.result index 5a5502981..83232f38c 100644 --- a/tests/results/examples/talk/6-code-json.result +++ b/tests/results/examples/talk/6-code-json.result @@ -1 +1 @@ -{"input": {"source_code": "@SuppressWarnings(\"unchecked\")\npublic static Map deserializeOffsetMap(String lastSourceOffset) throws IOException {\n Map offsetMap;\n if (lastSourceOffset == null || lastSourceOffset.isEmpty()) { \n offsetMap = new HashMap<>(); \n } else {\n offsetMap = JSON_MAPPER.readValue(lastSourceOffset, Map.class); \n }\n return offsetMap;\n}\n", "repo_info": {"repo": "streamsets/datacollector", "path": "stagesupport/src/main/java/com/.../OffsetUtil.java", "function_name": "OffsetUtil.deserializeOffsetMap"}}, "output": "\nThe function `deserializeOffsetMap` takes a string as input and returns a map. It first checks if the input string is null or empty. If it is, it creates a new empty map and returns it. Otherwise, it uses the Jackson library to parse the input string into a map and returns it.\n\nThe `@SuppressWarnings(\"unchecked\")` annotation is used to suppress the warning that the type of the parsed map is not checked. This is because the Jackson library is used to parse the input string into a map, but the specific type of the map is not known at compile time. Therefore, the warning is suppressed to avoid potential issues.", "metric": 0.9983818770226537} +{"input": {"source_code": "@SuppressWarnings(\"unchecked\")\npublic static Map deserializeOffsetMap(String lastSourceOffset) throws IOException {\n Map offsetMap;\n if (lastSourceOffset == null || lastSourceOffset.isEmpty()) { \n offsetMap = new HashMap<>(); \n } else {\n offsetMap = JSON_MAPPER.readValue(lastSourceOffset, Map.class); \n }\n return offsetMap;\n}\n", "repo_info": {"repo": "streamsets/datacollector", "path": "stagesupport/src/main/java/com/.../OffsetUtil.java", "function_name": "OffsetUtil.deserializeOffsetMap"}}, "output": "This Java method, `deserializeOffsetMap`, is used to deserialize a JSON string into a `Map`. Here's a breakdown of the code:\n\n1. The method takes a single parameter, `lastSourceOffset`, which is expected to be a JSON string.\n2. It first checks if `lastSourceOffset` is either `null` or an empty string. If true, it initializes an empty `HashMap` and assigns it to `offsetMap`.\n3. If `lastSourceOffset` is not `null` or empty, it uses the `JSON_MAPPER` object (presumably an instance of `ObjectMapper` from the Jackson library) to deserialize the JSON string into a `Map`. The `readValue` method is used for this purpose, with `Map.class` as the second argument, which specifies the expected type of the deserialized object.\n4. Finally, the method returns the `offsetMap`, which now contains the deserialized data.\n\nThe `@SuppressWarnings(\"unchecked\")` annotation is used to suppress a potential unchecked warning that might occur due to the raw `Map` type used in the method signature.", "metric": 0.30520117762512267} diff --git a/tests/results/examples/talk/7-chatbot-roles.result b/tests/results/examples/talk/7-chatbot-roles.result index 1d48a7bd6..293f0823e 100644 --- a/tests/results/examples/talk/7-chatbot-roles.result +++ b/tests/results/examples/talk/7-chatbot-roles.result @@ -1,6 +1,8 @@ Type `quit` to exit this chatbot. +APR stands for Annual Percentage Rate. It is the annual interest rate charged for borrowing or earned on an investment, expressed as a single percentage number that represents the actual yearly cost of funds over the term of a loan. It includes any fees or additional costs associated with the transaction. -APR, or Annual Percentage Rate, is a fundamental concept in finance that represents the yearly cost of borrowing money, including any fees or additional charges. It's expressed as a percentage and is used to calculate the total amount of interest paid over the life of a loan. APR is crucial for comparing different loan offers and understanding the true cost of borrowing. By taking into account both the interest rate and other fees, APR provides a more comprehensive view of the loan's overall expense compared to just the interest rate alone. -Thank you for using Granite Chat. If you have any more questions or need assistance, feel free to ask. Have a great day! +I'm sorry to see you go. If you have any more questions in the future, feel free to return. Have a great day! + + diff --git a/tests/results/examples/talk/9-react.result b/tests/results/examples/talk/9-react.result index 84d9e0f95..4620b2aa6 100644 --- a/tests/results/examples/talk/9-react.result +++ b/tests/results/examples/talk/9-react.result @@ -16,15 +16,36 @@ Obs: Elia Kazan was an American film and theatre director, producer, screenwrite Tho: Professions of Elia Kazan are director, producer, screenwriter, and actor. So profession Nicholas Ray and Elia Kazan have in common is director, screenwriter, and actor. Act: {"name": "Finish", "arguments": {"topic": "director, screenwriter, actor"}} -When was the discoverer of the Hudson River born? -Tho: I need to search the discoverer of the Hudson River, find when he was born. -Act:{"name": "Search", "arguments": {"topic": "discoverer of the Hudson River"}}Obs: The Hudson River is a 315-mile (507 km) river that flows from north to south primarily through eastern New York, United States. It originates in the Adirondack Mountains of upstate New York at Henderson Lake in the town of Newcomb, and flows southward through the Hudson Valley to the New York Harbor between New York City and Jersey City, eventually draining into the Atlantic Ocean at Upper New York Bay. The river serves as a physical boundary between the states of New Jersey and New York at its southern end. Farther north, it marks local boundaries between several New York counties. The lower half of the river is a tidal estuary, deeper than the body of water into which it flows, occupying the Hudson Fjord, an inlet that formed during the most recent period of North American glaciation, estimated at 26,000 to 13,300 years ago. Even as far north as the city of Troy, the flow of the river changes direction with the tides. -The Hudson River runs through the Munsee, Lenape, Mohican, Mohawk, and Haudenosaunee homelands. Prior to European exploration, the river was known as the Mahicannittuk by the Mohicans, Ka'nón:no by the Mohawks, and Muhheakantuck by the Lenape. The river was subsequently named after Henry Hudson, an Englishman sailing for the Dutch East India Company who explored it in 1609, and after whom Hudson Bay in Canada is also named. It had previously been observed by Italian explorer Giovanni da Verrazzano sailing for King Francis I of France in 1524, as he became the first European known to have entered the Upper New York Bay, but he considered the river to be an estuary. The Dutch called the river the North River, and they called the present-day Delaware River the South River, which formed the spine of the Dutch colony of New Netherland. Settlements of the colony clustered around the Hudson, and its strategic importance as the gateway to the American interior led to years of competition between the English and the Dutch over control of the river and colony. -During the 18th century, the river valley and its inhabitants were the subject and inspiration of Washington Irving, the first internationally acclaimed American author. In the nineteenth century, the area inspired the Hudson River School of landscape painting, an American pastoral style, as well as the concepts of environmentalism and wilderness. The Hudson River was also the eastern outlet for the Erie Canal, which, when completed in 1825, became an important transportation artery for the early 19th century United States. -Pollution in the Hudson River increased in the 20th century, more acutely by mid-century, particularly with industrial contamination from polychlorinated biphenyls, also known by their acronym PCBs. Pollution control regulations, enforcement actions and restoration projects initiated in the latter 20th century have begun to improve water quality, and restoration work has continued in the 21st century. -Tho: The discoverer of the Hudson River is Henry Hudson. I need to search Henry Hudson, find when he was born. -Act:{"name": "Search", "arguments": {"topic": "Henry Hudson"}}Obs: Henry Hudson (c. 1565 – disappeared 23 June 1611) was an English sea explorer and navigator during the early 17th century, best known for his explorations of present-day Canada and parts of the Northeastern United States. +What is 18 + 12 x 3? +Tho: I need to call a calculator. +Act: {"name": "Calc", "arguments": {"expr": "18 + 12 * 3"}} +Obs: 54 +Act: {"name": "Finish", "arguments": {"topic": "54"}} + +A total of 252 qualifying matches were played, and 723 goals were scored. What was the average number of goals per match? +Tho: I need to call a calculator. +Act: {"name": "Calc", "arguments": {"expr": "723 / 252"}} +Obs: 2.869047619047619 +Act: {"name": "Finish", "arguments": {"topic": "2.869047619047619"}} + +What is 18 + 12 x 3? +Tho: I need to call a calculator. +Act: {"name": "Calc", "arguments": {"expr": "18 + 12 * 3"}} +Obs: 54 +Act: {"name": "Finish", "arguments": {"topic": "54"}} + +A total of 252 qualifying matches were played, and 723 goals were scored. What was the average number of goals per match? +Tho: I need to call a calculator. +Act: {"name": "Calc", "arguments": {"expr": "723 / 252"}} +Obs: 2.869047619047619 +Act: {"name": "Finish", "arguments": {"topic": "2.869047619047619"}} + + +when was the discoverer of the Hudson River born? +Tho: I need to search Henry Hudson, find out when he was born. +Act: {"name": "Search", "arguments": {"topic": "Henry Hudson"}} +Obs: Henry Hudson (c. 1565 – disappeared 23 June 1611) was an English sea explorer and navigator during the early 17th century, best known for his explorations of present-day Canada and parts of the Northeastern United States. In 1607 and 1608, Hudson made two attempts on behalf of English merchants to find a rumoured Northeast Passage to Cathay via a route above the Arctic Circle. In 1609, he landed in North America on behalf of the Dutch East India Company and explored the region around the modern New York metropolitan area. Looking for a Northwest Passage to Asia on his ship Halve Maen ("Half Moon"), he sailed up the Hudson River, which was later named after him, and thereby laid the foundation for Dutch colonization of the region. His contributions to the exploration of the New World were significant and lasting. His voyages helped to establish European contact with the native peoples of North America and contributed to the development of trade and commerce. On his final expedition, while still searching for the Northwest Passage, Hudson became the first European to see Hudson Strait and the immense Hudson Bay. In 1611, after wintering on the shore of James Bay, Hudson wanted to press on to the west, but most of his crew mutinied. The mutineers cast Hudson, his son, and six others adrift; the Hudsons and their companions were never seen again. -Tho: Henry Hudson was born in 1565. -Act:{"name": "Finish", "arguments": {"topic": "1565"}} +Tho: Henry Hudson was born around 1565. +Act: {"name": "Finish", "arguments": {"topic": "around 1565"}} diff --git a/tests/results/examples/tutorial/calling_llm.result b/tests/results/examples/tutorial/calling_llm.result index 8ab686eaf..af17d2f87 100644 --- a/tests/results/examples/tutorial/calling_llm.result +++ b/tests/results/examples/tutorial/calling_llm.result @@ -1 +1,2 @@ -Hello, World! +Hello +Hello diff --git a/tests/results/examples/tutorial/calling_llm_with_input.result b/tests/results/examples/tutorial/calling_llm_with_input.result index 290e2f6e4..815854692 100644 --- a/tests/results/examples/tutorial/calling_llm_with_input.result +++ b/tests/results/examples/tutorial/calling_llm_with_input.result @@ -1,2 +1,2 @@ -Hello, : -Le mot 'world' en français est 'monde'. +Hello +The word 'Hello' translates to 'Bonjour' in French. diff --git a/tests/results/examples/tutorial/calling_llm_with_input_messages.result b/tests/results/examples/tutorial/calling_llm_with_input_messages.result new file mode 100644 index 000000000..815854692 --- /dev/null +++ b/tests/results/examples/tutorial/calling_llm_with_input_messages.result @@ -0,0 +1,2 @@ +Hello +The word 'Hello' translates to 'Bonjour' in French. diff --git a/tests/results/examples/tutorial/conditionals_loops.result b/tests/results/examples/tutorial/conditionals_loops.result index b3c16e7d3..a6b1fdf6b 100644 --- a/tests/results/examples/tutorial/conditionals_loops.result +++ b/tests/results/examples/tutorial/conditionals_loops.result @@ -1,17 +1,4 @@ - -Annual Percentage Rate (APR) is the total cost of borrowing money, expressed as a percentage. It includes the interest rate and any additional fees or charges associated with the loan. APR is used to compare different loan offers and to understand the true cost of borrowing money over time. -Say it as a poem -In the realm of loans, a crucial term you'll find, -A figure that's both intriguing and profound, -Annual Percentage Rate, or APR, it's called, -A beacon that shines, to help you choose your ground. - -It's not just the interest rate that it entails, -But other costs, both big and small, combined, -APR, like a lighthouse, guides you through the fray, -Illuminating the path to a wiser day. - -So when shopping for loans, don't shy away, -Ask for APR, and compare it all, -For in the end, it's this number you'll see, -That truly reveals the cost of being free. +APR stands for Annual Percentage Rate. It is the annual interest rate charged for borrowing or earned through an investment, and it represents the actual yearly cost of funds over the term of a loan. It includes any fees or additional costs associated with the transaction.Say it as a poemAPR, a rate so grand, +Annual Percentage Rate, across the land. +It's the cost of borrowing, or earning, you see, +Including fees, for a year's decree. diff --git a/tests/results/examples/tutorial/data_block.result b/tests/results/examples/tutorial/data_block.result index 68d247afd..5eca40198 100644 --- a/tests/results/examples/tutorial/data_block.result +++ b/tests/results/examples/tutorial/data_block.result @@ -1 +1 @@ -{'input': {'source_code': '@SuppressWarnings("unchecked")\npublic static Map deserializeOffsetMap(String lastSourceOffset) throws IOException {\n Map offsetMap;\n if (lastSourceOffset == null || lastSourceOffset.isEmpty()) { \n offsetMap = new HashMap<>(); \n } else {\n offsetMap = JSON_MAPPER.readValue(lastSourceOffset, Map.class); \n }\n return offsetMap;\n}\n', 'repo_info': {'repo': 'streamsets/datacollector', 'path': 'stagesupport/src/main/java/com/.../OffsetUtil.java', 'function_name': 'OffsetUtil.deserializeOffsetMap'}}, 'output': '\nThe function `deserializeOffsetMap` takes a string as input and returns a map. It first checks if the input string is null or empty. If it is, it creates a new empty map and returns it. Otherwise, it uses the Jackson library to parse the input string into a map and returns it.\n\nThe `@SuppressWarnings("unchecked")` annotation is used to suppress the warning that the type of the parsed map is not checked. This is because the Jackson library is used to parse the input string into a map, but the specific type of the map is not known at compile time. Therefore, the warning is suppressed to avoid potential issues.', 'metric': 0.9967637540453075} +{'input': {'source_code': '@SuppressWarnings("unchecked")\npublic static Map deserializeOffsetMap(String lastSourceOffset) throws IOException {\n Map offsetMap;\n if (lastSourceOffset == null || lastSourceOffset.isEmpty()) { \n offsetMap = new HashMap<>(); \n } else {\n offsetMap = JSON_MAPPER.readValue(lastSourceOffset, Map.class); \n }\n return offsetMap;\n}\n', 'repo_info': {'repo': 'streamsets/datacollector', 'path': 'stagesupport/src/main/java/com/.../OffsetUtil.java', 'function_name': 'OffsetUtil.deserializeOffsetMap'}}, 'output': 'This Java method, `deserializeOffsetMap`, is used to deserialize a JSON string into a `Map`. Here\'s a breakdown of the code:\n\n1. The method takes a single parameter, `lastSourceOffset`, which is expected to be a JSON string.\n2. It first checks if `lastSourceOffset` is either `null` or an empty string. If true, it initializes an empty `HashMap` and assigns it to `offsetMap`.\n3. If `lastSourceOffset` is not `null` or empty, it uses the `JSON_MAPPER` object (presumably an instance of `ObjectMapper` from the Jackson library) to deserialize the JSON string into a `Map`. The `readValue` method is used for this purpose, with `Map.class` as the second argument, which specifies the expected type of the deserialized object.\n4. Finally, the method returns the `offsetMap`, which now contains the deserialized data.\n\nThe `@SuppressWarnings("unchecked")` annotation is used to suppress a potential unchecked warning that might occur due to the raw `Map` type used in the method signature.', 'metric': 0.30421982335623154} diff --git a/tests/results/examples/tutorial/function_definition.result b/tests/results/examples/tutorial/function_definition.result index 4d6fd2d7b..bb60b1210 100644 --- a/tests/results/examples/tutorial/function_definition.result +++ b/tests/results/examples/tutorial/function_definition.result @@ -1,2 +1,2 @@ -The translation of 'I love Paris!' to French is 'J'aime Paris!'. -The translation of 'I love Madrid!' to Spanish is 'Me encanta Madrid!'. +The translation of 'I love Paris!' in French is 'Je t'aime, Paris!'. +The translation of 'I love Madrid!' in Spanish is 'Me encanta Madrid!'. diff --git a/tests/results/examples/tutorial/grouping_definitions.result b/tests/results/examples/tutorial/grouping_definitions.result index 4d6fd2d7b..bb60b1210 100644 --- a/tests/results/examples/tutorial/grouping_definitions.result +++ b/tests/results/examples/tutorial/grouping_definitions.result @@ -1,2 +1,2 @@ -The translation of 'I love Paris!' to French is 'J'aime Paris!'. -The translation of 'I love Madrid!' to Spanish is 'Me encanta Madrid!'. +The translation of 'I love Paris!' in French is 'Je t'aime, Paris!'. +The translation of 'I love Madrid!' in Spanish is 'Me encanta Madrid!'. diff --git a/tests/results/examples/tutorial/include.result b/tests/results/examples/tutorial/include.result index 7551f0f37..068d20d0d 100644 --- a/tests/results/examples/tutorial/include.result +++ b/tests/results/examples/tutorial/include.result @@ -1,32 +1,15 @@ -You are Granite Chat, an AI language model developed by IBM. You are a cautious assistant. You carefully follow instructions. You are helpful and harmless and you follow ethical guidelines and promote positive behavior. You always respond to greetings (for example, hi, hello, g'day, morning, afternoon, evening, night, what's up, nice to meet you, sup, etc) with "Hello! I am Granite Chat, created by IBM. How can I help you today?". Please do not say anything else and do not start a conversation. -What is APR? - -Annual Percentage Rate (APR) is a measure of the cost of borrowing money, expressed as a percentage rate. It takes into account both the interest rate and other fees associated with the loan. APR is typically higher than the interest rate because it includes the cost of insurance, points, and other charges imposed by the lender. By providing a single, standardized measure of the cost of a loan, APR facilitates comparisons between different loans and lenders. It's important to note that APR is not the same as simple interest, which is only the cost of borrowing the principal amount without any additional fees or charges.Can you write a poem about APR? - -In the realm of finance, there exists a term, -A percentage that signifies cost. -Annual Percentage Rate, or APR, you may inquire, -A comprehensive measure, complete and sincere. - -It's more than just the interest rate, mind you, -For it includes other costs, quite a bunch. -Insurance, points, and other fees they pile, -APR, in its glory, reveals the ill. - -A standardized figure, universal and bright, -Facilitating comparisons in the financial market light. -Though not the same as simple interest, which is but -The principal's price, without the extras' thrust. -So, when you borrow, look beyond the rate, -To APR's comprehensive view, be wise and sage. -For in the world of finance, transparency is key, -And understanding APR, is where it starts.Now explain APR to me like I'm 5 years old - -Sure, I'd be happy to explain APR in a simple way! - -Imagine you want to borrow some money from your friend, and they charge you a certain amount of money to lend you the money. This extra amount is like a gift they're giving you for letting them use their stuff. The Annual Percentage Rate (APR) is a way to tell you how much extra you have to pay back to your friend, including the gift and all the other little things that come with borrowing money. - -So, if your friend charges 3 apples as a gift, and you have to eat 5 apples (because you borrowed the money for a longer time than usual), then the APR is 5 apples divided by the number of apples you had to eat (which is the total cost of borrowing). In this case, the APR would be 1 apple. - -This way, you can easily understand how much more you'll be paying back to your friend compared to just the interest rate. APR helps everyone compare prices and make better decisions when borrowing money. +What is APR? +APR stands for Annual Percentage Rate. It is the annual interest rate charged for borrowing or earned through an investment, and it represents the actual yearly cost of funds over the term of a loan. It includes any fees or additional costs associated with the transaction. +Can you write a poem about APR? +In the world of finance, APR is the key, +A number that tells us what we'll pay, or earn, you see. +It's the annual rate, both simple and clear, +Including all fees, for loans and investments, near and far. + +It's the cost of borrowing, or the return we gain, +A figure that helps us make informed financial gain. +So, when you're comparing loans, or investments to choose, +Remember APR, and make the right move. +Now explain APR to me like I'm 5 years old +Sure! Imagine you have a piggy bank, and you want to borrow some money from your parents to buy a toy. Your parents will ask you to pay them back with some extra money, which is like interest. APR is the special number that tells you how much extra money you'll have to pay back, all in one year. It's like a special rule that helps you understand how much you'll owe your parents. diff --git a/tests/results/examples/tutorial/model_chaining.result b/tests/results/examples/tutorial/model_chaining.result index 511abb028..7e2cf3dc9 100644 --- a/tests/results/examples/tutorial/model_chaining.result +++ b/tests/results/examples/tutorial/model_chaining.result @@ -1,3 +1,4 @@ -Hello, World! -Translate this to French -Bonjour le monde ! +Hello +Hello +Did you just say Hello? +Yes, I did. How can I assist you today? diff --git a/tests/results/examples/tutorial/muting_block_output.result b/tests/results/examples/tutorial/muting_block_output.result index 5b39e815a..3a32b5abe 100644 --- a/tests/results/examples/tutorial/muting_block_output.result +++ b/tests/results/examples/tutorial/muting_block_output.result @@ -1 +1 @@ -The french sentence was: The translation of 'I love Paris!' to French is 'J'aime Paris!'. +The french sentence was: The translation of 'I love Paris!' in French is 'Je t'aime, Paris!'. diff --git a/tests/results/examples/tutorial/variable_def_use.result b/tests/results/examples/tutorial/variable_def_use.result index 61ee55a9d..c0e3cad60 100644 --- a/tests/results/examples/tutorial/variable_def_use.result +++ b/tests/results/examples/tutorial/variable_def_use.result @@ -1,2 +1,3 @@ -Hello, World! -GEN is equal to: World! +Hello +Hello +GEN is equal to: Hello diff --git a/tests/results/examples/weather/weather.result b/tests/results/examples/weather/weather.result index dba3ea432..ba7bbb4fe 100644 --- a/tests/results/examples/weather/weather.result +++ b/tests/results/examples/weather/weather.result @@ -1,2 +1 @@ - -The temperature is 15.6 degrees Celsius and the sky is partly cloudy. +The current weather in Yorktown Heights, New York, USA is sunny with a temperature of 13.3°C (55.9°F). The wind is blowing at a speed of 7.6 mph (12.2 kph) from the north (N) direction. The pressure is 1031.0 mb (30.43 inches). The humidity is 42% and the dew point is 3.5°C (38.3°F). The visibility is 16.0 km (9.0 miles). The UV index is 1.3 and the wind gusts are up to 9.3 mph (14.9 kph). diff --git a/tests/test_examples_run.py b/tests/test_examples_run.py index 7a9050d32..ff6569add 100644 --- a/tests/test_examples_run.py +++ b/tests/test_examples_run.py @@ -18,7 +18,7 @@ for name in [ pathlib.Path("examples") / "demo" / "2-teacher.pdl", # TODO: check why pathlib.Path("examples") / "talk" / "8-tools.pdl", # TODO: check why - pathlib.Path("examples") / "talk" / "11-sdg.pdl", # TODO: check why + pathlib.Path("examples") / "talk" / "10-sdg.pdl", # TODO: check why pathlib.Path("examples") / "teacher" / "teacher.pdl", # TODO: check why pathlib.Path("examples") / "tools" / "calc.pdl", # TODO: check why pathlib.Path("examples") / "tutorial" / "calling_apis.pdl", @@ -43,8 +43,9 @@ pathlib.Path("examples") / "joke" / "Joke.pdl", pathlib.Path("examples") / "react" / "multi-agent.pdl", pathlib.Path("examples") / "react" / "wikipedia.pdl", - pathlib.Path("examples") / "talk" / "11-sdg.pdl", + pathlib.Path("examples") / "talk" / "10-sdg.pdl", pathlib.Path("examples") / "talk" / "7-chatbot-roles.pdl", + pathlib.Path("examples") / "chatbot" / "chatbot.pdl", pathlib.Path("examples") / "talk" / "8-tools.pdl", pathlib.Path("examples") / "talk" / "9-react.pdl", pathlib.Path("examples") / "teacher" / "teacher.pdl", @@ -55,6 +56,7 @@ pathlib.Path("examples") / "demo" / "3-weather.pdl", pathlib.Path("examples") / "tutorial" / "conditionals_loops.pdl", pathlib.Path("examples") / "chatbot" / "chatbot.pdl", + pathlib.Path("examples") / "fibonacci" / "fib.pdl", ] } @@ -129,6 +131,7 @@ class InputsType: pathlib.Path("examples") / "hello" / "hello-type-code.pdl", pathlib.Path("examples") / "hello" / "hello-type-list.pdl", pathlib.Path("examples") / "hello" / "hello-type.pdl", + pathlib.Path("examples") / "hello" / "hello-parser-json.pdl", pathlib.Path("tests") / "data" / "line" / "hello12.pdl", pathlib.Path("tests") / "data" / "line" / "hello13.pdl", pathlib.Path("tests") / "data" / "line" / "hello14.pdl",