Cleanup of hello programs and changes to README (#18)

* update to hello examples and readme

Signed-off-by: Mandana Vaziri <[email protected]>

* cleanup

Signed-off-by: Mandana Vaziri <[email protected]>

---------

Signed-off-by: Mandana Vaziri <[email protected]>

docs/tutorial.md

@@ -21,7 +21,7 @@ This program has a `description` field, which contains a title. It also has a `d
 To render the program into an actual document, we have a PDL interpreter that can be invoked as follows:
 
 ```
-python3 -m pdl.pdl ./examples/hello/hello.pdl
+python3 -m pdl.pdl ./examples/tutorial/simple_program.pdl
 ```
 
 This results in the following output:

examples/hello/chatbot.pdl → examples/chatbot/chatbot.pdl

@@ -5,7 +5,7 @@ document:
 - repeat:
   - model: ibm/granite-13b-chat-v2
     spec: str
-    parameters:
+    params:
       stop_sequences: ["."]
   - read:
     def: eval

examples/code/code-defs.pdl

@@ -1,11 +1,11 @@
 description: Code explanation example
 defs: 
    CODE:
-      read: examples/code/data.json
-      parser: json
+      read: examples/code/data.yaml
+      parser: yaml
       def: CODE
    EXPLANATION:
-      model: ibm/granite-20b-code-instruct
+      model: ibm/granite-34b-code-instruct
       input:
            |
             Here is some info about the location of the function in the repo.
@@ -22,4 +22,4 @@ document:
 - |
 
   {{ CODE.source_code }}
-  {{ EXPLANATION }}
+  {{ EXPLANATION }}

examples/code/code-eval.pdl

@@ -1,11 +1,11 @@
 description: Code explanation example
 document:
-- read: examples/code/data.json
-  parser: json
+- read: examples/code/data.yaml
+  parser: yaml
   def: CODE
   show_result: False
 - "\n{{ CODE.source_code }}\n"
-- model: ibm/granite-20b-code-instruct
+- model: ibm/granite-34b-code-instruct
   def: EXPLANATION
   input: |
       Here is some info about the location of the function in the repo.

examples/code/code-json.pdl

@@ -1,13 +1,13 @@
 description: Code explanation example
 document:
-- read: examples/code/data.json
-  parser: json
+- read: examples/code/data.yaml
+  parser: yaml
   def: CODE
   show_result: False
 - read: examples/code/ground_truth.txt
   def: TRUTH
   show_result: False
-- model: ibm/granite-20b-code-instruct
+- model: ibm/granite-34b-code-instruct
   def: EXPLANATION
   show_result: False
   input:
@@ -39,5 +39,4 @@ document:
     input: "{{ CODE }}"
     output: "{{ EXPLANATION }}"
     metric: "{{ EVAL }}"
-  def: MINE
 - "\n\n"

examples/code/code.pdl

@@ -1,16 +1,15 @@
 description: Code explanation example
 document:
-- read: examples/code/data.json
-  parser: json
+- read: examples/code/data.yaml
+  parser: yaml
   def: CODE
   show_result: False
 - "\n{{ CODE.source_code }}\n"
-- model: ibm/granite-20b-code-instruct
+- model: ibm/granite-34b-code-instruct
   input:
       - |
         Here is some info about the location of the function in the repo.
-        repo: 
-        {{ CODE.repo_info.repo }}
+        repo: {{ CODE.repo_info.repo }}
         path: {{ CODE.repo_info.path }}
         Function_name: {{ CODE.repo_info.function_name }}
 

examples/code/data.yaml

@@ -0,0 +1,16 @@
+source_code: 
+  |
+  @SuppressWarnings("unchecked")
+  public static Map<String, String> deserializeOffsetMap(String lastSourceOffset) throws IOException {
+    Map<String, String> offsetMap;
+    if (lastSourceOffset == null || lastSourceOffset.isEmpty()) {    
+      offsetMap = new HashMap<>();  
+    } else {
+      offsetMap = JSON_MAPPER.readValue(lastSourceOffset, Map.class);  
+    }
+    return offsetMap;
+  }
+repo_info: 
+  repo: streamsets/datacollector
+  path: stagesupport/src/main/java/com/.../OffsetUtil.java
+  function_name: OffsetUtil.deserializeOffsetMap

examples/code/ground_truth.txt

@@ -1 +1,3 @@
-This code is used to deserialize a JSON string into a map. The function takes in a string `lastSourceOffset` and returns a map. If the `lastSourceOffset` is null or empty, it creates a new empty map. Otherwise, it uses the `JSON_MAPPER` to deserialize the `lastSourceOffset` into a map. The `@SuppressWarnings("unchecked")` annotation is used to suppress warnings about unchecked type conversions.
+The function `deserializeOffsetMap` takes a string as input and returns a map. It first checks if the input string is null or empty. If it is, it creates a new empty map and returns it. Otherwise, it uses the Jackson library to parse the input string into a map and returns it.
+
+The `@SuppressWarnings("unchecked")` annotation is used to suppress the warning that the type of the parsed map is not checked. This is because the Jackson library is used to parse the input string into a map, but the specific type of the map is not known at compile time. Therefore, the warning is suppressed to avoid potential issues.

examples/demo/3-weather.pdl

@@ -28,7 +28,7 @@ document:
   show_result: false
 - model: ibm/granite-20b-code-instruct
   input: |
-      Explain the weather from the following JSON. Don't write quotes.
-      `{{ WEATHER }}`
+      Explain the weather from the following JSON.
+      `{{ WEATHER }}` 
 - "\n"
 

examples/hello/hello6.pdl → examples/hello/hello-code.pdl

@@ -7,3 +7,4 @@ document:
     import string
     result = random.choice(string.ascii_lowercase)
 - '!'
+- "\n"

examples/hello/hello19.pdl → examples/hello/hello-data.pdl

@@ -1,10 +1,12 @@
-description: Hello world
+# Call with pdl --data '"something": "ABC"'
+description: Hello world with data
 document:
 - def: stutter
   function: 
   return:
     - "{{ something }}"
 - "Hello World!\n"
 - call: stutter
+- "\n"
 
 

examples/hello/hello4.pdl → examples/hello/hello-def-use.pdl

@@ -1,13 +1,14 @@
 description: Hello world with variable use
 document:
 - Hello,
-- model: ibm/granite-20b-code-instruct-v2
-  parameters:
+- model: ibm/granite-34b-code-instruct
+  params:
     decoding_method: greedy
     stop_sequences:
     - '!'
-    include_stop_sequence: true
+    include_stop_sequence: false
   def: NAME
 - |
 
   Who is{{ NAME }}?
+- "\n"

examples/hello/hello11.pdl → examples/hello/hello-defs.pdl

@@ -1,10 +1,11 @@
-description: Hello function
+description: Hello world with defs
 defs:
   hello:
     function:
       name: str
     return: Hello {{ name }}!
 document:
-  - call: hello
-    args:
-      name: World
+- call: hello
+  args:
+    name: World
+- "\n"

examples/hello/hello16.pdl → examples/hello/hello-for-loop.pdl

@@ -1,11 +1,11 @@
-description: for loop
+description: Hello world with for loop
 defs:
-  numbers: {
+  numbers: 
     data: [1, 2, 3, 4]
-  }
-  names: {
+
+  names: 
     data: ["Bob", "Carol", "David", "Ernest"]
-  }
+
 
 document:
 - for:

examples/hello/hello26.pdl → examples/hello/hello-function-alias.pdl

@@ -6,6 +6,7 @@ defs:
     return: Hello {{ name }}!
   alias: hello
 document:
-  - call: "{{ alias }}"
-    args:
-      name: World
+- call: "{{ alias }}"
+  args:
+    name: World
+- "\n"

examples/hello/hello-function.pdl

@@ -0,0 +1,10 @@
+description: Hello world with function definition and call
+document:
+- def: hello
+  function:
+    name: str
+  return: Hello {{ name }}!
+- call: hello
+  args:
+    name: World
+- "\n"

examples/hello/hello7.pdl → examples/hello/hello-if.pdl

@@ -1,4 +1,4 @@
-description: Hello world showing call out to python code with condition
+description: Hello world with condition
 document:
 - 'Hello! '
 - lan: python
@@ -9,4 +9,4 @@ document:
   show_result: false
 - if: '{{ QUESTION }}'
   then: How are you?
-  
+- "\n"

examples/hello/hello-iteration.pdl

@@ -0,0 +1,8 @@
+description: Hello world with iteration
+document:
+- Hello, world!
+- "\n"
+- repeat:
+    "This is your first PDL program\n"
+  num_iterations: 3
+- "\n"

examples/hello/hello-litellm.pdl

@@ -0,0 +1,8 @@
+description: Hello world with LiteLLM
+document:
+- Hello,
+- model: watsonx/ibm/granite-34b-code-instruct
+  parameters:
+    stop:
+    - ","
+- "\n"

examples/hello/hello5.pdl → examples/hello/hello-model-chaining.pdl

@@ -1,8 +1,8 @@
 description: Hello world showing model chaining
 document:
 - Hello,
-- model: ibm/granite-20b-code-instruct-v2
-  parameters:
+- model: ibm/granite-34b-code-instruct
+  params:
     decoding_method: greedy
     stop_sequences:
     - '!'
@@ -11,9 +11,11 @@ document:
 - |-
   !
   Who is{{ NAME }}?
+- "\n"
 - model: google/flan-t5-xl
-  parameters:
+  params:
     decoding_method: greedy
     stop_sequences:
     - '!'
     include_stop_sequence: false
+- "\n"

examples/hello/hello-model-input.pdl

@@ -0,0 +1,9 @@
+description: Hello world with model input
+document:
+- model: ibm/granite-34b-code-instruct
+  input: "Hello,"
+  params:
+    stop_sequences:
+    - '!'
+    include_stop_sequence: true
+- "\n"

examples/hello/hello-parser-json.pdl

@@ -0,0 +1,33 @@
+description: Creating JSON Data
+defs:
+  data:
+    read: examples/hello/hello21_data.yaml
+    parser: yaml
+    spec: { "questions": ["str"], "answers": ["obj"] }
+document:
+  - model: ibm/granite-34b-code-instruct
+    def: model_output
+    spec: {"name": str, "age": int}
+    input: 
+      document:
+      - for: 
+          question: "{{ data.questions }}"
+          answer: "{{ data.answers }}"
+        repeat:
+          - |
+            {{ question }}
+            {{ answer }}
+        as: document
+      - > 
+        Question: Create a JSON object with fields 'name' and 'age' 
+        and set them appropriately.
+    parser: yaml
+    params:
+      decoding_method: greedy
+      stop_sequences:
+      - '}'
+      include_stop_sequence: true
+  - "\n"
+
+
+

examples/hello/hello-parser-regex.pdl

@@ -0,0 +1,16 @@
+description: Hello world with parser using regex
+document:
+- model: ibm/granite-34b-code-instruct
+  input: "Hello,"
+  params:
+    decoding_method: greedy
+    stop_sequences:
+    - '!'
+    include_stop_sequence: false
+  spec: {"name": str}
+  parser:
+    spec:
+      name: str
+    regex: '\s*(?P<name>.*)\s*'
+- "\n"
+

examples/hello/hello27.pdl → examples/hello/hello-roles-array.pdl

@@ -1,14 +1,13 @@
 description: Model call with explicit messages input
-sequence:
+document:
 - def: prompt
   array:
   - role: system
     content: You are a helpful software engineer. You write clear, concise, well-commented code.
   - role: user
     content: Write a Python function that implement merge sort.
   show_result: false
-- model: watsonx/ibm/granite-8b-code-instruct
-  platform: litellm
+- model: ibm/granite-8b-code-instruct
   input: "{{ prompt }}"
-  parameters:
+  params:
     decoding_method: greedy