[EX-127] Investigate if the tokens generated are over Ollama's context window size

mix.exs

@@ -66,7 +66,7 @@ defmodule Exmeralda.MixProject do
       {:assent, "~> 0.3.0"},
       {:req, "~> 0.5"},
       {:req_hex, "~> 0.2.1"},
-      {:langchain, "~> 0.3.0"},
+      {:langchain, "~> 0.3.3"},
       {:mdex, "~> 0.2"},
       {:oban, "~> 2.19"},
       {:oban_web, "~> 2.11"},

mix.lock

@@ -33,7 +33,7 @@
   "igniter": {:hex, :igniter, "0.5.41", "76e943e9a2a8bb73f7f4683cccf7eabee7e4ecfd501cd9e4b3b159f734eb6654", [:mix], [{:glob_ex, "~> 0.1.7", [hex: :glob_ex, repo: "hexpm", optional: false]}, {:inflex, "~> 2.0", [hex: :inflex, repo: "hexpm", optional: false]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}, {:owl, "~> 0.11", [hex: :owl, repo: "hexpm", optional: false]}, {:phx_new, "~> 1.7", [hex: :phx_new, repo: "hexpm", optional: true]}, {:req, "~> 0.5", [hex: :req, repo: "hexpm", optional: false]}, {:rewrite, ">= 1.1.1 and < 2.0.0-0", [hex: :rewrite, repo: "hexpm", optional: false]}, {:sourceror, "~> 1.4", [hex: :sourceror, repo: "hexpm", optional: false]}, {:spitfire, ">= 0.1.3 and < 1.0.0-0", [hex: :spitfire, repo: "hexpm", optional: false]}], "hexpm", "b2bb1a7afd38d56db5a093c398eb3369b98cd25acaa0aca6128ef20428761f03"},
   "inflex": {:hex, :inflex, "2.1.0", "a365cf0821a9dacb65067abd95008ca1b0bb7dcdd85ae59965deef2aa062924c", [:mix], [], "hexpm", "14c17d05db4ee9b6d319b0bff1bdf22aa389a25398d1952c7a0b5f3d93162dd8"},
   "jason": {:hex, :jason, "1.4.4", "b9226785a9aa77b6857ca22832cffa5d5011a667207eb2a0ad56adb5db443b8a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "c5eb0cab91f094599f94d55bc63409236a8ec69a21a67814529e8d5f6cc90b3b"},
-  "langchain": {:hex, :langchain, "0.3.2", "60117628038cd972707c850e35243aa5afcee214bb91214dca5fb20589edd106", [:mix], [{:abacus, "~> 2.1.0", [hex: :abacus, repo: "hexpm", optional: true]}, {:ecto, "~> 3.10", [hex: :ecto, repo: "hexpm", optional: false]}, {:gettext, "~> 0.20", [hex: :gettext, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.4", [hex: :nimble_parsec, repo: "hexpm", optional: true]}, {:nx, ">= 0.7.0", [hex: :nx, repo: "hexpm", optional: true]}, {:req, ">= 0.5.2", [hex: :req, repo: "hexpm", optional: false]}], "hexpm", "29382c8a241289d1f5723b90d3d8002f51480c12d2265d001e1e0891fb534c55"},
+  "langchain": {:hex, :langchain, "0.3.3", "e69965fe760c4daac16eb08a44b9854929b290734aaa53febc611ba638f95b60", [:mix], [{:abacus, "~> 2.1.0", [hex: :abacus, repo: "hexpm", optional: true]}, {:ecto, "~> 3.10", [hex: :ecto, repo: "hexpm", optional: false]}, {:gettext, "~> 0.20", [hex: :gettext, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.4", [hex: :nimble_parsec, repo: "hexpm", optional: true]}, {:nx, ">= 0.7.0", [hex: :nx, repo: "hexpm", optional: true]}, {:req, ">= 0.5.2", [hex: :req, repo: "hexpm", optional: false]}], "hexpm", "7112945665d96adcc0fa8024f81dbdafbe1abbf85fb627ba83205610ba2e5f33"},
   "mdex": {:hex, :mdex, "0.5.0", "252c83cebc6a089801dfc1e142b4d98c9c358378ec7096a94796bce8bd13b0fc", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:nimble_options, "~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:rustler, "~> 0.32", [hex: :rustler, repo: "hexpm", optional: false]}, {:rustler_precompiled, "~> 0.7", [hex: :rustler_precompiled, repo: "hexpm", optional: false]}], "hexpm", "73e3ddee03130267e3be6aaf47a7f423c6f86add4bb5c62b352465cd9fb87d95"},
   "mime": {:hex, :mime, "2.0.7", "b8d739037be7cd402aee1ba0306edfdef982687ee7e9859bee6198c1e7e2f128", [:mix], [], "hexpm", "6171188e399ee16023ffc5b76ce445eb6d9672e2e241d2df6050f3c771e80ccd"},
   "mint": {:hex, :mint, "1.7.1", "113fdb2b2f3b59e47c7955971854641c61f378549d73e829e1768de90fc1abf1", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1 or ~> 0.2.0 or ~> 1.0", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "fceba0a4d0f24301ddee3024ae116df1c3f4bb7a563a731f45fdfeb9d39a231b"},

test/support/seeds.ex

@@ -88,7 +88,14 @@ defmodule Exmeralda.Seeds do
         insert_idempotently(%Exmeralda.LLM.ModelConfig{
           id: "7420d870-b10b-46ba-b30b-5c4630ee3a99",
           name: "llama3.2:latest",
-          config: %{stream: true}
+          # Ollama has a context window of 2048 by default, and this is also Langchain's default
+          # https://github.com/brainlid/langchain/blob/47de3e44e09c51a811e1e3262e161e1a92a4b77d/lib/chat_models/chat_ollama_ai.ex#L114
+          # This context window is likely too small for our prompts.
+          # Setting `num_ctx` allows to increase the context window.
+          #
+          # `num_predict: -2` means that the rest of the models context can be used to predit tokens for the response.
+          # The default is 128 and leads to truncated responses with the increased `num_ctx`.
+          config: %{stream: true, num_ctx: 32_768, num_predict: -2}
         })
 
       insert_idempotently(%Exmeralda.LLM.ModelConfigProvider{
@@ -108,7 +115,7 @@ defmodule Exmeralda.Seeds do
 
       qwen_25_32b_model_config =
         insert_idempotently(%Exmeralda.LLM.ModelConfig{
-          id: "eff70662-1576-491d-a1ef-1d025772e637",
+          id: "eff70662-1576-491d-a1ef-1d025772e638",
           name: "qwen25-coder-32b",
           config: %{stream: true}
         })