Skip to content

Reject malformed JSON in LLM coercion #161

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Fu Xiaonan (FU-max-boop) wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Reject malformed JSON in LLM coercion #161

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions tests/unit/test_llm_chat.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -408,6 +408,11 @@ def test_coerce_to_dict_or_list_valid_inputs(self):
result = chat._coerce_to_dict_or_list(json_array)
assert result == [{"name": "John"}, {"name": "Jane"}]

# Valid empty JSON object with whitespace
empty_json_object = " { }\n"
result = chat._coerce_to_dict_or_list(empty_json_object)
assert result == {}

# Already a dict
dict_input = {"key": "value"}
result = chat._coerce_to_dict_or_list(dict_input)
Expand All @@ -428,6 +433,15 @@ def test_coerce_to_dict_or_list_invalid_inputs(self):
with pytest.raises(ValueError, match="Cannot convert"):
chat._coerce_to_dict_or_list("42") # valid JSON but not dict/list

for malformed_json in (
"[{[}]",
"{not json}",
'prefix {"name": } suffix',
"malformed {[}] followed by {}",
):
with pytest.raises(ValueError, match="Cannot convert"):
chat._coerce_to_dict_or_list(malformed_json)

def test_coerce_to_list_valid_inputs(self):
"""Test list coercion with valid inputs."""
chat = LLMChat(system_prompt="Test", user_prompt="Test")
Expand Down
2 changes: 1 addition & 1 deletion tinytroupe/utils/llm.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -933,7 +933,7 @@ def _coerce_to_dict_or_list(self, llm_output: str):
# extract_json returns {} on failure, but we need dict or list
if result == {} and not (
isinstance(llm_output, str)
and ("{}" in llm_output or "{" in llm_output and "}" in llm_output)
and re.fullmatch(r"\s*\{\s*\}\s*", llm_output)
):
raise ValueError(
"Cannot convert the LLM output to a dict or list value."
Expand Down