fix: robust JSON parsing with 4-level repair + retry with higher max_tokens for CV parsing

ai_service.py

@@ -37,30 +37,66 @@ def llm_chat(messages, temperature=0.3, max_tokens=4000, response_format=None):
 
 
 def extract_json(text):
-    """Extract JSON from LLM response, handling markdown code blocks and truncation."""
+    """Extract JSON from LLM response, handling markdown code blocks, truncation, and common LLM JSON errors."""
     json_str = text.strip()
     if json_str.startswith("```"):
         json_str = re.sub(r'^```(?:json)?\s*', '', json_str)
         json_str = re.sub(r'\s*```$', '', json_str)
     
+    # First attempt: parse as-is
     try:
         return json.loads(json_str)
     except json.JSONDecodeError:
-        # Try to fix truncated JSON by closing open braces/brackets
+        pass
+    
+    # Second attempt: fix truncated JSON by closing open braces/brackets
+    try:
         open_b = json_str.count('{')
         close_b = json_str.count('}')
         open_arr = json_str.count('[')
         close_arr = json_str.count(']')
         if open_b > close_b or open_arr > close_arr:
-            # Remove any trailing partial content after last complete element
-            # then close the structures
             json_str = json_str.rstrip()
-            # Remove trailing incomplete key-value (e.g. cut off mid-string)
             json_str = re.sub(r'[\s,]*"[^"]*":\s*"?[^",}\]]*$', '', json_str)
             json_str += '}' * max(0, open_b - json_str.count('}'))
             json_str += ']' * max(0, open_arr - json_str.count(']'))
             return json.loads(json_str)
-        raise
+    except json.JSONDecodeError:
+        pass
+    
+    # Third attempt: fix common LLM JSON errors
+    # - Unescaped newlines inside string values
+    # - Single quotes instead of double quotes
+    # - Trailing commas before closing brackets
+    # - Unescaped quotes inside string values (the "Expecting ',' delimiter" error)
+    try:
+        fixed = json_str
+        # Remove trailing commas
+        fixed = re.sub(r',\s*([}\]])', r'\1', fixed)
+        # Fix unescaped newlines inside strings
+        fixed = fixed.replace('\\\n', '\\n')
+        return json.loads(fixed)
+    except json.JSONDecodeError:
+        pass
+    
+    # Fourth attempt: line-by-line repair — find the error location and try to fix it
+    try:
+        return json.loads(json_str, strict=False)
+    except json.JSONDecodeError as e:
+        # Last resort: try to salvage by removing the problematic section
+        # and parsing what we can
+        lines = json_str.split('\n')
+        if e.lineno and e.lineno <= len(lines):
+            # Remove the problematic line and try again
+            repaired = '\n'.join(lines[:e.lineno-1] + lines[e.lineno:])
+            try:
+                return json.loads(repaired, strict=False)
+            except json.JSONDecodeError:
+                pass
+        
+        # If all else fails, raise with context
+        error_line = lines[e.lineno-1] if e.lineno and e.lineno <= len(lines) else '?'
+        raise ValueError(f"JSON parse failed at line {e.lineno}: {e.msg}\nProblem line: {error_line[:200]}\nFull response length: {len(text)} chars")
 
 
 # ============================================================
@@ -144,11 +180,20 @@ def parse_cv(cv_text: str) -> dict:
     
     prompt = PARSE_PROMPT.replace("__CV_TEXT__", cv_text)
     messages = [
-        {"role": "system", "content": "You are a precise CV parser that outputs only valid JSON."},
+        {"role": "system", "content": "You are a precise CV parser that outputs only valid JSON. Make sure all string values are properly escaped (no unescaped quotes, no unescaped newlines)."},
         {"role": "user", "content": prompt}
     ]
-    resp = llm_chat(messages, temperature=0.1, max_tokens=4000)
-    return extract_json(resp)
+    
+    # Retry with increasing max_tokens in case of truncation
+    for max_tokens in [4000, 6000, 8000]:
+        try:
+            resp = llm_chat(messages, temperature=0.1, max_tokens=max_tokens)
+            return extract_json(resp)
+        except (ValueError, json.JSONDecodeError) as e:
+            if max_tokens == 8000:
+                raise
+            # Try again with more tokens
+            continue
 
 
 # ============================================================