gemini_service.py 31 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800
  1. # #gemini_service.py
  2. # import google.generativeai as genai
  3. # import json
  4. # import logging
  5. # import re
  6. # from typing import Dict, List
  7. # from django.conf import settings
  8. # from tenacity import retry, stop_after_attempt, wait_exponential, retry_if_exception_type
  9. # logger = logging.getLogger(__name__)
  10. # class GeminiAttributeService:
  11. # """Service to interact with Google Gemini API for attribute and SEO suggestions"""
  12. # def __init__(self):
  13. # # Configure Gemini API
  14. # api_key = getattr(settings, 'GEMINI_API_KEY', None)
  15. # if not api_key:
  16. # raise ValueError("GEMINI_API_KEY not found in settings")
  17. # genai.configure(api_key=api_key)
  18. # self.model = genai.GenerativeModel('gemini-2.0-flash-exp') # Use latest model
  19. # @retry(
  20. # stop=stop_after_attempt(3),
  21. # wait=wait_exponential(multiplier=1, min=2, max=10),
  22. # retry=retry_if_exception_type(Exception),
  23. # before_sleep=lambda retry_state: logger.info(f"Retrying Gemini API call, attempt {retry_state.attempt_number}")
  24. # )
  25. # def _call_gemini_api(self, prompt, max_tokens=8192):
  26. # """Helper method to call Gemini API with retry logic"""
  27. # return self.model.generate_content(
  28. # prompt,
  29. # generation_config=genai.types.GenerationConfig(
  30. # temperature=0.2, # Lower for more consistent JSON
  31. # top_p=0.9,
  32. # top_k=40,
  33. # max_output_tokens=max_tokens, # Increased default
  34. # response_mime_type="application/json" # Force JSON output
  35. # ),
  36. # safety_settings={
  37. # genai.types.HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT: genai.types.HarmBlockThreshold.BLOCK_NONE,
  38. # genai.types.HarmCategory.HARM_CATEGORY_HARASSMENT: genai.types.HarmBlockThreshold.BLOCK_NONE,
  39. # genai.types.HarmCategory.HARM_CATEGORY_HATE_SPEECH: genai.types.HarmBlockThreshold.BLOCK_NONE,
  40. # genai.types.HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT: genai.types.HarmBlockThreshold.BLOCK_NONE
  41. # }
  42. # )
  43. # def generate_attribute_suggestions(
  44. # self,
  45. # product: Dict,
  46. # issues: List[str],
  47. # category_rules: List[Dict]
  48. # ) -> Dict:
  49. # """
  50. # Use Gemini to generate intelligent suggestions for fixing attribute issues
  51. # Includes SEO-aware recommendations with robust error handling
  52. # """
  53. # try:
  54. # # Limit issues to prevent prompt overflow
  55. # limited_issues = issues[:15] if len(issues) > 15 else issues
  56. # prompt = self._build_prompt(product, limited_issues, category_rules)
  57. # response = self._call_gemini_api(prompt, max_tokens=8192)
  58. # # Check if response exists
  59. # if not response or not response.candidates:
  60. # logger.error(f"No candidates returned for SKU: {product.get('sku')}")
  61. # return {
  62. # 'error': 'No candidates returned by Gemini API',
  63. # 'fallback_suggestions': self._generate_fallback_suggestions(limited_issues)
  64. # }
  65. # candidate = response.candidates[0]
  66. # finish_reason_name = candidate.finish_reason.name
  67. # # Handle different finish reasons
  68. # if finish_reason_name == "MAX_TOKENS":
  69. # logger.warning(f"Max tokens reached for SKU: {product.get('sku')}, attempting partial parse")
  70. # # Try to parse partial response
  71. # try:
  72. # partial_result = self._parse_response(response.text)
  73. # if partial_result and 'error' not in partial_result:
  74. # return partial_result
  75. # except:
  76. # pass
  77. # # Retry with fewer issues
  78. # if len(issues) > 5:
  79. # logger.info("Retrying with fewer issues")
  80. # return self.generate_attribute_suggestions(product, issues[:5], category_rules)
  81. # else:
  82. # return {
  83. # 'error': 'Response too long, using fallback',
  84. # 'fallback_suggestions': self._generate_fallback_suggestions(limited_issues)
  85. # }
  86. # elif finish_reason_name in ("SAFETY", "RECITATION", "OTHER"):
  87. # logger.error(f"Response blocked by {finish_reason_name} for SKU: {product.get('sku')}")
  88. # return {
  89. # 'error': f'Response blocked by {finish_reason_name} filters',
  90. # 'safety_ratings': [
  91. # {'category': str(r.category), 'probability': str(r.probability)}
  92. # for r in candidate.safety_ratings
  93. # ],
  94. # 'fallback_suggestions': self._generate_fallback_suggestions(limited_issues)
  95. # }
  96. # elif finish_reason_name != "STOP":
  97. # logger.warning(f"Unexpected finish reason: {finish_reason_name}")
  98. # return {
  99. # 'error': f'Unexpected finish reason: {finish_reason_name}',
  100. # 'fallback_suggestions': self._generate_fallback_suggestions(limited_issues)
  101. # }
  102. # # Parse successful response
  103. # logger.info(f"Successfully received response for SKU: {product.get('sku')}")
  104. # suggestions = self._parse_response(response.text)
  105. # if 'error' in suggestions:
  106. # logger.warning(f"Parse error for SKU: {product.get('sku')}, using fallback")
  107. # suggestions['fallback_suggestions'] = self._generate_fallback_suggestions(limited_issues)
  108. # return suggestions
  109. # except Exception as e:
  110. # logger.error(f"Gemini API error for SKU {product.get('sku')}: {str(e)}", exc_info=True)
  111. # return {
  112. # 'error': str(e),
  113. # 'fallback_suggestions': self._generate_fallback_suggestions(issues[:10])
  114. # }
  115. # def _build_prompt(self, product: Dict, issues: List[str], rules: List[Dict]) -> str:
  116. # """Build a concise, structured prompt for Gemini with SEO awareness"""
  117. # mandatory_attrs = [r['attribute_name'] for r in rules if r.get('is_mandatory')]
  118. # valid_values_map = {
  119. # r['attribute_name']: r.get('valid_values', [])[:5] # Limit to 5 values
  120. # for r in rules if r.get('valid_values')
  121. # }
  122. # # Sanitize and categorize issues
  123. # cleaned_issues = [
  124. # issue.replace("suspiciously short", "short value")
  125. # .replace("not recognized", "invalid")
  126. # .replace("likely means", "should be")
  127. # .replace("not clearly mentioned", "missing")
  128. # for issue in issues
  129. # ]
  130. # seo_issues = [i for i in cleaned_issues if i.startswith("SEO:")][:5]
  131. # attribute_issues = [i for i in cleaned_issues if not i.startswith("SEO:")][:8]
  132. # # Shortened prompt
  133. # prompt = f"""Analyze this e-commerce product and provide JSON suggestions.
  134. # PRODUCT:
  135. # SKU: {product.get('sku')}
  136. # Category: {product.get('category')}
  137. # Title: {product.get('title', '')[:200]}
  138. # Description: {product.get('description', '')[:300]}
  139. # Attributes: {json.dumps(product.get('attributes', {}), ensure_ascii=False)}
  140. # RULES:
  141. # Mandatory: {', '.join(mandatory_attrs)}
  142. # Valid Values: {json.dumps(valid_values_map, ensure_ascii=False)}
  143. # ISSUES ({len(attribute_issues)} attribute, {len(seo_issues)} SEO):
  144. # {chr(10).join(f"• {i}" for i in attribute_issues[:8])}
  145. # {chr(10).join(f"• {i}" for i in seo_issues[:5])}
  146. # Return ONLY this JSON structure (no markdown, no explanation):
  147. # {{
  148. # "corrected_attributes": {{"attr": "value"}},
  149. # "missing_attributes": {{"attr": "value"}},
  150. # "seo_optimizations": {{
  151. # "optimized_title": "50-100 char title",
  152. # "optimized_description": "50-150 word description",
  153. # "recommended_keywords": ["kw1", "kw2", "kw3"]
  154. # }},
  155. # "improvements": [
  156. # {{"issue": "...", "suggestion": "...", "confidence": "high/medium/low", "type": "attribute/seo"}}
  157. # ],
  158. # "quality_score_prediction": 85,
  159. # "reasoning": "Brief explanation"
  160. # }}
  161. # IMPORTANT: Keep response under 6000 tokens. Prioritize top 3 most critical improvements."""
  162. # return prompt
  163. # def _parse_response(self, response_text: str) -> Dict:
  164. # """Enhanced JSON parsing with multiple fallback strategies"""
  165. # if not response_text or not response_text.strip():
  166. # return {'error': 'Empty response from API'}
  167. # try:
  168. # # Strategy 1: Direct JSON parse (works with response_mime_type="application/json")
  169. # try:
  170. # parsed = json.loads(response_text)
  171. # logger.info("Successfully parsed JSON directly")
  172. # return parsed
  173. # except json.JSONDecodeError:
  174. # pass
  175. # # Strategy 2: Remove markdown code blocks
  176. # cleaned = response_text.strip()
  177. # if '```' in cleaned:
  178. # # Extract content between code blocks
  179. # match = re.search(r'```(?:json)?\s*(\{.*\})\s*```', cleaned, re.DOTALL)
  180. # if match:
  181. # cleaned = match.group(1)
  182. # else:
  183. # # Remove all code block markers
  184. # cleaned = re.sub(r'```(?:json)?', '', cleaned).strip()
  185. # # Strategy 3: Find first { and last }
  186. # first_brace = cleaned.find('{')
  187. # last_brace = cleaned.rfind('}')
  188. # if first_brace != -1 and last_brace != -1 and last_brace > first_brace:
  189. # cleaned = cleaned[first_brace:last_brace + 1]
  190. # # Strategy 4: Try parsing cleaned JSON
  191. # try:
  192. # parsed = json.loads(cleaned)
  193. # logger.info("Successfully parsed JSON after cleaning")
  194. # return parsed
  195. # except json.JSONDecodeError as e:
  196. # logger.warning(f"JSON parse error at position {e.pos}: {e.msg}")
  197. # # Strategy 5: Attempt to fix common JSON issues
  198. # cleaned = self._fix_json_syntax(cleaned)
  199. # try:
  200. # parsed = json.loads(cleaned)
  201. # logger.info("Successfully parsed JSON after syntax fixes")
  202. # return parsed
  203. # except json.JSONDecodeError:
  204. # pass
  205. # # Strategy 6: Extract partial valid JSON
  206. # partial_json = self._extract_partial_json(cleaned)
  207. # if partial_json:
  208. # logger.warning("Using partial JSON response")
  209. # return partial_json
  210. # # All strategies failed
  211. # logger.error(f"All JSON parsing strategies failed. Response length: {len(response_text)}")
  212. # logger.error(f"Response preview: {response_text[:500]}...")
  213. # return {
  214. # 'error': 'Failed to parse AI response',
  215. # 'raw_response': response_text[:1000], # Limit size
  216. # 'parse_attempts': 6
  217. # }
  218. # except Exception as e:
  219. # logger.error(f"Unexpected error in _parse_response: {e}", exc_info=True)
  220. # return {
  221. # 'error': f'Parse exception: {str(e)}',
  222. # 'raw_response': response_text[:500] if response_text else 'None'
  223. # }
  224. # def _fix_json_syntax(self, json_str: str) -> str:
  225. # """Attempt to fix common JSON syntax issues"""
  226. # try:
  227. # # Remove trailing commas before closing braces/brackets
  228. # json_str = re.sub(r',\s*([}\]])', r'\1', json_str)
  229. # # Fix unescaped quotes in strings (simple heuristic)
  230. # # This is risky but can help in some cases
  231. # json_str = re.sub(r'(?<!\\)"(?=[^,:}\]]*[,:}\]])', '\\"', json_str)
  232. # # Remove any trailing content after final }
  233. # last_brace = json_str.rfind('}')
  234. # if last_brace != -1:
  235. # json_str = json_str[:last_brace + 1]
  236. # return json_str
  237. # except:
  238. # return json_str
  239. # def _extract_partial_json(self, json_str: str) -> Dict:
  240. # """Extract valid partial JSON by finding complete objects"""
  241. # try:
  242. # # Try to find complete nested structures
  243. # depth = 0
  244. # start_idx = json_str.find('{')
  245. # if start_idx == -1:
  246. # return None
  247. # for i in range(start_idx, len(json_str)):
  248. # if json_str[i] == '{':
  249. # depth += 1
  250. # elif json_str[i] == '}':
  251. # depth -= 1
  252. # if depth == 0:
  253. # # Found complete JSON object
  254. # try:
  255. # return json.loads(json_str[start_idx:i+1])
  256. # except:
  257. # continue
  258. # return None
  259. # except:
  260. # return None
  261. # def _generate_fallback_suggestions(self, issues: List[str]) -> List[Dict]:
  262. # """Generate enhanced fallback suggestions based on issues"""
  263. # suggestions = []
  264. # # Group similar issues
  265. # issue_categories = {
  266. # 'missing': [],
  267. # 'invalid': [],
  268. # 'seo': [],
  269. # 'other': []
  270. # }
  271. # for issue in issues:
  272. # if 'missing' in issue.lower() or 'mandatory' in issue.lower():
  273. # issue_categories['missing'].append(issue)
  274. # elif 'invalid' in issue.lower() or 'not in valid' in issue.lower():
  275. # issue_categories['invalid'].append(issue)
  276. # elif issue.startswith('SEO:'):
  277. # issue_categories['seo'].append(issue)
  278. # else:
  279. # issue_categories['other'].append(issue)
  280. # # Generate consolidated suggestions
  281. # for category, category_issues in issue_categories.items():
  282. # if not category_issues:
  283. # continue
  284. # for issue in category_issues[:5]: # Limit to 5 per category
  285. # suggestion = "Review and correct this issue"
  286. # confidence = "medium"
  287. # issue_type = "seo" if category == 'seo' else "attribute"
  288. # # Specific suggestions
  289. # if "Missing mandatory field" in issue:
  290. # attr = issue.split(":")[-1].strip()
  291. # suggestion = f"Add {attr} - check product details or title/description"
  292. # confidence = "high"
  293. # elif "not in valid values" in issue or "invalid" in issue.lower():
  294. # suggestion = "Use one of the valid values from category rules"
  295. # confidence = "high"
  296. # elif "placeholder" in issue.lower():
  297. # suggestion = "Replace with actual product data"
  298. # confidence = "high"
  299. # elif "too short" in issue.lower():
  300. # if "title" in issue.lower():
  301. # suggestion = "Expand to 50-100 characters with key attributes"
  302. # confidence = "high"
  303. # issue_type = "seo"
  304. # elif "description" in issue.lower():
  305. # suggestion = "Expand to 50-150 words with details"
  306. # confidence = "high"
  307. # issue_type = "seo"
  308. # else:
  309. # suggestion = "Provide more detailed information"
  310. # confidence = "medium"
  311. # elif "keyword" in issue.lower() or "search term" in issue.lower():
  312. # suggestion = "Add relevant keywords to improve discoverability"
  313. # confidence = "medium"
  314. # issue_type = "seo"
  315. # suggestions.append({
  316. # 'issue': issue,
  317. # 'suggestion': suggestion,
  318. # 'confidence': confidence,
  319. # 'type': issue_type,
  320. # 'category': category
  321. # })
  322. # return suggestions[:15] # Return top 15 suggestions
  323. # def extract_attributes_with_ai(self, title: str, description: str, category: str) -> Dict:
  324. # """
  325. # Use Gemini to extract attributes from unstructured text
  326. # """
  327. # try:
  328. # prompt = f"""Extract product attributes from this text. Return ONLY valid JSON.
  329. # Category: {category}
  330. # Title: {title[:200]}
  331. # Description: {description[:400]}
  332. # Return format:
  333. # {{
  334. # "brand": "value or null",
  335. # "color": "value or null",
  336. # "size": "value or null",
  337. # "material": "value or null",
  338. # "model": "value or null"
  339. # }}"""
  340. # response = self._call_gemini_api(prompt, max_tokens=1024)
  341. # if not response or not response.candidates:
  342. # return {'error': 'No response'}
  343. # return self._parse_response(response.text)
  344. # except Exception as e:
  345. # logger.error(f"AI extraction error: {str(e)}")
  346. # return {'error': str(e)}
  347. # gemini_service_enhanced.py
  348. """
  349. Enhanced Gemini service with comprehensive suggestions for all components
  350. """
  351. import google.generativeai as genai
  352. import json
  353. import logging
  354. import re
  355. from typing import Dict, List
  356. from django.conf import settings
  357. from tenacity import retry, stop_after_attempt, wait_exponential, retry_if_exception_type
  358. logger = logging.getLogger(__name__)
  359. class GeminiAttributeService:
  360. """Enhanced service with comprehensive AI suggestions"""
  361. def __init__(self):
  362. api_key = getattr(settings, 'GEMINI_API_KEY', None)
  363. if not api_key:
  364. raise ValueError("GEMINI_API_KEY not found in settings")
  365. genai.configure(api_key=api_key)
  366. self.model = genai.GenerativeModel('gemini-2.0-flash-exp')
  367. @retry(
  368. stop=stop_after_attempt(3),
  369. wait=wait_exponential(multiplier=1, min=2, max=10),
  370. retry=retry_if_exception_type(Exception)
  371. )
  372. def _call_gemini_api(self, prompt, max_tokens=8192):
  373. """Helper method to call Gemini API with retry logic"""
  374. return self.model.generate_content(
  375. prompt,
  376. generation_config=genai.types.GenerationConfig(
  377. temperature=0.2,
  378. top_p=0.9,
  379. top_k=40,
  380. max_output_tokens=max_tokens,
  381. response_mime_type="application/json"
  382. ),
  383. safety_settings={
  384. genai.types.HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT: genai.types.HarmBlockThreshold.BLOCK_NONE,
  385. genai.types.HarmCategory.HARM_CATEGORY_HARASSMENT: genai.types.HarmBlockThreshold.BLOCK_NONE,
  386. genai.types.HarmCategory.HARM_CATEGORY_HATE_SPEECH: genai.types.HarmBlockThreshold.BLOCK_NONE,
  387. genai.types.HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT: genai.types.HarmBlockThreshold.BLOCK_NONE
  388. }
  389. )
  390. def generate_comprehensive_suggestions(
  391. self,
  392. product: Dict,
  393. issues: List[str],
  394. category_rules: List[Dict],
  395. scores: Dict
  396. ) -> Dict:
  397. """
  398. Generate comprehensive AI suggestions covering ALL quality aspects
  399. """
  400. try:
  401. limited_issues = issues[:20] if len(issues) > 20 else issues
  402. prompt = self._build_comprehensive_prompt(product, limited_issues, category_rules, scores)
  403. response = self._call_gemini_api(prompt, max_tokens=8192)
  404. if not response or not response.candidates:
  405. logger.error(f"No candidates returned for SKU: {product.get('sku')}")
  406. return {
  407. 'error': 'No response from AI',
  408. 'fallback_suggestions': self._generate_fallback_suggestions(limited_issues)
  409. }
  410. candidate = response.candidates[0]
  411. finish_reason = candidate.finish_reason.name
  412. if finish_reason != "STOP":
  413. logger.warning(f"Non-STOP finish reason: {finish_reason}")
  414. if finish_reason == "MAX_TOKENS" and len(issues) > 10:
  415. return self.generate_comprehensive_suggestions(product, issues[:10], category_rules, scores)
  416. return {
  417. 'error': f'Response blocked: {finish_reason}',
  418. 'fallback_suggestions': self._generate_fallback_suggestions(limited_issues)
  419. }
  420. logger.info(f"Successfully received comprehensive suggestions for SKU: {product.get('sku')}")
  421. suggestions = self._parse_response(response.text)
  422. if 'error' in suggestions:
  423. suggestions['fallback_suggestions'] = self._generate_fallback_suggestions(limited_issues)
  424. return suggestions
  425. except Exception as e:
  426. logger.error(f"Gemini API error: {str(e)}", exc_info=True)
  427. return {
  428. 'error': str(e),
  429. 'fallback_suggestions': self._generate_fallback_suggestions(issues[:15])
  430. }
  431. def _build_comprehensive_prompt(
  432. self,
  433. product: Dict,
  434. issues: List[str],
  435. rules: List[Dict],
  436. scores: Dict
  437. ) -> str:
  438. """Build comprehensive prompt for all quality aspects"""
  439. mandatory_attrs = [r['attribute_name'] for r in rules if r.get('is_mandatory')]
  440. valid_values_map = {
  441. r['attribute_name']: r.get('valid_values', [])[:5]
  442. for r in rules if r.get('valid_values')
  443. }
  444. # Categorize issues
  445. attribute_issues = [i for i in issues if not any(prefix in i for prefix in ['Title:', 'Description:', 'SEO:'])]
  446. title_issues = [i for i in issues if i.startswith('Title:')]
  447. desc_issues = [i for i in issues if i.startswith('Description:')]
  448. seo_issues = [i for i in issues if i.startswith('SEO:')]
  449. prompt = f"""Analyze this e-commerce product and provide comprehensive quality improvements.
  450. PRODUCT DATA:
  451. SKU: {product.get('sku')}
  452. Category: {product.get('category')}
  453. Title: {product.get('title', '')[:250]}
  454. Description: {product.get('description', '')[:400]}
  455. Attributes: {json.dumps(product.get('attributes', {}), ensure_ascii=False)}
  456. QUALITY SCORES (out of 100):
  457. - Mandatory Fields: {scores.get('mandatory_fields', 0):.1f}
  458. - Standardization: {scores.get('standardization', 0):.1f}
  459. - Missing Values: {scores.get('missing_values', 0):.1f}
  460. - Consistency: {scores.get('consistency', 0):.1f}
  461. - SEO: {scores.get('seo_discoverability', 0):.1f}
  462. - Title Quality: {scores.get('title_quality', 0):.1f}
  463. - Description Quality: {scores.get('description_quality', 0):.1f}
  464. CATEGORY RULES:
  465. Mandatory Attributes: {', '.join(mandatory_attrs)}
  466. Valid Values: {json.dumps(valid_values_map, ensure_ascii=False)}
  467. ISSUES FOUND:
  468. Attributes ({len(attribute_issues)}):
  469. {chr(10).join(f" • {i}" for i in attribute_issues[:8])}
  470. Title ({len(title_issues)}):
  471. {chr(10).join(f" • {i}" for i in title_issues[:5])}
  472. Description ({len(desc_issues)}):
  473. {chr(10).join(f" • {i}" for i in desc_issues[:5])}
  474. SEO ({len(seo_issues)}):
  475. {chr(10).join(f" • {i}" for i in seo_issues[:5])}
  476. Return ONLY this JSON structure:
  477. {{
  478. "corrected_attributes": {{
  479. "attr_name": "corrected_value"
  480. }},
  481. "missing_attributes": {{
  482. "attr_name": "suggested_value"
  483. }},
  484. "improved_title": "optimized title (50-100 chars, includes brand, model, key features)",
  485. "improved_description": "enhanced description (50-150 words, features, benefits, specs, use cases)",
  486. "seo_keywords": ["keyword1", "keyword2", "keyword3"],
  487. "improvements": [
  488. {{
  489. "component": "attributes/title/description/seo",
  490. "issue": "specific issue",
  491. "suggestion": "how to fix",
  492. "priority": "high/medium/low",
  493. "confidence": "high/medium/low"
  494. }}
  495. ],
  496. "quality_score_prediction": 85,
  497. "summary": "Brief 2-3 sentence summary of key improvements needed"
  498. }}
  499. CRITICAL: Keep response under 7000 tokens. Focus on top 5 most impactful improvements."""
  500. return prompt
  501. def _parse_response(self, response_text: str) -> Dict:
  502. """Enhanced JSON parsing with fallback strategies"""
  503. if not response_text or not response_text.strip():
  504. return {'error': 'Empty response from API'}
  505. try:
  506. # Direct JSON parse
  507. try:
  508. parsed = json.loads(response_text)
  509. logger.info("Successfully parsed JSON directly")
  510. return parsed
  511. except json.JSONDecodeError:
  512. pass
  513. # Remove markdown code blocks
  514. cleaned = response_text.strip()
  515. if '```' in cleaned:
  516. match = re.search(r'```(?:json)?\s*(\{.*\})\s*```', cleaned, re.DOTALL)
  517. if match:
  518. cleaned = match.group(1)
  519. else:
  520. cleaned = re.sub(r'```(?:json)?', '', cleaned).strip()
  521. # Find first { and last }
  522. first_brace = cleaned.find('{')
  523. last_brace = cleaned.rfind('}')
  524. if first_brace != -1 and last_brace != -1 and last_brace > first_brace:
  525. cleaned = cleaned[first_brace:last_brace + 1]
  526. # Try parsing cleaned JSON
  527. try:
  528. parsed = json.loads(cleaned)
  529. logger.info("Successfully parsed JSON after cleaning")
  530. return parsed
  531. except json.JSONDecodeError as e:
  532. logger.warning(f"JSON parse error: {e}")
  533. # Fix common JSON issues
  534. cleaned = self._fix_json_syntax(cleaned)
  535. try:
  536. parsed = json.loads(cleaned)
  537. logger.info("Successfully parsed JSON after syntax fixes")
  538. return parsed
  539. except json.JSONDecodeError:
  540. pass
  541. # Extract partial valid JSON
  542. partial_json = self._extract_partial_json(cleaned)
  543. if partial_json:
  544. logger.warning("Using partial JSON response")
  545. return partial_json
  546. logger.error(f"All JSON parsing failed. Response length: {len(response_text)}")
  547. return {
  548. 'error': 'Failed to parse AI response',
  549. 'raw_response': response_text[:500]
  550. }
  551. except Exception as e:
  552. logger.error(f"Parse exception: {e}", exc_info=True)
  553. return {
  554. 'error': f'Parse exception: {str(e)}',
  555. 'raw_response': response_text[:500] if response_text else 'None'
  556. }
  557. def _fix_json_syntax(self, json_str: str) -> str:
  558. """Fix common JSON syntax issues"""
  559. try:
  560. # Remove trailing commas
  561. json_str = re.sub(r',\s*([}\]])', r'\1', json_str)
  562. # Remove trailing content after final }
  563. last_brace = json_str.rfind('}')
  564. if last_brace != -1:
  565. json_str = json_str[:last_brace + 1]
  566. return json_str
  567. except:
  568. return json_str
  569. def _extract_partial_json(self, json_str: str) -> Dict:
  570. """Extract valid partial JSON"""
  571. try:
  572. depth = 0
  573. start_idx = json_str.find('{')
  574. if start_idx == -1:
  575. return None
  576. for i in range(start_idx, len(json_str)):
  577. if json_str[i] == '{':
  578. depth += 1
  579. elif json_str[i] == '}':
  580. depth -= 1
  581. if depth == 0:
  582. try:
  583. return json.loads(json_str[start_idx:i+1])
  584. except:
  585. continue
  586. return None
  587. except:
  588. return None
  589. def _generate_fallback_suggestions(self, issues: List[str]) -> List[Dict]:
  590. """Generate fallback suggestions based on issues"""
  591. suggestions = []
  592. for issue in issues[:15]:
  593. suggestion_text = "Review and correct this issue"
  594. confidence = "medium"
  595. component = "attribute"
  596. priority = "medium"
  597. issue_lower = issue.lower()
  598. # Determine component
  599. if issue.startswith('Title:'):
  600. component = "title"
  601. elif issue.startswith('Description:'):
  602. component = "description"
  603. elif issue.startswith('SEO:'):
  604. component = "seo"
  605. # Specific suggestions
  606. if "missing mandatory" in issue_lower:
  607. attr = issue.split(":")[-1].strip()
  608. suggestion_text = f"Add required {attr} - check product packaging or manufacturer details"
  609. priority = "high"
  610. confidence = "high"
  611. elif "too short" in issue_lower:
  612. if "title" in issue_lower:
  613. suggestion_text = "Expand title to 50-100 characters including brand, model, and key features"
  614. component = "title"
  615. priority = "high"
  616. elif "description" in issue_lower:
  617. suggestion_text = "Write comprehensive 50-150 word description with features, benefits, and specifications"
  618. component = "description"
  619. priority = "high"
  620. else:
  621. suggestion_text = "Provide more detailed information"
  622. elif "placeholder" in issue_lower:
  623. suggestion_text = "Replace with actual product data from manufacturer or packaging"
  624. priority = "high"
  625. elif "grammar" in issue_lower or "spelling" in issue_lower:
  626. suggestion_text = "Run spell-check and grammar review, ensure professional language"
  627. component = "description"
  628. priority = "medium"
  629. elif "keyword" in issue_lower or "seo" in issue_lower:
  630. suggestion_text = "Add relevant search keywords and product attributes"
  631. component = "seo"
  632. priority = "medium"
  633. elif "duplicate" in issue_lower or "repetit" in issue_lower:
  634. suggestion_text = "Remove duplicate content, provide varied information with unique details"
  635. component = "description"
  636. priority = "medium"
  637. elif "not recognized" in issue_lower or "invalid" in issue_lower:
  638. suggestion_text = "Use standardized values from category rules"
  639. priority = "high"
  640. confidence = "high"
  641. suggestions.append({
  642. 'component': component,
  643. 'issue': issue,
  644. 'suggestion': suggestion_text,
  645. 'priority': priority,
  646. 'confidence': confidence
  647. })
  648. return suggestions