|
|
@@ -600,6 +600,206 @@ class BatchScoreView(View):
|
|
|
return JsonResponse({'error': str(e)}, status=500)
|
|
|
|
|
|
|
|
|
+
|
|
|
+import json
|
|
|
+import logging
|
|
|
+import concurrent.futures
|
|
|
+from django.http import JsonResponse
|
|
|
+from django.utils.decorators import method_decorator
|
|
|
+from django.views import View
|
|
|
+from django.views.decorators.csrf import csrf_exempt
|
|
|
+from django.db.models import Q
|
|
|
+from textblob import download_corpora
|
|
|
+from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
|
+import time
|
|
|
+
|
|
|
+from core.models import Product, CategoryAttributeRule, ProductContentRule
|
|
|
+# from .scorers.attribute_scorer import AttributeQualityScorer
|
|
|
+# from .scorers.image_scorer import ImageQualityScorer
|
|
|
+# from .utils import resolve_image_path, categorize_issues_and_suggestions
|
|
|
+
|
|
|
+logger = logging.getLogger(__name__)
|
|
|
+
|
|
|
+# Try to ensure corpora are downloaded once at import
|
|
|
+try:
|
|
|
+ download_corpora.download_all()
|
|
|
+ logger.info("[INIT] TextBlob corpora verified.")
|
|
|
+except Exception as e:
|
|
|
+ logger.warning(f"[INIT] Failed to auto-download TextBlob corpora: {e}")
|
|
|
+
|
|
|
+
|
|
|
+@method_decorator(csrf_exempt, name='dispatch')
|
|
|
+class BatchScoreViewV2(View):
|
|
|
+ """Optimized Batch scoring with image support and parallel processing"""
|
|
|
+
|
|
|
+ def __init__(self, *args, **kwargs):
|
|
|
+ super().__init__(*args, **kwargs)
|
|
|
+ self.attribute_scorer = AttributeQualityScorer(use_ai=True)
|
|
|
+ self.image_scorer = ImageQualityScorer(use_ai=True)
|
|
|
+ logger.info("[INIT] BatchScoreView initialized with AI scorers preloaded")
|
|
|
+
|
|
|
+ def _process_single_product(self, product_data, idx, total):
|
|
|
+ """Process a single product safely (run inside thread pool)"""
|
|
|
+ start = time.time()
|
|
|
+ sku = product_data.get('sku')
|
|
|
+ category = product_data.get('category')
|
|
|
+
|
|
|
+ if not sku or not category:
|
|
|
+ return {'sku': sku, 'error': 'Missing SKU or category'}
|
|
|
+
|
|
|
+ try:
|
|
|
+ logger.info(f"[BATCH][{idx}/{total}] Processing SKU: {sku}")
|
|
|
+
|
|
|
+ # Fetch rules
|
|
|
+ category_rules = list(CategoryAttributeRule.objects.filter(category=category).values())
|
|
|
+ if not category_rules:
|
|
|
+ return {'sku': sku, 'error': f'No attribute rules for category {category}'}
|
|
|
+
|
|
|
+ content_rules = list(
|
|
|
+ ProductContentRule.objects.filter(
|
|
|
+ Q(category__isnull=True) | Q(category=category)
|
|
|
+ ).values()
|
|
|
+ )
|
|
|
+
|
|
|
+ # Score attributes
|
|
|
+ score_result = self.attribute_scorer.score_product(
|
|
|
+ product_data,
|
|
|
+ category_rules,
|
|
|
+ content_rules=content_rules,
|
|
|
+ generate_ai_suggestions=True
|
|
|
+ )
|
|
|
+
|
|
|
+ # Score image
|
|
|
+ image_path = resolve_image_path(product_data.get('image_path'))
|
|
|
+ image_score_result = None
|
|
|
+
|
|
|
+ if image_path:
|
|
|
+ try:
|
|
|
+ image_score_result = self.image_scorer.score_image(
|
|
|
+ product_data,
|
|
|
+ image_path=image_path
|
|
|
+ )
|
|
|
+ except Exception as img_err:
|
|
|
+ logger.warning(f"[IMG] Scoring failed for {sku}: {img_err}")
|
|
|
+ image_score_result = {
|
|
|
+ 'image_score': None,
|
|
|
+ 'breakdown': {},
|
|
|
+ 'issues': ['Image scoring failed'],
|
|
|
+ 'suggestions': ['Check image file'],
|
|
|
+ 'image_metadata': {}
|
|
|
+ }
|
|
|
+ else:
|
|
|
+ image_score_result = {
|
|
|
+ 'image_score': None,
|
|
|
+ 'breakdown': {},
|
|
|
+ 'issues': ['No image provided'],
|
|
|
+ 'suggestions': ['Upload product image'],
|
|
|
+ 'image_metadata': {}
|
|
|
+ }
|
|
|
+
|
|
|
+ # Combine results
|
|
|
+ all_issues = score_result['issues'] + image_score_result.get('issues', [])
|
|
|
+ all_suggestions = score_result['suggestions'] + image_score_result.get('suggestions', [])
|
|
|
+ categorized = categorize_issues_and_suggestions(all_issues, all_suggestions)
|
|
|
+
|
|
|
+ total_time = round(time.time() - start, 2)
|
|
|
+
|
|
|
+ return {
|
|
|
+ 'sku': sku,
|
|
|
+ 'title': product_data.get('title'),
|
|
|
+ 'description': product_data.get('description'),
|
|
|
+ 'image_path': image_path,
|
|
|
+ 'final_score': score_result['final_score'],
|
|
|
+ 'max_score': score_result['max_score'],
|
|
|
+ 'breakdown': {**score_result['breakdown'], 'image_score': image_score_result.get('image_score')},
|
|
|
+ 'image_score': image_score_result.get('image_score'),
|
|
|
+ 'image_breakdown': image_score_result.get('breakdown', {}),
|
|
|
+ 'image_metadata': image_score_result.get('image_metadata', {}),
|
|
|
+ 'ai_suggestions': {
|
|
|
+ 'content': score_result.get('ai_suggestions', {}),
|
|
|
+ 'image': image_score_result.get('ai_improvements', {})
|
|
|
+ },
|
|
|
+ 'categorized_feedback': categorized,
|
|
|
+ 'issues': all_issues,
|
|
|
+ 'suggestions': all_suggestions,
|
|
|
+ 'processing_time': total_time
|
|
|
+ }
|
|
|
+
|
|
|
+ except Exception as e:
|
|
|
+ logger.error(f"[BATCH][{sku}] Error: {e}", exc_info=True)
|
|
|
+ return {'sku': sku, 'error': str(e)}
|
|
|
+
|
|
|
+ def _run_batch(self, products):
|
|
|
+ """Run products in parallel using thread pool"""
|
|
|
+ results, errors = [], []
|
|
|
+ total = len(products)
|
|
|
+ start_time = time.time()
|
|
|
+
|
|
|
+ # Use up to 8 threads (tune based on your CPU)
|
|
|
+ with ThreadPoolExecutor(max_workers=8) as executor:
|
|
|
+ future_to_product = {
|
|
|
+ executor.submit(self._process_single_product, product, idx + 1, total): product
|
|
|
+ for idx, product in enumerate(products)
|
|
|
+ }
|
|
|
+
|
|
|
+ for future in as_completed(future_to_product):
|
|
|
+ res = future.result()
|
|
|
+ if "error" in res:
|
|
|
+ errors.append(res)
|
|
|
+ else:
|
|
|
+ results.append(res)
|
|
|
+
|
|
|
+ elapsed = round(time.time() - start_time, 2)
|
|
|
+ logger.info(f"[BATCH] Completed {len(results)} products in {elapsed}s ({len(errors)} errors)")
|
|
|
+ return results, errors, elapsed
|
|
|
+
|
|
|
+ def get(self, request):
|
|
|
+ """Score all products from DB automatically"""
|
|
|
+ try:
|
|
|
+ products = list(Product.objects.all().values())
|
|
|
+ if not products:
|
|
|
+ return JsonResponse({'error': 'No products found in database'}, status=404)
|
|
|
+
|
|
|
+ results, errors, elapsed = self._run_batch(products)
|
|
|
+ return JsonResponse({
|
|
|
+ 'success': True,
|
|
|
+ 'processed': len(results),
|
|
|
+ 'results': results,
|
|
|
+ 'errors': errors,
|
|
|
+ 'elapsed_seconds': elapsed
|
|
|
+ })
|
|
|
+
|
|
|
+ except Exception as e:
|
|
|
+ logger.error(f"[BATCH][GET] Error: {e}", exc_info=True)
|
|
|
+ return JsonResponse({'error': str(e)}, status=500)
|
|
|
+
|
|
|
+ def post(self, request):
|
|
|
+ """Batch score via payload"""
|
|
|
+ try:
|
|
|
+ data = json.loads(request.body)
|
|
|
+ products = data.get('products', [])
|
|
|
+ if not products:
|
|
|
+ return JsonResponse({'error': 'No products provided'}, status=400)
|
|
|
+
|
|
|
+ results, errors, elapsed = self._run_batch(products)
|
|
|
+ return JsonResponse({
|
|
|
+ 'success': True,
|
|
|
+ 'processed': len(results),
|
|
|
+ 'results': results,
|
|
|
+ 'errors': errors,
|
|
|
+ 'elapsed_seconds': elapsed
|
|
|
+ })
|
|
|
+
|
|
|
+ except Exception as e:
|
|
|
+ logger.error(f"[BATCH][POST] Error: {e}", exc_info=True)
|
|
|
+ return JsonResponse({'error': str(e)}, status=500)
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
@method_decorator(csrf_exempt, name='dispatch')
|
|
|
class ContentRulesView(View):
|
|
|
"""API to manage ProductContentRules"""
|
