optimized with version 2

core/urls.py

@@ -17,6 +17,7 @@ from django.urls import path
 from core.views import (
     AttributeScoreView,
     BatchScoreView,
+    BatchScoreViewV2,
     ContentRulesView,
     ExcelUploadView,
 )
@@ -25,12 +26,11 @@ urlpatterns = [
     # Single product scoring
     path('api/score/', AttributeScoreView.as_view(), name='score_product'),
     
-    # Batch scoring
     path('api/batch-score/', BatchScoreView.as_view(), name='batch_score'),
+
+    path('api/batch-score-v2/', BatchScoreViewV2.as_view(), name='batch_score'),
     
-    # Content rules management
     path('api/content-rules/', ContentRulesView.as_view(), name='content_rules'),
     
     path('api/upload-rules/', ExcelUploadView.as_view(), name='upload_rules')
-
 ]

core/views.py

@@ -600,6 +600,206 @@ class BatchScoreView(View):
             return JsonResponse({'error': str(e)}, status=500)
 
 
+
+import json
+import logging
+import concurrent.futures
+from django.http import JsonResponse
+from django.utils.decorators import method_decorator
+from django.views import View
+from django.views.decorators.csrf import csrf_exempt
+from django.db.models import Q
+from textblob import download_corpora
+from concurrent.futures import ThreadPoolExecutor, as_completed
+import time
+
+from core.models import Product, CategoryAttributeRule, ProductContentRule
+# from .scorers.attribute_scorer import AttributeQualityScorer
+# from .scorers.image_scorer import ImageQualityScorer
+# from .utils import resolve_image_path, categorize_issues_and_suggestions
+
+logger = logging.getLogger(__name__)
+
+# Try to ensure corpora are downloaded once at import
+try:
+    download_corpora.download_all()
+    logger.info("[INIT] TextBlob corpora verified.")
+except Exception as e:
+    logger.warning(f"[INIT] Failed to auto-download TextBlob corpora: {e}")
+
+
+@method_decorator(csrf_exempt, name='dispatch')
+class BatchScoreViewV2(View):
+    """Optimized Batch scoring with image support and parallel processing"""
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.attribute_scorer = AttributeQualityScorer(use_ai=True)
+        self.image_scorer = ImageQualityScorer(use_ai=True)
+        logger.info("[INIT] BatchScoreView initialized with AI scorers preloaded")
+
+    def _process_single_product(self, product_data, idx, total):
+        """Process a single product safely (run inside thread pool)"""
+        start = time.time()
+        sku = product_data.get('sku')
+        category = product_data.get('category')
+
+        if not sku or not category:
+            return {'sku': sku, 'error': 'Missing SKU or category'}
+
+        try:
+            logger.info(f"[BATCH][{idx}/{total}] Processing SKU: {sku}")
+
+            # Fetch rules
+            category_rules = list(CategoryAttributeRule.objects.filter(category=category).values())
+            if not category_rules:
+                return {'sku': sku, 'error': f'No attribute rules for category {category}'}
+
+            content_rules = list(
+                ProductContentRule.objects.filter(
+                    Q(category__isnull=True) | Q(category=category)
+                ).values()
+            )
+
+            # Score attributes
+            score_result = self.attribute_scorer.score_product(
+                product_data,
+                category_rules,
+                content_rules=content_rules,
+                generate_ai_suggestions=True
+            )
+
+            # Score image
+            image_path = resolve_image_path(product_data.get('image_path'))
+            image_score_result = None
+
+            if image_path:
+                try:
+                    image_score_result = self.image_scorer.score_image(
+                        product_data,
+                        image_path=image_path
+                    )
+                except Exception as img_err:
+                    logger.warning(f"[IMG] Scoring failed for {sku}: {img_err}")
+                    image_score_result = {
+                        'image_score': None,
+                        'breakdown': {},
+                        'issues': ['Image scoring failed'],
+                        'suggestions': ['Check image file'],
+                        'image_metadata': {}
+                    }
+            else:
+                image_score_result = {
+                    'image_score': None,
+                    'breakdown': {},
+                    'issues': ['No image provided'],
+                    'suggestions': ['Upload product image'],
+                    'image_metadata': {}
+                }
+
+            # Combine results
+            all_issues = score_result['issues'] + image_score_result.get('issues', [])
+            all_suggestions = score_result['suggestions'] + image_score_result.get('suggestions', [])
+            categorized = categorize_issues_and_suggestions(all_issues, all_suggestions)
+
+            total_time = round(time.time() - start, 2)
+
+            return {
+                'sku': sku,
+                'title': product_data.get('title'),
+                'description': product_data.get('description'),
+                'image_path': image_path,
+                'final_score': score_result['final_score'],
+                'max_score': score_result['max_score'],
+                'breakdown': {**score_result['breakdown'], 'image_score': image_score_result.get('image_score')},
+                'image_score': image_score_result.get('image_score'),
+                'image_breakdown': image_score_result.get('breakdown', {}),
+                'image_metadata': image_score_result.get('image_metadata', {}),
+                'ai_suggestions': {
+                    'content': score_result.get('ai_suggestions', {}),
+                    'image': image_score_result.get('ai_improvements', {})
+                },
+                'categorized_feedback': categorized,
+                'issues': all_issues,
+                'suggestions': all_suggestions,
+                'processing_time': total_time
+            }
+
+        except Exception as e:
+            logger.error(f"[BATCH][{sku}] Error: {e}", exc_info=True)
+            return {'sku': sku, 'error': str(e)}
+
+    def _run_batch(self, products):
+        """Run products in parallel using thread pool"""
+        results, errors = [], []
+        total = len(products)
+        start_time = time.time()
+
+        # Use up to 8 threads (tune based on your CPU)
+        with ThreadPoolExecutor(max_workers=8) as executor:
+            future_to_product = {
+                executor.submit(self._process_single_product, product, idx + 1, total): product
+                for idx, product in enumerate(products)
+            }
+
+            for future in as_completed(future_to_product):
+                res = future.result()
+                if "error" in res:
+                    errors.append(res)
+                else:
+                    results.append(res)
+
+        elapsed = round(time.time() - start_time, 2)
+        logger.info(f"[BATCH] Completed {len(results)} products in {elapsed}s ({len(errors)} errors)")
+        return results, errors, elapsed
+
+    def get(self, request):
+        """Score all products from DB automatically"""
+        try:
+            products = list(Product.objects.all().values())
+            if not products:
+                return JsonResponse({'error': 'No products found in database'}, status=404)
+
+            results, errors, elapsed = self._run_batch(products)
+            return JsonResponse({
+                'success': True,
+                'processed': len(results),
+                'results': results,
+                'errors': errors,
+                'elapsed_seconds': elapsed
+            })
+
+        except Exception as e:
+            logger.error(f"[BATCH][GET] Error: {e}", exc_info=True)
+            return JsonResponse({'error': str(e)}, status=500)
+
+    def post(self, request):
+        """Batch score via payload"""
+        try:
+            data = json.loads(request.body)
+            products = data.get('products', [])
+            if not products:
+                return JsonResponse({'error': 'No products provided'}, status=400)
+
+            results, errors, elapsed = self._run_batch(products)
+            return JsonResponse({
+                'success': True,
+                'processed': len(results),
+                'results': results,
+                'errors': errors,
+                'elapsed_seconds': elapsed
+            })
+
+        except Exception as e:
+            logger.error(f"[BATCH][POST] Error: {e}", exc_info=True)
+            return JsonResponse({'error': str(e)}, status=500)
+
+
+
+
+
+
+
 @method_decorator(csrf_exempt, name='dispatch')
 class ContentRulesView(View):
     """API to manage ProductContentRules"""