|
|
@@ -132,11 +132,14 @@ class ExtractProductAttributesView(APIView):
|
|
|
return Response(result, status=status.HTTP_200_OK)
|
|
|
|
|
|
|
|
|
+
|
|
|
+# Replace the BatchExtractProductAttributesView in your views.py with this updated version
|
|
|
+
|
|
|
# class BatchExtractProductAttributesView(APIView):
|
|
|
# """
|
|
|
# API endpoint to extract product attributes for multiple products in batch.
|
|
|
# Uses item-specific mandatory_attrs with source tracking.
|
|
|
-# Returns attributes in array format: [{"value": "...", "source": "..."}]
|
|
|
+# Returns attributes in array format with original_value field.
|
|
|
# Includes OCR and Visual Processing results.
|
|
|
# """
|
|
|
|
|
|
@@ -147,15 +150,6 @@ class ExtractProductAttributesView(APIView):
|
|
|
|
|
|
# validated_data = serializer.validated_data
|
|
|
|
|
|
-# # DEBUG: Print what we received
|
|
|
-# print("\n" + "="*80)
|
|
|
-# print("BATCH REQUEST - RECEIVED DATA")
|
|
|
-# print("="*80)
|
|
|
-# print(f"Raw request data keys: {request.data.keys()}")
|
|
|
-# print(f"Multiple field in request: {request.data.get('multiple')}")
|
|
|
-# print(f"Validated multiple field: {validated_data.get('multiple')}")
|
|
|
-# print("="*80 + "\n")
|
|
|
-
|
|
|
# # Get batch-level settings
|
|
|
# product_list = validated_data.get("products", [])
|
|
|
# model = validated_data.get("model")
|
|
|
@@ -168,27 +162,33 @@ class ExtractProductAttributesView(APIView):
|
|
|
# use_adaptive_margin = validated_data.get("use_adaptive_margin", True)
|
|
|
# use_semantic_clustering = validated_data.get("use_semantic_clustering", True)
|
|
|
|
|
|
-# # DEBUG: Print extracted settings
|
|
|
-# print(f"Extracted multiple parameter: {multiple}")
|
|
|
-# print(f"Type: {type(multiple)}")
|
|
|
-
|
|
|
# # Extract all item_ids to query the database efficiently
|
|
|
# item_ids = [p['item_id'] for p in product_list]
|
|
|
|
|
|
# # Fetch all products in one query
|
|
|
# products_queryset = Product.objects.filter(item_id__in=item_ids)
|
|
|
-
|
|
|
-# # Create a dictionary for easy lookup: item_id -> Product object
|
|
|
# product_map = {product.item_id: product for product in products_queryset}
|
|
|
# found_ids = set(product_map.keys())
|
|
|
|
|
|
+# # Fetch all original attribute values for these products in one query
|
|
|
+# original_values_qs = ProductAttributeValue.objects.filter(
|
|
|
+# product__item_id__in=item_ids
|
|
|
+# ).select_related('product')
|
|
|
+
|
|
|
+# # Create a nested dictionary: {item_id: {attribute_name: original_value}}
|
|
|
+# original_values_map = {}
|
|
|
+# for attr_val in original_values_qs:
|
|
|
+# item_id = attr_val.product.item_id
|
|
|
+# if item_id not in original_values_map:
|
|
|
+# original_values_map[item_id] = {}
|
|
|
+# original_values_map[item_id][attr_val.attribute_name] = attr_val.original_value
|
|
|
+
|
|
|
# results = []
|
|
|
# successful = 0
|
|
|
# failed = 0
|
|
|
|
|
|
# for product_entry in product_list:
|
|
|
# item_id = product_entry['item_id']
|
|
|
-# # Get item-specific mandatory attributes
|
|
|
# mandatory_attrs = product_entry['mandatory_attrs']
|
|
|
|
|
|
# if item_id not in found_ids:
|
|
|
@@ -206,7 +206,7 @@ class ExtractProductAttributesView(APIView):
|
|
|
# short_desc = product.product_short_description
|
|
|
# long_desc = product.product_long_description
|
|
|
# image_url = product.image_path
|
|
|
-# # image_url = "https://images.unsplash.com/photo-1595777457583-95e059d581b8"
|
|
|
+
|
|
|
# ocr_results = None
|
|
|
# ocr_text = None
|
|
|
# visual_results = None
|
|
|
@@ -216,7 +216,6 @@ class ExtractProductAttributesView(APIView):
|
|
|
# # OCR Processing
|
|
|
# ocr_service = OCRService()
|
|
|
# ocr_results = ocr_service.process_image(image_url)
|
|
|
-# print(f"OCR results for {item_id}: {ocr_results}")
|
|
|
|
|
|
# if ocr_results and ocr_results.get("detected_text"):
|
|
|
# ocr_attrs = ProductAttributeService.extract_attributes_from_ocr(
|
|
|
@@ -232,7 +231,6 @@ class ExtractProductAttributesView(APIView):
|
|
|
# visual_service = VisualProcessingService()
|
|
|
# product_type_hint = product.product_type if hasattr(product, 'product_type') else None
|
|
|
# visual_results = visual_service.process_image(image_url, product_type_hint)
|
|
|
-# print(f"Visual results for {item_id}: {visual_results.get('visual_attributes', {})}")
|
|
|
|
|
|
# # Format visual attributes to array format with source tracking
|
|
|
# if visual_results and visual_results.get('visual_attributes'):
|
|
|
@@ -248,10 +246,6 @@ class ExtractProductAttributesView(APIView):
|
|
|
# ocr_text=ocr_text
|
|
|
# )
|
|
|
|
|
|
-# # DEBUG: Print before extraction
|
|
|
-# print(f"\n>>> Extracting for product {item_id}")
|
|
|
-# print(f" Passing multiple: {multiple}")
|
|
|
-
|
|
|
# # Attribute Extraction with source tracking (returns array format)
|
|
|
# extracted = ProductAttributeService.extract_attributes(
|
|
|
# product_text=product_text,
|
|
|
@@ -267,6 +261,25 @@ class ExtractProductAttributesView(APIView):
|
|
|
# use_semantic_clustering=use_semantic_clustering
|
|
|
# )
|
|
|
|
|
|
+# # Add original_value to each extracted attribute
|
|
|
+# original_attrs = original_values_map.get(item_id, {})
|
|
|
+
|
|
|
+# # Process mandatory attributes
|
|
|
+# for attr_name, attr_values in extracted.get("mandatory", {}).items():
|
|
|
+# if isinstance(attr_values, list):
|
|
|
+# for attr_obj in attr_values:
|
|
|
+# if isinstance(attr_obj, dict):
|
|
|
+# # Add original_value if it exists
|
|
|
+# attr_obj["original_value"] = original_attrs.get(attr_name, "")
|
|
|
+
|
|
|
+# # Process additional attributes
|
|
|
+# for attr_name, attr_values in extracted.get("additional", {}).items():
|
|
|
+# if isinstance(attr_values, list):
|
|
|
+# for attr_obj in attr_values:
|
|
|
+# if isinstance(attr_obj, dict):
|
|
|
+# # Add original_value if it exists
|
|
|
+# attr_obj["original_value"] = original_attrs.get(attr_name, "")
|
|
|
+
|
|
|
# result = {
|
|
|
# "product_id": product.item_id,
|
|
|
# "mandatory": extracted.get("mandatory", {}),
|
|
|
@@ -305,51 +318,244 @@ class ExtractProductAttributesView(APIView):
|
|
|
# return Response(batch_result, status=status.HTTP_200_OK)
|
|
|
|
|
|
|
|
|
+# views.py - OPTIMIZED WITHOUT REDIS/CELERY
|
|
|
|
|
|
+# class BatchExtractProductAttributesView(APIView):
|
|
|
+# """
|
|
|
+# Optimized batch extraction using ThreadPoolExecutor (built-in Python)
|
|
|
+# """
|
|
|
|
|
|
-# Replace the BatchExtractProductAttributesView in your views.py with this updated version
|
|
|
+# def post(self, request):
|
|
|
+# serializer = BatchProductRequestSerializer(data=request.data)
|
|
|
+# if not serializer.is_valid():
|
|
|
+# return Response({"error": serializer.errors}, status=status.HTTP_400_BAD_REQUEST)
|
|
|
+
|
|
|
+# validated_data = serializer.validated_data
|
|
|
+# product_list = validated_data.get("products", [])
|
|
|
+
|
|
|
+# # OPTIMIZATION 1: Single optimized database query
|
|
|
+# item_ids = [p['item_id'] for p in product_list]
|
|
|
+# products_queryset = Product.objects.filter(
|
|
|
+# item_id__in=item_ids
|
|
|
+# ).prefetch_related('attribute_values') # Single query!
|
|
|
+
|
|
|
+# product_map = {product.item_id: product for product in products_queryset}
|
|
|
+
|
|
|
+# # OPTIMIZATION 2: Prefetch ALL original attribute values in ONE query
|
|
|
+# original_values_qs = ProductAttributeValue.objects.filter(
|
|
|
+# product__item_id__in=item_ids
|
|
|
+# ).select_related('product')
|
|
|
+
|
|
|
+# original_values_map = {}
|
|
|
+# for attr_val in original_values_qs:
|
|
|
+# item_id = attr_val.product.item_id
|
|
|
+# if item_id not in original_values_map:
|
|
|
+# original_values_map[item_id] = {}
|
|
|
+# original_values_map[item_id][attr_val.attribute_name] = attr_val.original_value
|
|
|
+
|
|
|
+# # Extract settings
|
|
|
+# model = validated_data.get("model")
|
|
|
+# extract_additional = validated_data.get("extract_additional", True)
|
|
|
+# process_image = validated_data.get("process_image", True)
|
|
|
+# multiple = validated_data.get("multiple", [])
|
|
|
+# threshold_abs = validated_data.get("threshold_abs", 0.65)
|
|
|
+# margin = validated_data.get("margin", 0.15)
|
|
|
+# use_dynamic_thresholds = validated_data.get("use_dynamic_thresholds", True)
|
|
|
+# use_adaptive_margin = validated_data.get("use_adaptive_margin", True)
|
|
|
+# use_semantic_clustering = validated_data.get("use_semantic_clustering", True)
|
|
|
+
|
|
|
+# results = []
|
|
|
+# successful = 0
|
|
|
+# failed = 0
|
|
|
+
|
|
|
+# # OPTIMIZATION 3: Initialize services once
|
|
|
+# ocr_service = OCRService() if process_image else None
|
|
|
+# visual_service = VisualProcessingService() if process_image else None
|
|
|
+
|
|
|
+# # OPTIMIZATION 4: Process in parallel using ThreadPoolExecutor
|
|
|
+# def process_single_product(product_entry):
|
|
|
+# """Process a single product (runs in parallel)"""
|
|
|
+# item_id = product_entry['item_id']
|
|
|
+# mandatory_attrs = product_entry['mandatory_attrs']
|
|
|
+
|
|
|
+# if item_id not in product_map:
|
|
|
+# return {
|
|
|
+# "product_id": item_id,
|
|
|
+# "error": "Product not found in database"
|
|
|
+# }, False
|
|
|
+
|
|
|
+# product = product_map[item_id]
|
|
|
+
|
|
|
+# try:
|
|
|
+# title = product.product_name
|
|
|
+# short_desc = product.product_short_description
|
|
|
+# long_desc = product.product_long_description
|
|
|
+# image_url = product.image_path
|
|
|
+
|
|
|
+# ocr_results = None
|
|
|
+# ocr_text = None
|
|
|
+# visual_results = None
|
|
|
+
|
|
|
+# # Image processing (if enabled)
|
|
|
+# if process_image and image_url:
|
|
|
+# if ocr_service:
|
|
|
+# ocr_results = ocr_service.process_image(image_url)
|
|
|
+
|
|
|
+# if ocr_results and ocr_results.get("detected_text"):
|
|
|
+# ocr_attrs = ProductAttributeService.extract_attributes_from_ocr(
|
|
|
+# ocr_results, model
|
|
|
+# )
|
|
|
+# ocr_results["extracted_attributes"] = ocr_attrs
|
|
|
+# ocr_text = "\n".join([
|
|
|
+# f"{item['text']} (confidence: {item['confidence']:.2f})"
|
|
|
+# for item in ocr_results["detected_text"]
|
|
|
+# ])
|
|
|
+
|
|
|
+# if visual_service:
|
|
|
+# product_type_hint = product.product_type if hasattr(product, 'product_type') else None
|
|
|
+# visual_results = visual_service.process_image(image_url, product_type_hint)
|
|
|
+
|
|
|
+# if visual_results and visual_results.get('visual_attributes'):
|
|
|
+# visual_results['visual_attributes'] = ProductAttributeService.format_visual_attributes(
|
|
|
+# visual_results['visual_attributes']
|
|
|
+# )
|
|
|
+
|
|
|
+# # Combine product text with source tracking
|
|
|
+# product_text, source_map = ProductAttributeService.combine_product_text(
|
|
|
+# title=title,
|
|
|
+# short_desc=short_desc,
|
|
|
+# long_desc=long_desc,
|
|
|
+# ocr_text=ocr_text
|
|
|
+# )
|
|
|
+
|
|
|
+# # Extract attributes (WITH CACHING ENABLED)
|
|
|
+# extracted = ProductAttributeService.extract_attributes(
|
|
|
+# product_text=product_text,
|
|
|
+# mandatory_attrs=mandatory_attrs,
|
|
|
+# source_map=source_map,
|
|
|
+# model=model,
|
|
|
+# extract_additional=extract_additional,
|
|
|
+# multiple=multiple,
|
|
|
+# threshold_abs=threshold_abs,
|
|
|
+# margin=margin,
|
|
|
+# use_dynamic_thresholds=use_dynamic_thresholds,
|
|
|
+# use_adaptive_margin=use_adaptive_margin,
|
|
|
+# use_semantic_clustering=use_semantic_clustering,
|
|
|
+# use_cache=True # Enable caching!
|
|
|
+# )
|
|
|
+
|
|
|
+# # Add original values
|
|
|
+# original_attrs = original_values_map.get(item_id, {})
|
|
|
+
|
|
|
+# for attr_name, attr_values in extracted.get("mandatory", {}).items():
|
|
|
+# if isinstance(attr_values, list):
|
|
|
+# for attr_obj in attr_values:
|
|
|
+# if isinstance(attr_obj, dict):
|
|
|
+# attr_obj["original_value"] = original_attrs.get(attr_name, "")
|
|
|
+
|
|
|
+# for attr_name, attr_values in extracted.get("additional", {}).items():
|
|
|
+# if isinstance(attr_values, list):
|
|
|
+# for attr_obj in attr_values:
|
|
|
+# if isinstance(attr_obj, dict):
|
|
|
+# attr_obj["original_value"] = original_attrs.get(attr_name, "")
|
|
|
+
|
|
|
+# result = {
|
|
|
+# "product_id": product.item_id,
|
|
|
+# "mandatory": extracted.get("mandatory", {}),
|
|
|
+# "additional": extracted.get("additional", {}),
|
|
|
+# }
|
|
|
+
|
|
|
+# if ocr_results:
|
|
|
+# result["ocr_results"] = ocr_results
|
|
|
+
|
|
|
+# if visual_results:
|
|
|
+# result["visual_results"] = visual_results
|
|
|
+
|
|
|
+# return result, True
|
|
|
+
|
|
|
+# except Exception as e:
|
|
|
+# return {
|
|
|
+# "product_id": item_id,
|
|
|
+# "error": str(e)
|
|
|
+# }, False
|
|
|
+
|
|
|
+# # OPTIMIZATION 5: Use ThreadPoolExecutor for parallel processing
|
|
|
+# import concurrent.futures
|
|
|
+# max_workers = min(10, len(product_list)) # Up to 10 parallel workers
|
|
|
+
|
|
|
+# with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
|
|
|
+# # Submit all tasks
|
|
|
+# future_to_product = {
|
|
|
+# executor.submit(process_single_product, product): product
|
|
|
+# for product in product_list
|
|
|
+# }
|
|
|
+
|
|
|
+# # Collect results as they complete
|
|
|
+# for future in concurrent.futures.as_completed(future_to_product):
|
|
|
+# try:
|
|
|
+# result, success = future.result()
|
|
|
+# results.append(result)
|
|
|
+# if success:
|
|
|
+# successful += 1
|
|
|
+# else:
|
|
|
+# failed += 1
|
|
|
+# except Exception as e:
|
|
|
+# failed += 1
|
|
|
+# logger.error(f"Unexpected error: {str(e)}")
|
|
|
+# results.append({
|
|
|
+# "product_id": "unknown",
|
|
|
+# "error": str(e)
|
|
|
+# })
|
|
|
+
|
|
|
+# batch_result = {
|
|
|
+# "results": results,
|
|
|
+# "total_products": len(product_list),
|
|
|
+# "successful": successful,
|
|
|
+# "failed": failed
|
|
|
+# }
|
|
|
+
|
|
|
+# response_serializer = BatchProductResponseSerializer(data=batch_result)
|
|
|
+# if response_serializer.is_valid():
|
|
|
+# return Response(response_serializer.data, status=status.HTTP_200_OK)
|
|
|
+
|
|
|
+# return Response(batch_result, status=status.HTTP_200_OK)
|
|
|
+
|
|
|
+
|
|
|
+# ==================== OPTIMIZED BATCH VIEW ====================
|
|
|
+import concurrent.futures
|
|
|
|
|
|
class BatchExtractProductAttributesView(APIView):
|
|
|
"""
|
|
|
- API endpoint to extract product attributes for multiple products in batch.
|
|
|
- Uses item-specific mandatory_attrs with source tracking.
|
|
|
- Returns attributes in array format with original_value field.
|
|
|
- Includes OCR and Visual Processing results.
|
|
|
+ ⚡ PERFORMANCE OPTIMIZED: Batch extraction with intelligent parallelization
|
|
|
+ Expected performance: 10 products in 30-60 seconds (with image processing)
|
|
|
"""
|
|
|
|
|
|
def post(self, request):
|
|
|
+ import time
|
|
|
+ start_time = time.time()
|
|
|
+
|
|
|
serializer = BatchProductRequestSerializer(data=request.data)
|
|
|
if not serializer.is_valid():
|
|
|
return Response({"error": serializer.errors}, status=status.HTTP_400_BAD_REQUEST)
|
|
|
|
|
|
validated_data = serializer.validated_data
|
|
|
-
|
|
|
- # Get batch-level settings
|
|
|
product_list = validated_data.get("products", [])
|
|
|
- model = validated_data.get("model")
|
|
|
- extract_additional = validated_data.get("extract_additional", True)
|
|
|
- process_image = validated_data.get("process_image", True)
|
|
|
- multiple = validated_data.get("multiple", [])
|
|
|
- threshold_abs = validated_data.get("threshold_abs", 0.65)
|
|
|
- margin = validated_data.get("margin", 0.15)
|
|
|
- use_dynamic_thresholds = validated_data.get("use_dynamic_thresholds", True)
|
|
|
- use_adaptive_margin = validated_data.get("use_adaptive_margin", True)
|
|
|
- use_semantic_clustering = validated_data.get("use_semantic_clustering", True)
|
|
|
|
|
|
- # Extract all item_ids to query the database efficiently
|
|
|
- item_ids = [p['item_id'] for p in product_list]
|
|
|
+ logger.info(f"🚀 Starting batch processing for {len(product_list)} products")
|
|
|
+
|
|
|
+ # ==================== OPTIMIZATION 1: Bulk DB Query ====================
|
|
|
+ item_ids = [p['item_id'] for p in product_list]
|
|
|
+ products_queryset = Product.objects.filter(
|
|
|
+ item_id__in=item_ids
|
|
|
+ ).prefetch_related('attribute_values')
|
|
|
|
|
|
- # Fetch all products in one query
|
|
|
- products_queryset = Product.objects.filter(item_id__in=item_ids)
|
|
|
product_map = {product.item_id: product for product in products_queryset}
|
|
|
- found_ids = set(product_map.keys())
|
|
|
|
|
|
- # Fetch all original attribute values for these products in one query
|
|
|
+ # Prefetch ALL original attribute values in ONE query
|
|
|
original_values_qs = ProductAttributeValue.objects.filter(
|
|
|
product__item_id__in=item_ids
|
|
|
).select_related('product')
|
|
|
|
|
|
- # Create a nested dictionary: {item_id: {attribute_name: original_value}}
|
|
|
original_values_map = {}
|
|
|
for attr_val in original_values_qs:
|
|
|
item_id = attr_val.product.item_id
|
|
|
@@ -357,25 +563,53 @@ class BatchExtractProductAttributesView(APIView):
|
|
|
original_values_map[item_id] = {}
|
|
|
original_values_map[item_id][attr_val.attribute_name] = attr_val.original_value
|
|
|
|
|
|
+ logger.info(f"✓ Loaded {len(product_map)} products from database")
|
|
|
+
|
|
|
+ # Extract settings
|
|
|
+ model = validated_data.get("model")
|
|
|
+ extract_additional = validated_data.get("extract_additional", True)
|
|
|
+ process_image = validated_data.get("process_image", True)
|
|
|
+ multiple = validated_data.get("multiple", [])
|
|
|
+ threshold_abs = validated_data.get("threshold_abs", 0.65)
|
|
|
+ margin = validated_data.get("margin", 0.15)
|
|
|
+ use_dynamic_thresholds = validated_data.get("use_dynamic_thresholds", True)
|
|
|
+ use_adaptive_margin = validated_data.get("use_adaptive_margin", True)
|
|
|
+ use_semantic_clustering = validated_data.get("use_semantic_clustering", True)
|
|
|
+
|
|
|
results = []
|
|
|
successful = 0
|
|
|
failed = 0
|
|
|
+
|
|
|
+ # ==================== OPTIMIZATION 2: Conditional Service Init ====================
|
|
|
+ # Only initialize if processing images
|
|
|
+ ocr_service = None
|
|
|
+ visual_service = None
|
|
|
+
|
|
|
+ if process_image:
|
|
|
+ from .ocr_service import OCRService
|
|
|
+ from .visual_processing_service import VisualProcessingService
|
|
|
+ ocr_service = OCRService()
|
|
|
+ visual_service = VisualProcessingService()
|
|
|
+ logger.info("✓ Image processing services initialized")
|
|
|
|
|
|
- for product_entry in product_list:
|
|
|
+ # ==================== OPTIMIZATION 3: Smart Parallelization ====================
|
|
|
+ def process_single_product(product_entry):
|
|
|
+ """Process a single product (runs in parallel)"""
|
|
|
+ import time
|
|
|
+ product_start = time.time()
|
|
|
+
|
|
|
item_id = product_entry['item_id']
|
|
|
- mandatory_attrs = product_entry['mandatory_attrs']
|
|
|
+ mandatory_attrs = product_entry['mandatory_attrs']
|
|
|
|
|
|
- if item_id not in found_ids:
|
|
|
- failed += 1
|
|
|
- results.append({
|
|
|
+ if item_id not in product_map:
|
|
|
+ return {
|
|
|
"product_id": item_id,
|
|
|
"error": "Product not found in database"
|
|
|
- })
|
|
|
- continue
|
|
|
+ }, False
|
|
|
|
|
|
product = product_map[item_id]
|
|
|
|
|
|
- try:
|
|
|
+ try:
|
|
|
title = product.product_name
|
|
|
short_desc = product.product_short_description
|
|
|
long_desc = product.product_long_description
|
|
|
@@ -385,32 +619,29 @@ class BatchExtractProductAttributesView(APIView):
|
|
|
ocr_text = None
|
|
|
visual_results = None
|
|
|
|
|
|
- # Image Processing Logic
|
|
|
+ # ⚡ SKIP IMAGE PROCESSING IF DISABLED (HUGE TIME SAVER)
|
|
|
if process_image and image_url:
|
|
|
- # OCR Processing
|
|
|
- ocr_service = OCRService()
|
|
|
- ocr_results = ocr_service.process_image(image_url)
|
|
|
-
|
|
|
- if ocr_results and ocr_results.get("detected_text"):
|
|
|
- ocr_attrs = ProductAttributeService.extract_attributes_from_ocr(
|
|
|
- ocr_results, model
|
|
|
- )
|
|
|
- ocr_results["extracted_attributes"] = ocr_attrs
|
|
|
- ocr_text = "\n".join([
|
|
|
- f"{item['text']} (confidence: {item['confidence']:.2f})"
|
|
|
- for item in ocr_results["detected_text"]
|
|
|
- ])
|
|
|
+ if ocr_service:
|
|
|
+ ocr_results = ocr_service.process_image(image_url)
|
|
|
+
|
|
|
+ if ocr_results and ocr_results.get("detected_text"):
|
|
|
+ ocr_attrs = ProductAttributeService.extract_attributes_from_ocr(
|
|
|
+ ocr_results, model
|
|
|
+ )
|
|
|
+ ocr_results["extracted_attributes"] = ocr_attrs
|
|
|
+ ocr_text = "\n".join([
|
|
|
+ f"{item['text']} (confidence: {item['confidence']:.2f})"
|
|
|
+ for item in ocr_results["detected_text"]
|
|
|
+ ])
|
|
|
|
|
|
- # Visual Processing
|
|
|
- visual_service = VisualProcessingService()
|
|
|
- product_type_hint = product.product_type if hasattr(product, 'product_type') else None
|
|
|
- visual_results = visual_service.process_image(image_url, product_type_hint)
|
|
|
-
|
|
|
- # Format visual attributes to array format with source tracking
|
|
|
- if visual_results and visual_results.get('visual_attributes'):
|
|
|
- visual_results['visual_attributes'] = ProductAttributeService.format_visual_attributes(
|
|
|
- visual_results['visual_attributes']
|
|
|
- )
|
|
|
+ if visual_service:
|
|
|
+ product_type_hint = product.product_type if hasattr(product, 'product_type') else None
|
|
|
+ visual_results = visual_service.process_image(image_url, product_type_hint)
|
|
|
+
|
|
|
+ if visual_results and visual_results.get('visual_attributes'):
|
|
|
+ visual_results['visual_attributes'] = ProductAttributeService.format_visual_attributes(
|
|
|
+ visual_results['visual_attributes']
|
|
|
+ )
|
|
|
|
|
|
# Combine product text with source tracking
|
|
|
product_text, source_map = ProductAttributeService.combine_product_text(
|
|
|
@@ -420,7 +651,7 @@ class BatchExtractProductAttributesView(APIView):
|
|
|
ocr_text=ocr_text
|
|
|
)
|
|
|
|
|
|
- # Attribute Extraction with source tracking (returns array format)
|
|
|
+ # ⚡ EXTRACT ATTRIBUTES WITH CACHING ENABLED
|
|
|
extracted = ProductAttributeService.extract_attributes(
|
|
|
product_text=product_text,
|
|
|
mandatory_attrs=mandatory_attrs,
|
|
|
@@ -432,26 +663,23 @@ class BatchExtractProductAttributesView(APIView):
|
|
|
margin=margin,
|
|
|
use_dynamic_thresholds=use_dynamic_thresholds,
|
|
|
use_adaptive_margin=use_adaptive_margin,
|
|
|
- use_semantic_clustering=use_semantic_clustering
|
|
|
+ use_semantic_clustering=use_semantic_clustering,
|
|
|
+ use_cache=True # ⚡ CRITICAL: Enable caching
|
|
|
)
|
|
|
|
|
|
- # Add original_value to each extracted attribute
|
|
|
+ # Add original values
|
|
|
original_attrs = original_values_map.get(item_id, {})
|
|
|
|
|
|
- # Process mandatory attributes
|
|
|
for attr_name, attr_values in extracted.get("mandatory", {}).items():
|
|
|
if isinstance(attr_values, list):
|
|
|
for attr_obj in attr_values:
|
|
|
if isinstance(attr_obj, dict):
|
|
|
- # Add original_value if it exists
|
|
|
attr_obj["original_value"] = original_attrs.get(attr_name, "")
|
|
|
|
|
|
- # Process additional attributes
|
|
|
for attr_name, attr_values in extracted.get("additional", {}).items():
|
|
|
if isinstance(attr_values, list):
|
|
|
for attr_obj in attr_values:
|
|
|
if isinstance(attr_obj, dict):
|
|
|
- # Add original_value if it exists
|
|
|
attr_obj["original_value"] = original_attrs.get(attr_name, "")
|
|
|
|
|
|
result = {
|
|
|
@@ -460,29 +688,82 @@ class BatchExtractProductAttributesView(APIView):
|
|
|
"additional": extracted.get("additional", {}),
|
|
|
}
|
|
|
|
|
|
- # Attach OCR results if available
|
|
|
if ocr_results:
|
|
|
result["ocr_results"] = ocr_results
|
|
|
|
|
|
- # Attach Visual Processing results if available
|
|
|
if visual_results:
|
|
|
result["visual_results"] = visual_results
|
|
|
+
|
|
|
+ processing_time = time.time() - product_start
|
|
|
+ logger.info(f"✓ Processed {item_id} in {processing_time:.2f}s")
|
|
|
|
|
|
- results.append(result)
|
|
|
- successful += 1
|
|
|
+ return result, True
|
|
|
|
|
|
except Exception as e:
|
|
|
- failed += 1
|
|
|
- results.append({
|
|
|
+ logger.error(f"❌ Error processing {item_id}: {str(e)}")
|
|
|
+ return {
|
|
|
"product_id": item_id,
|
|
|
"error": str(e)
|
|
|
- })
|
|
|
+ }, False
|
|
|
+
|
|
|
+ # ==================== OPTIMIZATION 4: Parallel Execution ====================
|
|
|
+ # Adjust workers based on whether image processing is enabled
|
|
|
+ max_workers = min(3 if process_image else 10, len(product_list))
|
|
|
+
|
|
|
+ logger.info(f"⚡ Using {max_workers} parallel workers")
|
|
|
+
|
|
|
+ with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
|
|
|
+ # Submit all tasks
|
|
|
+ future_to_product = {
|
|
|
+ executor.submit(process_single_product, product): product
|
|
|
+ for product in product_list
|
|
|
+ }
|
|
|
+
|
|
|
+ # Collect results as they complete
|
|
|
+ for future in concurrent.futures.as_completed(future_to_product):
|
|
|
+ try:
|
|
|
+ result, success = future.result()
|
|
|
+ results.append(result)
|
|
|
+ if success:
|
|
|
+ successful += 1
|
|
|
+ else:
|
|
|
+ failed += 1
|
|
|
+ except Exception as e:
|
|
|
+ failed += 1
|
|
|
+ logger.error(f"❌ Future execution error: {str(e)}")
|
|
|
+ results.append({
|
|
|
+ "product_id": "unknown",
|
|
|
+ "error": str(e)
|
|
|
+ })
|
|
|
+
|
|
|
+ total_time = time.time() - start_time
|
|
|
+
|
|
|
+ # Get cache statistics
|
|
|
+ cache_stats = ProductAttributeService.get_cache_stats()
|
|
|
+
|
|
|
+ logger.info(f"""
|
|
|
+🎉 BATCH PROCESSING COMPLETE
|
|
|
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
|
|
+ Total products: {len(product_list)}
|
|
|
+ Successful: {successful}
|
|
|
+ Failed: {failed}
|
|
|
+ Total time: {total_time:.2f}s
|
|
|
+ Avg time/product: {total_time/len(product_list):.2f}s
|
|
|
+ Cache hit rate: {cache_stats['embedding_cache']['hit_rate_percent']:.1f}%
|
|
|
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
|
|
+ """)
|
|
|
|
|
|
batch_result = {
|
|
|
"results": results,
|
|
|
"total_products": len(product_list),
|
|
|
"successful": successful,
|
|
|
- "failed": failed
|
|
|
+ "failed": failed,
|
|
|
+ "performance": {
|
|
|
+ "total_time_seconds": round(total_time, 2),
|
|
|
+ "avg_time_per_product": round(total_time / len(product_list), 2),
|
|
|
+ "workers_used": max_workers
|
|
|
+ },
|
|
|
+ "cache_stats": cache_stats
|
|
|
}
|
|
|
|
|
|
response_serializer = BatchProductResponseSerializer(data=batch_result)
|
|
|
@@ -493,7 +774,6 @@ class BatchExtractProductAttributesView(APIView):
|
|
|
|
|
|
|
|
|
|
|
|
-
|
|
|
class ProductListView(APIView):
|
|
|
"""
|
|
|
GET API to list all products with details
|
|
|
@@ -512,77 +792,6 @@ import pandas as pd
|
|
|
from .models import Product
|
|
|
|
|
|
|
|
|
-# class ProductUploadExcelView(APIView):
|
|
|
-# """
|
|
|
-# POST API to upload an Excel file and add/update data in Product model.
|
|
|
-# - Creates new records if they don't exist.
|
|
|
-# - Updates existing ones (e.g., when image_path or other fields change).
|
|
|
-# """
|
|
|
-# parser_classes = (MultiPartParser, FormParser)
|
|
|
-
|
|
|
-# def post(self, request, *args, **kwargs):
|
|
|
-# file_obj = request.FILES.get('file')
|
|
|
-# if not file_obj:
|
|
|
-# return Response({'error': 'No file provided'}, status=status.HTTP_400_BAD_REQUEST)
|
|
|
-
|
|
|
-# try:
|
|
|
-# # Read Excel into DataFrame
|
|
|
-# df = pd.read_excel(file_obj)
|
|
|
-# df.columns = [c.strip().lower().replace(' ', '_') for c in df.columns]
|
|
|
-
|
|
|
-# expected_cols = {
|
|
|
-# 'item_id',
|
|
|
-# 'product_name',
|
|
|
-# 'product_long_description',
|
|
|
-# 'product_short_description',
|
|
|
-# 'product_type',
|
|
|
-# 'image_path'
|
|
|
-# }
|
|
|
-
|
|
|
-# # Check required columns
|
|
|
-# if not expected_cols.issubset(df.columns):
|
|
|
-# return Response({
|
|
|
-# 'error': 'Missing required columns',
|
|
|
-# 'required_columns': list(expected_cols)
|
|
|
-# }, status=status.HTTP_400_BAD_REQUEST)
|
|
|
-
|
|
|
-# created_count = 0
|
|
|
-# updated_count = 0
|
|
|
-
|
|
|
-# # Loop through rows and update or create
|
|
|
-# for _, row in df.iterrows():
|
|
|
-# item_id = str(row.get('item_id', '')).strip()
|
|
|
-# if not item_id:
|
|
|
-# continue # Skip rows without an item_id
|
|
|
-
|
|
|
-# defaults = {
|
|
|
-# 'product_name': row.get('product_name', ''),
|
|
|
-# 'product_long_description': row.get('product_long_description', ''),
|
|
|
-# 'product_short_description': row.get('product_short_description', ''),
|
|
|
-# 'product_type': row.get('product_type', ''),
|
|
|
-# 'image_path': row.get('image_path', ''),
|
|
|
-# }
|
|
|
-
|
|
|
-# obj, created = Product.objects.update_or_create(
|
|
|
-# item_id=item_id,
|
|
|
-# defaults=defaults
|
|
|
-# )
|
|
|
-
|
|
|
-# if created:
|
|
|
-# created_count += 1
|
|
|
-# else:
|
|
|
-# updated_count += 1
|
|
|
-
|
|
|
-# return Response({
|
|
|
-# 'message': f'Upload successful.',
|
|
|
-# 'created': f'{created_count} new records added.',
|
|
|
-# 'updated': f'{updated_count} existing records updated.'
|
|
|
-# }, status=status.HTTP_201_CREATED)
|
|
|
-
|
|
|
-# except Exception as e:
|
|
|
-# return Response({'error': str(e)}, status=status.HTTP_500_INTERNAL_SERVER_ERROR)
|
|
|
-# Replace the ProductUploadExcelView in your views.py with this updated version
|
|
|
-
|
|
|
from rest_framework.views import APIView
|
|
|
from rest_framework.response import Response
|
|
|
from rest_framework import status
|
|
|
@@ -592,835 +801,12 @@ import pandas as pd
|
|
|
from .models import Product, ProductAttributeValue
|
|
|
|
|
|
|
|
|
-# class ProductUploadExcelView(APIView):
|
|
|
-# """
|
|
|
-# POST API to upload an Excel file with two sheets:
|
|
|
-# 1. 'Products' sheet - Product details
|
|
|
-# 2. 'Attribute_values' sheet - Original attribute values
|
|
|
-
|
|
|
-# Creates/updates both products and their attribute values in a single transaction.
|
|
|
-# """
|
|
|
-# parser_classes = (MultiPartParser, FormParser)
|
|
|
|
|
|
-# def post(self, request, *args, **kwargs):
|
|
|
-# file_obj = request.FILES.get('file')
|
|
|
-# if not file_obj:
|
|
|
-# return Response({'error': 'No file provided'}, status=status.HTTP_400_BAD_REQUEST)
|
|
|
|
|
|
-# try:
|
|
|
-# # Read all sheets from Excel file
|
|
|
-# excel_file = pd.ExcelFile(file_obj)
|
|
|
-
|
|
|
-# # Check if required sheets exist
|
|
|
-# if 'Products' not in excel_file.sheet_names:
|
|
|
-# return Response({
|
|
|
-# 'error': "Missing 'Products' sheet",
|
|
|
-# 'available_sheets': excel_file.sheet_names
|
|
|
-# }, status=status.HTTP_400_BAD_REQUEST)
|
|
|
-
|
|
|
-# # Read Products sheet
|
|
|
-# df_products = pd.read_excel(excel_file, sheet_name='Products')
|
|
|
-# df_products.columns = [c.strip().lower().replace(' ', '_') for c in df_products.columns]
|
|
|
-
|
|
|
-# # Check required columns for Products
|
|
|
-# expected_product_cols = {
|
|
|
-# 'item_id',
|
|
|
-# 'product_name',
|
|
|
-# 'product_long_description',
|
|
|
-# 'product_short_description',
|
|
|
-# 'product_type',
|
|
|
-# 'image_path'
|
|
|
-# }
|
|
|
|
|
|
-# if not expected_product_cols.issubset(df_products.columns):
|
|
|
-# return Response({
|
|
|
-# 'error': 'Missing required columns in Products sheet',
|
|
|
-# 'required_columns': list(expected_product_cols),
|
|
|
-# 'found_columns': list(df_products.columns)
|
|
|
-# }, status=status.HTTP_400_BAD_REQUEST)
|
|
|
|
|
|
-# # Read Attribute_values sheet if it exists
|
|
|
-# df_attributes = None
|
|
|
-# has_attributes_sheet = 'Attribute_values' in excel_file.sheet_names
|
|
|
-
|
|
|
-# if has_attributes_sheet:
|
|
|
-# df_attributes = pd.read_excel(excel_file, sheet_name='Attribute_values')
|
|
|
-# df_attributes.columns = [c.strip().lower().replace(' ', '_') for c in df_attributes.columns]
|
|
|
-
|
|
|
-# # Check required columns for Attribute_values
|
|
|
-# expected_attr_cols = {'item_id', 'attribute_name', 'original_value'}
|
|
|
-# if not expected_attr_cols.issubset(df_attributes.columns):
|
|
|
-# return Response({
|
|
|
-# 'error': 'Missing required columns in Attribute_values sheet',
|
|
|
-# 'required_columns': list(expected_attr_cols),
|
|
|
-# 'found_columns': list(df_attributes.columns)
|
|
|
-# }, status=status.HTTP_400_BAD_REQUEST)
|
|
|
-
|
|
|
-# # Initialize counters
|
|
|
-# products_created = 0
|
|
|
-# products_updated = 0
|
|
|
-# attributes_created = 0
|
|
|
-# attributes_updated = 0
|
|
|
-# products_failed = 0
|
|
|
-# attributes_failed = 0
|
|
|
-# errors = []
|
|
|
-
|
|
|
-# # Use transaction to ensure atomicity
|
|
|
-# with transaction.atomic():
|
|
|
-# # Process Products sheet
|
|
|
-# for idx, row in df_products.iterrows():
|
|
|
-# item_id = str(row.get('item_id', '')).strip()
|
|
|
-# if not item_id:
|
|
|
-# products_failed += 1
|
|
|
-# errors.append(f"Products Row {idx + 2}: Missing item_id")
|
|
|
-# continue
|
|
|
-
|
|
|
-# try:
|
|
|
-# defaults = {
|
|
|
-# 'product_name': str(row.get('product_name', '')),
|
|
|
-# 'product_long_description': str(row.get('product_long_description', '')),
|
|
|
-# 'product_short_description': str(row.get('product_short_description', '')),
|
|
|
-# 'product_type': str(row.get('product_type', '')),
|
|
|
-# 'image_path': str(row.get('image_path', '')),
|
|
|
-# }
|
|
|
-
|
|
|
-# obj, created = Product.objects.update_or_create(
|
|
|
-# item_id=item_id,
|
|
|
-# defaults=defaults
|
|
|
-# )
|
|
|
|
|
|
-# if created:
|
|
|
-# products_created += 1
|
|
|
-# else:
|
|
|
-# products_updated += 1
|
|
|
-# except Exception as e:
|
|
|
-# products_failed += 1
|
|
|
-# errors.append(f"Products Row {idx + 2} (item_id: {item_id}): {str(e)}")
|
|
|
-
|
|
|
-# # Process Attribute_values sheet if it exists
|
|
|
-# if has_attributes_sheet and df_attributes is not None:
|
|
|
-# # Group by item_id to optimize lookups
|
|
|
-# item_ids_in_attrs = df_attributes['item_id'].unique()
|
|
|
-
|
|
|
-# # Fetch all products at once
|
|
|
-# existing_products = {
|
|
|
-# p.item_id: p
|
|
|
-# for p in Product.objects.filter(item_id__in=item_ids_in_attrs)
|
|
|
-# }
|
|
|
-
|
|
|
-# for idx, row in df_attributes.iterrows():
|
|
|
-# item_id = str(row.get('item_id', '')).strip()
|
|
|
-# attribute_name = str(row.get('attribute_name', '')).strip()
|
|
|
-# original_value = str(row.get('original_value', '')).strip()
|
|
|
-
|
|
|
-# if not item_id or not attribute_name:
|
|
|
-# attributes_failed += 1
|
|
|
-# errors.append(
|
|
|
-# f"Attribute_values Row {idx + 2}: Missing item_id or attribute_name"
|
|
|
-# )
|
|
|
-# continue
|
|
|
-
|
|
|
-# # Check if product exists
|
|
|
-# product = existing_products.get(item_id)
|
|
|
-# if not product:
|
|
|
-# attributes_failed += 1
|
|
|
-# errors.append(
|
|
|
-# f"Attribute_values Row {idx + 2}: Product with item_id '{item_id}' not found. "
|
|
|
-# "Make sure it exists in Products sheet."
|
|
|
-# )
|
|
|
-# continue
|
|
|
-
|
|
|
-# try:
|
|
|
-# attr_obj, created = ProductAttributeValue.objects.update_or_create(
|
|
|
-# product=product,
|
|
|
-# attribute_name=attribute_name,
|
|
|
-# defaults={'original_value': original_value}
|
|
|
-# )
|
|
|
-
|
|
|
-# if created:
|
|
|
-# attributes_created += 1
|
|
|
-# else:
|
|
|
-# attributes_updated += 1
|
|
|
-# except Exception as e:
|
|
|
-# attributes_failed += 1
|
|
|
-# errors.append(
|
|
|
-# f"Attribute_values Row {idx + 2} "
|
|
|
-# f"(item_id: {item_id}, attribute: {attribute_name}): {str(e)}"
|
|
|
-# )
|
|
|
-
|
|
|
-# # Prepare response
|
|
|
-# response_data = {
|
|
|
-# 'message': 'Upload completed successfully',
|
|
|
-# 'products': {
|
|
|
-# 'created': products_created,
|
|
|
-# 'updated': products_updated,
|
|
|
-# 'failed': products_failed,
|
|
|
-# 'total_processed': products_created + products_updated + products_failed
|
|
|
-# }
|
|
|
-# }
|
|
|
-
|
|
|
-# if has_attributes_sheet:
|
|
|
-# response_data['attribute_values'] = {
|
|
|
-# 'created': attributes_created,
|
|
|
-# 'updated': attributes_updated,
|
|
|
-# 'failed': attributes_failed,
|
|
|
-# 'total_processed': attributes_created + attributes_updated + attributes_failed
|
|
|
-# }
|
|
|
-# else:
|
|
|
-# response_data['attribute_values'] = {
|
|
|
-# 'message': 'Attribute_values sheet not found in Excel file'
|
|
|
-# }
|
|
|
|
|
|
-# if errors:
|
|
|
-# response_data['errors'] = errors[:50] # Limit to first 50 errors
|
|
|
-# if len(errors) > 50:
|
|
|
-# response_data['errors'].append(f"... and {len(errors) - 50} more errors")
|
|
|
-
|
|
|
-# # Determine status code
|
|
|
-# if products_failed > 0 or attributes_failed > 0:
|
|
|
-# status_code = status.HTTP_207_MULTI_STATUS
|
|
|
-# else:
|
|
|
-# status_code = status.HTTP_201_CREATED
|
|
|
-
|
|
|
-# return Response(response_data, status=status_code)
|
|
|
-
|
|
|
-# except pd.errors.EmptyDataError:
|
|
|
-# return Response({
|
|
|
-# 'error': 'The uploaded Excel file is empty or invalid'
|
|
|
-# }, status=status.HTTP_400_BAD_REQUEST)
|
|
|
-# except Exception as e:
|
|
|
-# return Response({
|
|
|
-# 'error': f'An error occurred while processing the file: {str(e)}'
|
|
|
-# }, status=status.HTTP_500_INTERNAL_SERVER_ERROR)
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
-# import logging
|
|
|
-# import json
|
|
|
-# from rest_framework.views import APIView
|
|
|
-# from rest_framework.response import Response
|
|
|
-# from rest_framework import status
|
|
|
-# from rest_framework.parsers import MultiPartParser, FormParser
|
|
|
-# from django.db import transaction
|
|
|
-# from django.db.models import Prefetch, F
|
|
|
-# import pandas as pd
|
|
|
-# # Import ALL your models
|
|
|
-# from .models import Product, ProductAttributeValue, ProductType, ProductAttribute, AttributePossibleValue
|
|
|
-# from .services import ProductAttributeService
|
|
|
-# from .ocr_service import OCRService
|
|
|
-# from .visual_processing_service import VisualProcessingService
|
|
|
-# from openpyxl import Workbook
|
|
|
-# from openpyxl.styles import Font, PatternFill, Alignment
|
|
|
-# from django.conf import settings
|
|
|
-# import os
|
|
|
-# import threading
|
|
|
-# from datetime import datetime
|
|
|
-
|
|
|
-# # --- Logging Setup ---
|
|
|
-# # Define log and status file paths in MEDIA_ROOT
|
|
|
-# LOG_FILE_PATH = os.path.join(settings.MEDIA_ROOT, 'excel_generation.log')
|
|
|
-# STATUS_FILE_PATH = os.path.join(settings.MEDIA_ROOT, 'excel_generation_status.json')
|
|
|
-
|
|
|
-# # Ensure the MEDIA_ROOT exists for files to be saved
|
|
|
-# if not os.path.exists(settings.MEDIA_ROOT):
|
|
|
-# os.makedirs(settings.MEDIA_ROOT)
|
|
|
-
|
|
|
-# # Configure basic logging
|
|
|
-# logging.basicConfig(
|
|
|
-# filename=LOG_FILE_PATH,
|
|
|
-# level=logging.INFO,
|
|
|
-# format='%(asctime)s - %(levelname)s - %(message)s'
|
|
|
-# )
|
|
|
-# logger = logging.getLogger(__name__)
|
|
|
-
|
|
|
-# # -------------------------------------------------------------------------------------------------
|
|
|
-
|
|
|
-# def generate_product_excel_background():
|
|
|
-# """
|
|
|
-# Function to perform batch attribute extraction for all products and generate an Excel file.
|
|
|
-# Runs in a background thread to avoid blocking the API response.
|
|
|
-# Logs success/failure and saves a status file for external monitoring.
|
|
|
-# """
|
|
|
-# logger.info(f"[{datetime.now().isoformat()}] Starting background product Excel generation and attribute extraction.")
|
|
|
-
|
|
|
-# successful = 0
|
|
|
-# failed = 0
|
|
|
-# results = [] # To store detailed extraction results for Excel sheet 2
|
|
|
-
|
|
|
-# # Function to write status file (SUCCESS/FAILED)
|
|
|
-# def write_status(status_type, error_msg=None):
|
|
|
-# status_data = {
|
|
|
-# "status": status_type,
|
|
|
-# "timestamp": datetime.now().isoformat(),
|
|
|
-# "products_processed": successful + failed,
|
|
|
-# "products_successful": successful,
|
|
|
-# "products_failed": failed,
|
|
|
-# "excel_path": os.path.join(settings.MEDIA_URL, 'generated_products.xlsx') if status_type == "SUCCESS" else None,
|
|
|
-# "log_path": os.path.join(settings.MEDIA_URL, 'excel_generation.log'),
|
|
|
-# "error_message": error_msg
|
|
|
-# }
|
|
|
-# try:
|
|
|
-# with open(STATUS_FILE_PATH, 'w') as f:
|
|
|
-# json.dump(status_data, f, indent=4)
|
|
|
-# except Exception as e:
|
|
|
-# logger.exception(f"CRITICAL ERROR: Failed to write status file at {STATUS_FILE_PATH}: {e}")
|
|
|
-
|
|
|
-# try:
|
|
|
-# # 1. PREFETCH all necessary related data to minimize database queries
|
|
|
-
|
|
|
-# # Prefetch possible values for mandatory attributes
|
|
|
-# possible_values_prefetch = Prefetch(
|
|
|
-# 'attributes',
|
|
|
-# queryset=ProductAttribute.objects.filter(is_mandatory=True).prefetch_related('possible_values')
|
|
|
-# )
|
|
|
-
|
|
|
-# # Fetch all ProductTypes with their mandatory attributes and possible values
|
|
|
-# all_product_types = ProductType.objects.prefetch_related(possible_values_prefetch)
|
|
|
-# product_type_map = {
|
|
|
-# pt.name: pt for pt in all_product_types
|
|
|
-# }
|
|
|
-
|
|
|
-# # Prepare product_list for batch extraction
|
|
|
-# all_products = Product.objects.all()
|
|
|
-# product_list = []
|
|
|
-
|
|
|
-# for p in all_products:
|
|
|
-# # mandatory_attrs will be the dictionary required by the service
|
|
|
-# mandatory_attrs_dict = {}
|
|
|
-# product_type_name = p.product_type.strip() if p.product_type else None
|
|
|
-
|
|
|
-# if product_type_name and product_type_name in product_type_map:
|
|
|
-# pt = product_type_map[product_type_name]
|
|
|
-
|
|
|
-# # Build the mandatory_attrs dictionary: { "Attribute Name": ["Value 1", "Value 2"], ... }
|
|
|
-# for attr in pt.attributes.all(): # .all() here works because we used Prefetch for 'attributes'
|
|
|
-# # attr.possible_values.all() works because we used prefetch_related('possible_values')
|
|
|
-# mandatory_attrs_dict[attr.name] = [
|
|
|
-# pv.value for pv in attr.possible_values.all()
|
|
|
-# ]
|
|
|
-
|
|
|
-# product_list.append({
|
|
|
-# "item_id": p.item_id,
|
|
|
-# "product_type_name": product_type_name,
|
|
|
-# "mandatory_attrs": mandatory_attrs_dict # <-- FIX: Pass the dictionary here
|
|
|
-# })
|
|
|
-
|
|
|
-# # Batch settings (using defaults)
|
|
|
-# model = "llama-3.1-8b-instant"
|
|
|
-# extract_additional = True
|
|
|
-# process_image = False
|
|
|
-# multiple = []
|
|
|
-# threshold_abs = 0.65
|
|
|
-# margin = 0.15
|
|
|
-# use_dynamic_thresholds = True
|
|
|
-# use_adaptive_margin = True
|
|
|
-# use_semantic_clustering = True
|
|
|
-
|
|
|
-# # Batch extraction logic
|
|
|
-# item_ids = [p['item_id'] for p in product_list]
|
|
|
-# products_queryset = Product.objects.filter(item_id__in=item_ids)
|
|
|
-# product_map = {product.item_id: product for product in products_queryset}
|
|
|
-# found_ids = set(product_map.keys())
|
|
|
-
|
|
|
-# for product_entry in product_list:
|
|
|
-# item_id = product_entry['item_id']
|
|
|
-# # FIX: mandatory_attrs is now correctly a dictionary (or an empty dictionary)
|
|
|
-# mandatory_attrs = product_entry['mandatory_attrs']
|
|
|
-
|
|
|
-# if item_id not in found_ids:
|
|
|
-# failed += 1
|
|
|
-# results.append({
|
|
|
-# "product_id": item_id,
|
|
|
-# "error": "Product not found in database"
|
|
|
-# })
|
|
|
-# logger.warning(f"Product {item_id} not found in database. Skipping extraction.")
|
|
|
-# continue
|
|
|
-
|
|
|
-# product = product_map[item_id]
|
|
|
-
|
|
|
-# try:
|
|
|
-# title = product.product_name
|
|
|
-# short_desc = product.product_short_description
|
|
|
-# long_desc = product.product_long_description
|
|
|
-# image_url = product.image_path
|
|
|
-
|
|
|
-# ocr_results = None
|
|
|
-# ocr_text = None
|
|
|
-# visual_results = None
|
|
|
-
|
|
|
-# if process_image and image_url:
|
|
|
-# logger.info(f"Processing image for product {item_id}...")
|
|
|
-# # OCR Processing
|
|
|
-# ocr_service = OCRService()
|
|
|
-# ocr_results = ocr_service.process_image(image_url)
|
|
|
-
|
|
|
-# if ocr_results and ocr_results.get("detected_text"):
|
|
|
-# # NOTE: Assuming ProductAttributeService.extract_attributes_from_ocr exists
|
|
|
-# ocr_attrs = ProductAttributeService.extract_attributes_from_ocr(
|
|
|
-# ocr_results, model
|
|
|
-# )
|
|
|
-# ocr_results["extracted_attributes"] = ocr_attrs
|
|
|
-# ocr_text = "\n".join([
|
|
|
-# f"{item['text']} (confidence: {item['confidence']:.2f})"
|
|
|
-# for item in ocr_results["detected_text"]
|
|
|
-# ])
|
|
|
-
|
|
|
-# # Visual Processing
|
|
|
-# visual_service = VisualProcessingService()
|
|
|
-# product_type_hint = product.product_type if product.product_type else None
|
|
|
-# visual_results = visual_service.process_image(image_url, product_type_hint)
|
|
|
-
|
|
|
-# if visual_results and visual_results.get('visual_attributes'):
|
|
|
-# visual_results['visual_attributes'] = ProductAttributeService.format_visual_attributes(
|
|
|
-# visual_results['visual_attributes']
|
|
|
-# )
|
|
|
-# logger.info(f"Image processing done for product {item_id}.")
|
|
|
-
|
|
|
-
|
|
|
-# # Combine product text with source tracking
|
|
|
-# product_text, source_map = ProductAttributeService.combine_product_text(
|
|
|
-# title=title,
|
|
|
-# short_desc=short_desc,
|
|
|
-# long_desc=long_desc,
|
|
|
-# ocr_text=ocr_text
|
|
|
-# )
|
|
|
-
|
|
|
-# # Attribute Extraction with source tracking
|
|
|
-# extracted = ProductAttributeService.extract_attributes(
|
|
|
-# product_text=product_text,
|
|
|
-# mandatory_attrs=mandatory_attrs, # <-- This is now the dictionary with possible values
|
|
|
-# source_map=source_map,
|
|
|
-# model=model,
|
|
|
-# extract_additional=extract_additional,
|
|
|
-# multiple=multiple,
|
|
|
-# threshold_abs=threshold_abs,
|
|
|
-# margin=margin,
|
|
|
-# use_dynamic_thresholds=use_dynamic_thresholds,
|
|
|
-# use_adaptive_margin=use_adaptive_margin,
|
|
|
-# use_semantic_clustering=use_semantic_clustering
|
|
|
-# )
|
|
|
-
|
|
|
-# result = {
|
|
|
-# "product_id": item_id,
|
|
|
-# "mandatory": extracted.get("mandatory", {}),
|
|
|
-# "additional": extracted.get("additional", {}),
|
|
|
-# }
|
|
|
-
|
|
|
-# if ocr_results:
|
|
|
-# result["ocr_results"] = ocr_results
|
|
|
-
|
|
|
-# if visual_results:
|
|
|
-# result["visual_results"] = visual_results
|
|
|
-
|
|
|
-# results.append(result)
|
|
|
-# successful += 1
|
|
|
-# logger.info(f"Attribute extraction successful for product {item_id}.")
|
|
|
-
|
|
|
-# except Exception as e:
|
|
|
-# failed += 1
|
|
|
-# results.append({
|
|
|
-# "product_id": item_id,
|
|
|
-# "error": str(e)
|
|
|
-# })
|
|
|
-# # Original Error: AttributeError: 'list' object has no attribute 'items'
|
|
|
-# # This should now be fixed, but we keep the robust exception handling.
|
|
|
-# logger.exception(f"Error during attribute extraction for product {item_id}.")
|
|
|
-
|
|
|
-# logger.info(f"Batch extraction phase complete. Successful: {successful}, Failed: {failed}")
|
|
|
-
|
|
|
-# # --------------------------------------------------------------------------------
|
|
|
-# # Generate and save the Excel file (Unchanged)
|
|
|
-# # --------------------------------------------------------------------------------
|
|
|
-# wb = Workbook()
|
|
|
-
|
|
|
-# # Sheet 1: Products (from DB)
|
|
|
-# ws_products = wb.active
|
|
|
-# ws_products.title = "Products"
|
|
|
-# products_headers = ['ITEM ID', 'PRODUCT NAME', 'PRODUCT TYPE', 'Product Short Description', 'Product Long Description', 'image_path']
|
|
|
-# header_fill = PatternFill(start_color="366092", end_color="366092", fill_type="solid")
|
|
|
-# header_font = Font(bold=True, color="FFFFFF")
|
|
|
-
|
|
|
-# for col_num, header in enumerate(products_headers, 1):
|
|
|
-# cell = ws_products.cell(row=1, column=col_num)
|
|
|
-# cell.value = header
|
|
|
-# cell.fill = header_fill
|
|
|
-# cell.font = header_font
|
|
|
-# cell.alignment = Alignment(horizontal="center", vertical="center")
|
|
|
-
|
|
|
-# all_products_db = Product.objects.all()
|
|
|
-# for row_num, product in enumerate(all_products_db, 2):
|
|
|
-# ws_products.cell(row=row_num, column=1, value=product.item_id)
|
|
|
-# ws_products.cell(row=row_num, column=2, value=product.product_name)
|
|
|
-# ws_products.cell(row=row_num, column=3, value=product.product_type)
|
|
|
-# ws_products.cell(row=row_num, column=4, value=product.product_short_description)
|
|
|
-# ws_products.cell(row=row_num, column=5, value=product.product_long_description)
|
|
|
-# ws_products.cell(row=row_num, column=6, value=product.image_path)
|
|
|
-
|
|
|
-# # Adjust column widths
|
|
|
-# for col_dim, width in zip(['A', 'B', 'C', 'D', 'E', 'F'], [15, 25, 15, 35, 50, 45]):
|
|
|
-# ws_products.column_dimensions[col_dim].width = width
|
|
|
-
|
|
|
-# # Sheet 2: Attribute_values (item_id, attribute_name, original_value, generated_value)
|
|
|
-# ws_attributes = wb.create_sheet("Attribute_values")
|
|
|
-# attributes_headers = ['item_id', 'attribute_name', 'original_value', 'generated_value']
|
|
|
-# for col_num, header in enumerate(attributes_headers, 1):
|
|
|
-# cell = ws_attributes.cell(row=1, column=col_num)
|
|
|
-# cell.value = header
|
|
|
-# cell.fill = header_fill
|
|
|
-# cell.font = header_font
|
|
|
-# cell.alignment = Alignment(horizontal="center", vertical="center")
|
|
|
-
|
|
|
-# # Fetch all original attributes
|
|
|
-# row_num = 2
|
|
|
-# all_original_attrs = ProductAttributeValue.objects.all()
|
|
|
-# # Create a lookup for original attributes by item_id and attribute_name
|
|
|
-# original_attrs_lookup = {
|
|
|
-# (attr.product.item_id, attr.attribute_name): attr.original_value
|
|
|
-# for attr in all_original_attrs
|
|
|
-# }
|
|
|
-
|
|
|
-# # Add attributes (original and generated)
|
|
|
-# processed_original_keys = set()
|
|
|
-# for res in results:
|
|
|
-# item_id = res["product_id"]
|
|
|
-
|
|
|
-# if "error" in res:
|
|
|
-# # Add existing original attributes for failed products to the sheet
|
|
|
-# for (orig_item_id, orig_attr_name), orig_value in original_attrs_lookup.items():
|
|
|
-# if orig_item_id == item_id:
|
|
|
-# ws_attributes.cell(row=row_num, column=1, value=orig_item_id)
|
|
|
-# ws_attributes.cell(row=row_num, column=2, value=orig_attr_name)
|
|
|
-# ws_attributes.cell(row=row_num, column=3, value=orig_value)
|
|
|
-# ws_attributes.cell(row=row_num, column=4, value=f"Extraction Failed: {res['error']}")
|
|
|
-# processed_original_keys.add((orig_item_id, orig_attr_name))
|
|
|
-# row_num += 1
|
|
|
-# continue
|
|
|
-
|
|
|
-# # Combine all generated attributes (mandatory, additional, OCR, visual)
|
|
|
-# generated_attrs = {}
|
|
|
-# for cat in ["mandatory", "additional"]:
|
|
|
-# attrs = res.get(cat, {})
|
|
|
-# for attr_name, values in attrs.items():
|
|
|
-# for val in values:
|
|
|
-# key = (item_id, attr_name)
|
|
|
-# if key not in generated_attrs:
|
|
|
-# generated_attrs[key] = []
|
|
|
-# generated_attrs[key].append(f"{val['value']} (source: {val['source']})")
|
|
|
-
|
|
|
-# # OCR extracted
|
|
|
-# ocr = res.get("ocr_results")
|
|
|
-# if ocr and "extracted_attributes" in ocr and isinstance(ocr["extracted_attributes"], dict):
|
|
|
-# for attr_name, values in ocr["extracted_attributes"].items():
|
|
|
-# for val in values:
|
|
|
-# key = (item_id, attr_name)
|
|
|
-# if key not in generated_attrs:
|
|
|
-# generated_attrs[key] = []
|
|
|
-# generated_attrs[key].append(f"{val['value']} (source: {val['source']})")
|
|
|
-
|
|
|
-# # Visual extracted
|
|
|
-# visual = res.get("visual_results")
|
|
|
-# if visual and "visual_attributes" in visual:
|
|
|
-# vis_attrs = visual["visual_attributes"]
|
|
|
-# if isinstance(vis_attrs, dict):
|
|
|
-# # Handle dict format where value might be a list of dicts or a single value
|
|
|
-# for attr_name, values in vis_attrs.items():
|
|
|
-# if not isinstance(values, list):
|
|
|
-# values = [{"value": values, "source": "visual"}]
|
|
|
-# for val in values:
|
|
|
-# key = (item_id, attr_name)
|
|
|
-# if key not in generated_attrs:
|
|
|
-# generated_attrs[key] = []
|
|
|
-# generated_attrs[key].append(f"{val['value']} (source: {val.get('source', 'visual')})")
|
|
|
-# elif isinstance(vis_attrs, list):
|
|
|
-# # Handle list of dicts format
|
|
|
-# for item in vis_attrs:
|
|
|
-# attr_name = item.get("attribute_name") or item.get("name")
|
|
|
-# if not attr_name: continue
|
|
|
-# value = item.get("value", "")
|
|
|
-# source = item.get("source", "visual")
|
|
|
-# key = (item_id, attr_name)
|
|
|
-# if key not in generated_attrs:
|
|
|
-# generated_attrs[key] = []
|
|
|
-# generated_attrs[key].append(f"{value} (source: {source})")
|
|
|
-
|
|
|
-
|
|
|
-# # Write attributes to Excel
|
|
|
-# for (attr_item_id, attr_name), gen_values in generated_attrs.items():
|
|
|
-# # Get original value from lookup (if it exists)
|
|
|
-# original_value = original_attrs_lookup.get((attr_item_id, attr_name), "")
|
|
|
-# # Combine multiple generated values into a single string
|
|
|
-# generated_value = "; ".join(gen_values) if gen_values else ""
|
|
|
-# # Write row
|
|
|
-# ws_attributes.cell(row=row_num, column=1, value=attr_item_id)
|
|
|
-# ws_attributes.cell(row=row_num, column=2, value=attr_name)
|
|
|
-# ws_attributes.cell(row=row_num, column=3, value=original_value)
|
|
|
-# ws_attributes.cell(row=row_num, column=4, value=generated_value)
|
|
|
-# processed_original_keys.add((attr_item_id, attr_name))
|
|
|
-# row_num += 1
|
|
|
-
|
|
|
-# # Add original attributes that have no generated values for this item_id
|
|
|
-# for (orig_item_id, orig_attr_name), orig_value in original_attrs_lookup.items():
|
|
|
-# if orig_item_id == item_id and (orig_item_id, orig_attr_name) not in processed_original_keys:
|
|
|
-# ws_attributes.cell(row=row_num, column=1, value=orig_item_id)
|
|
|
-# ws_attributes.cell(row=row_num, column=2, value=orig_attr_name)
|
|
|
-# ws_attributes.cell(row=row_num, column=3, value=orig_value)
|
|
|
-# ws_attributes.cell(row=row_num, column=4, value="") # No generated value
|
|
|
-# processed_original_keys.add((orig_item_id, orig_attr_name))
|
|
|
-# row_num += 1
|
|
|
-
|
|
|
-# # Add original attributes for products not included in the 'results' (e.g. if they didn't exist in product_list)
|
|
|
-# # We assume all products are in product_list, so this step might be redundant, but safe for completeness.
|
|
|
-# for (orig_item_id, orig_attr_name), orig_value in original_attrs_lookup.items():
|
|
|
-# if (orig_item_id, orig_attr_name) not in processed_original_keys:
|
|
|
-# ws_attributes.cell(row=row_num, column=1, value=orig_item_id)
|
|
|
-# ws_attributes.cell(row=row_num, column=2, value=orig_attr_name)
|
|
|
-# ws_attributes.cell(row=row_num, column=3, value=orig_value)
|
|
|
-# ws_attributes.cell(row=row_num, column=4, value="Original value only (Product not processed in batch)")
|
|
|
-# row_num += 1
|
|
|
-
|
|
|
-
|
|
|
-# # Adjust column widths for attributes
|
|
|
-# for col_dim, width in zip(['A', 'B', 'C', 'D'], [15, 35, 50, 50]):
|
|
|
-# ws_attributes.column_dimensions[col_dim].width = width
|
|
|
-
|
|
|
-# # Save the generated Excel (replace existing)
|
|
|
-# save_path = os.path.join(settings.MEDIA_ROOT, 'generated_products.xlsx')
|
|
|
-# wb.save(save_path)
|
|
|
-# logger.info(f"Excel file successfully saved to {save_path}")
|
|
|
-
|
|
|
-# # Write SUCCESS status
|
|
|
-# write_status("SUCCESS")
|
|
|
-# logger.info("Background task finished successfully.")
|
|
|
-
|
|
|
-
|
|
|
-# except Exception as e:
|
|
|
-# # Log the critical error and write FAILED status
|
|
|
-# logger.exception("CRITICAL ERROR during background Excel generation process.")
|
|
|
-# write_status("FAILED", error_msg=str(e))
|
|
|
-
|
|
|
-
|
|
|
-# # -------------------------------------------------------------------------------------------------
|
|
|
-
|
|
|
-# class ProductUploadExcelView(APIView):
|
|
|
-# """
|
|
|
-# POST API to upload an Excel file. (Unchanged)
|
|
|
-# """
|
|
|
-# parser_classes = (MultiPartParser, FormParser)
|
|
|
-
|
|
|
-# def post(self, request, *args, **kwargs):
|
|
|
-# file_obj = request.FILES.get('file')
|
|
|
-# if not file_obj:
|
|
|
-# return Response({'error': 'No file provided'}, status=status.HTTP_400_BAD_REQUEST)
|
|
|
-
|
|
|
-# try:
|
|
|
-# # Read all sheets from Excel file
|
|
|
-# excel_file = pd.ExcelFile(file_obj)
|
|
|
-
|
|
|
-# # Check if required sheets exist
|
|
|
-# if 'Products' not in excel_file.sheet_names:
|
|
|
-# logger.error(f"Upload failed: Missing 'Products' sheet in file.")
|
|
|
-# return Response({
|
|
|
-# 'error': "Missing 'Products' sheet",
|
|
|
-# 'available_sheets': excel_file.sheet_names
|
|
|
-# }, status=status.HTTP_400_BAD_REQUEST)
|
|
|
-
|
|
|
-# # Read Products sheet
|
|
|
-# df_products = pd.read_excel(excel_file, sheet_name='Products')
|
|
|
-# df_products.columns = [c.strip().lower().replace(' ', '_') for c in df_products.columns]
|
|
|
-
|
|
|
-# # Check required columns for Products
|
|
|
-# expected_product_cols = {
|
|
|
-# 'item_id', 'product_name', 'product_long_description',
|
|
|
-# 'product_short_description', 'product_type', 'image_path'
|
|
|
-# }
|
|
|
-
|
|
|
-# if not expected_product_cols.issubset(df_products.columns):
|
|
|
-# logger.error(f"Upload failed: Missing required columns in Products sheet.")
|
|
|
-# return Response({
|
|
|
-# 'error': 'Missing required columns in Products sheet',
|
|
|
-# 'required_columns': list(expected_product_cols),
|
|
|
-# 'found_columns': list(df_products.columns)
|
|
|
-# }, status=status.HTTP_400_BAD_REQUEST)
|
|
|
-
|
|
|
-# # Read Attribute_values sheet if it exists
|
|
|
-# df_attributes = None
|
|
|
-# has_attributes_sheet = 'Attribute_values' in excel_file.sheet_names
|
|
|
-
|
|
|
-# if has_attributes_sheet:
|
|
|
-# df_attributes = pd.read_excel(excel_file, sheet_name='Attribute_values')
|
|
|
-# df_attributes.columns = [c.strip().lower().replace(' ', '_') for c in df_attributes.columns]
|
|
|
-
|
|
|
-# # Check required columns for Attribute_values
|
|
|
-# expected_attr_cols = {'item_id', 'attribute_name', 'original_value'}
|
|
|
-# if not expected_attr_cols.issubset(df_attributes.columns):
|
|
|
-# logger.error(f"Upload failed: Missing required columns in Attribute_values sheet.")
|
|
|
-# return Response({
|
|
|
-# 'error': 'Missing required columns in Attribute_values sheet',
|
|
|
-# 'required_columns': list(expected_attr_cols),
|
|
|
-# 'found_columns': list(df_attributes.columns)
|
|
|
-# }, status=status.HTTP_400_BAD_REQUEST)
|
|
|
-
|
|
|
-# # Initialize counters
|
|
|
-# products_created = 0
|
|
|
-# products_updated = 0
|
|
|
-# attributes_created = 0
|
|
|
-# attributes_updated = 0
|
|
|
-# products_failed = 0
|
|
|
-# attributes_failed = 0
|
|
|
-# errors = []
|
|
|
-
|
|
|
-# # Use transaction to ensure atomicity
|
|
|
-# with transaction.atomic():
|
|
|
-# # Process Products sheet
|
|
|
-# for idx, row in df_products.iterrows():
|
|
|
-# item_id = str(row.get('item_id', '')).strip()
|
|
|
-# product_type = str(row.get('product_type', '')).strip()
|
|
|
-
|
|
|
-# if not item_id:
|
|
|
-# products_failed += 1
|
|
|
-# errors.append(f"Products Row {idx + 2}: Missing item_id")
|
|
|
-# continue
|
|
|
-
|
|
|
-# try:
|
|
|
-# # Auto-create ProductType if provided and doesn't exist
|
|
|
-# if product_type:
|
|
|
-# ProductType.objects.get_or_create(name=product_type)
|
|
|
-
|
|
|
-# defaults = {
|
|
|
-# 'product_name': str(row.get('product_name', '')),
|
|
|
-# 'product_long_description': str(row.get('product_long_description', '')),
|
|
|
-# 'product_short_description': str(row.get('product_short_description', '')),
|
|
|
-# 'product_type': product_type,
|
|
|
-# 'image_path': str(row.get('image_path', '')),
|
|
|
-# }
|
|
|
-
|
|
|
-# obj, created = Product.objects.update_or_create(
|
|
|
-# item_id=item_id,
|
|
|
-# defaults=defaults
|
|
|
-# )
|
|
|
-
|
|
|
-# if created:
|
|
|
-# products_created += 1
|
|
|
-# else:
|
|
|
-# products_updated += 1
|
|
|
-# except Exception as e:
|
|
|
-# products_failed += 1
|
|
|
-# errors.append(f"Products Row {idx + 2} (item_id: {item_id}): {str(e)}")
|
|
|
-# logger.error(f"Error processing product {item_id} in Products sheet: {e}")
|
|
|
-
|
|
|
-
|
|
|
-# # Process Attribute_values sheet if it exists
|
|
|
-# if has_attributes_sheet and df_attributes is not None:
|
|
|
-# # Group by item_id to optimize lookups
|
|
|
-# item_ids_in_attrs = df_attributes['item_id'].astype(str).unique()
|
|
|
-
|
|
|
-# # Fetch all products at once
|
|
|
-# existing_products = {
|
|
|
-# p.item_id: p
|
|
|
-# for p in Product.objects.filter(item_id__in=item_ids_in_attrs)
|
|
|
-# }
|
|
|
-
|
|
|
-# for idx, row in df_attributes.iterrows():
|
|
|
-# item_id = str(row.get('item_id', '')).strip()
|
|
|
-# attribute_name = str(row.get('attribute_name', '')).strip()
|
|
|
-# original_value = str(row.get('original_value', '')).strip()
|
|
|
-
|
|
|
-# if not item_id or not attribute_name:
|
|
|
-# attributes_failed += 1
|
|
|
-# errors.append(
|
|
|
-# f"Attribute_values Row {idx + 2}: Missing item_id or attribute_name"
|
|
|
-# )
|
|
|
-# continue
|
|
|
-
|
|
|
-# # Check if product exists
|
|
|
-# product = existing_products.get(item_id)
|
|
|
-# if not product:
|
|
|
-# attributes_failed += 1
|
|
|
-# errors.append(
|
|
|
-# f"Attribute_values Row {idx + 2}: Product with item_id '{item_id}' not found. "
|
|
|
-# "Make sure it exists in Products sheet."
|
|
|
-# )
|
|
|
-# continue
|
|
|
-
|
|
|
-# try:
|
|
|
-# attr_obj, created = ProductAttributeValue.objects.update_or_create(
|
|
|
-# product=product,
|
|
|
-# attribute_name=attribute_name,
|
|
|
-# defaults={'original_value': original_value}
|
|
|
-# )
|
|
|
-
|
|
|
-# if created:
|
|
|
-# attributes_created += 1
|
|
|
-# else:
|
|
|
-# attributes_updated += 1
|
|
|
-# except Exception as e:
|
|
|
-# attributes_failed += 1
|
|
|
-# errors.append(
|
|
|
-# f"Attribute_values Row {idx + 2} "
|
|
|
-# f"(item_id: {item_id}, attribute: {attribute_name}): {str(e)}"
|
|
|
-# )
|
|
|
-# logger.error(f"Error processing attribute {attribute_name} for product {item_id}: {e}")
|
|
|
-
|
|
|
-# # Prepare response data
|
|
|
-# response_data = {
|
|
|
-# 'message': 'Upload completed',
|
|
|
-# 'products': {
|
|
|
-# 'created': products_created,
|
|
|
-# 'updated': products_updated,
|
|
|
-# 'failed': products_failed,
|
|
|
-# 'total_processed': products_created + products_updated + products_failed
|
|
|
-# },
|
|
|
-# 'attribute_values': {
|
|
|
-# 'created': attributes_created,
|
|
|
-# 'updated': attributes_updated,
|
|
|
-# 'failed': attributes_failed,
|
|
|
-# 'total_processed': attributes_created + attributes_updated + attributes_failed
|
|
|
-# } if has_attributes_sheet else {'message': 'Attribute_values sheet not found in Excel file'},
|
|
|
-# 'generated_excel_status': 'Excel generation started in the background.'
|
|
|
-# }
|
|
|
-
|
|
|
-# if errors:
|
|
|
-# response_data['errors'] = errors[:50]
|
|
|
-# if len(errors) > 50:
|
|
|
-# response_data['errors'].append(f"... and {len(errors) - 50} more errors")
|
|
|
-
|
|
|
-# # Determine status code for upload
|
|
|
-# upload_status = status.HTTP_201_CREATED if products_failed == 0 and attributes_failed == 0 else status.HTTP_207_MULTI_STATUS
|
|
|
-
|
|
|
-# # Start background thread for Excel generation if upload was successful
|
|
|
-# if products_failed == 0 and attributes_failed == 0:
|
|
|
-# logger.info("API call successful. Triggering background Excel generation thread.")
|
|
|
-# threading.Thread(target=generate_product_excel_background, daemon=True).start()
|
|
|
-
|
|
|
-# # Update response to provide monitoring paths
|
|
|
-# response_data['generated_excel_status'] = 'Background Excel generation triggered successfully.'
|
|
|
-# response_data['monitoring'] = {
|
|
|
-# 'excel_file': os.path.join(settings.MEDIA_URL, 'generated_products.xlsx'),
|
|
|
-# 'status_file': os.path.join(settings.MEDIA_URL, 'excel_generation_status.json'),
|
|
|
-# 'log_file': os.path.join(settings.MEDIA_URL, 'excel_generation.log'),
|
|
|
-# 'note': 'These files will be available once the background process completes.'
|
|
|
-# }
|
|
|
-# else:
|
|
|
-# logger.warning(f"API call finished with errors ({products_failed} products, {attributes_failed} attributes). Not triggering background excel generation.")
|
|
|
-# response_data['generated_excel_status'] = 'Background Excel generation was NOT triggered due to upload errors. Fix upload errors and re-upload.'
|
|
|
-
|
|
|
-
|
|
|
-# return Response(response_data, status=upload_status)
|
|
|
-
|
|
|
-# except pd.errors.EmptyDataError:
|
|
|
-# logger.error('The uploaded Excel file is empty or invalid.')
|
|
|
-# return Response({
|
|
|
-# 'error': 'The uploaded Excel file is empty or invalid'
|
|
|
-# }, status=status.HTTP_400_BAD_REQUEST)
|
|
|
-# except Exception as e:
|
|
|
-# logger.exception(f'An unexpected error occurred while processing the file.')
|
|
|
-# return Response({
|
|
|
-# 'error': f'An unexpected error occurred while processing the file: {str(e)}'
|
|
|
-# }, status=status.HTTP_500_INTERNAL_SERVER_ERROR)
|
|
|
|
|
|
|
|
|
|
