Update parse_bhel.py

parse_bhel.py

@@ -4,26 +4,38 @@ import pandas as pd
 def preprocess_rows(rows, expected_columns):
     aligned_rows = []
     buffer = []
-    
+    unalignable_rows = []  # Capture unaligned rows for inspection
+
     for row in rows:
-        # If the row has the correct number of columns, add it as-is
+        # Check if the row contains irrelevant metadata or headers
+        if any(keyword in row[0] for keyword in ["GSTIN", "Currency", "Payment Terms", "General Terms", "Delivery Schedule"]):
+            continue
+
+        # If row matches expected length, add directly
         if len(row) == expected_columns:
-            # If there's buffered content from previous rows, add it before this row
             if buffer:
-                aligned_rows.append(buffer)
-                buffer = []  # Clear the buffer
+                aligned_rows.append(buffer)  # Add any buffered row first
+                buffer = []  # Reset buffer
             aligned_rows.append(row)
-        else:
-            # If row has fewer columns, treat it as a continuation and add to the buffer
+
+        # If row contains part of an entry (such as "Material Number" or "HSN Code")
+        elif "Material Number" in row[0] or "HSN Code" in row[0] or "IGST" in row[0]:
             if buffer:
-                buffer = [f"{b} {r}" if r else b for b, r in zip(buffer, row + [""] * (len(buffer) - len(row)))]
+                buffer[-1] += " " + row[0]  # Append to last column in buffer
             else:
-                buffer = row  # Initialize the buffer with the row
+                buffer = row  # Initialize buffer with this part-row
+        else:
+            # If unalignable, add to unalignable_rows for debugging
+            unalignable_rows.append(row)
 
-    # If there's any remaining buffered row, add it to aligned rows
+    # Log any remaining buffered content
     if buffer:
         aligned_rows.append(buffer)
 
+    # Print unalignable rows for analysis
+    for row in unalignable_rows:
+        print(f"Unalignable row: {row}")
+
     return aligned_rows
 
 def parse_bhel_pdf(pdf_path):
@@ -38,15 +50,14 @@ def parse_bhel_pdf(pdf_path):
         for page in pdf.pages:
             table = page.extract_table()
             if table:
-                # Skip the header row and preprocess rows to align data
+                # Preprocess and align rows before DataFrame conversion
                 rows = preprocess_rows(table[1:], expected_columns)
                 for row in rows:
-                    # Only add rows that match the expected number of columns after preprocessing
                     if len(row) == expected_columns:
                         data.append(row)
                     else:
                         print(f"Skipping unalignable row: {row}")
 
-    # Create a DataFrame with the specified columns
+    # Convert aligned rows into a DataFrame
     df = pd.DataFrame(data, columns=columns)
     return df