Update app.py

app.py

@@ -188,51 +188,33 @@ def load_vector_store(embeddings):
                 if len(matches) > 100:
                     logger.info(f"   Found {len(matches)} potential document fragments")
                     
-                    # Get total vectors in index
-                    num_vectors = index.ntotal
-                    logger.info(f"   FAISS index has {num_vectors} vectors")
-                    
-                    # Create documents matching the number of vectors
-                    new_docstore_dict = {}
-                    index_to_docstore_id = {}
-                    
-                    # Use the actual number of vectors, not extracted matches
-                    for idx in range(min(num_vectors, len(matches))):
+                    # Create documents from extracted text
+                    documents = []
+                    for idx, match in enumerate(matches[:5000]):  # Use first 5000 quality matches
                         try:
-                            # Get content from matches
-                            if idx < len(matches):
-                                content = matches[idx].decode('utf-8', errors='ignore').strip()
-                            else:
-                                content = f"Fashion document {idx}"
-                            
-                            if len(content) < 50:
-                                content = f"Fashion advice and style guide entry {idx}"
-                            
-                            # Create document with string ID
-                            doc_id = str(idx)
-                            new_doc = Document(
-                                page_content=content,
-                                metadata={"source": "reconstructed"}
-                            )
-                            new_docstore_dict[doc_id] = new_doc
-                            # CRITICAL: Use string keys for index_to_docstore_id
-                            index_to_docstore_id[str(idx)] = doc_id
-                        except Exception as e:
-                            logger.warning(f"   Error creating doc {idx}: {e}")
+                            content = match.decode('utf-8', errors='ignore').strip()
+                            if len(content) >= 100:  # Only high-quality, substantial content
+                                doc = Document(
+                                    page_content=content,
+                                    metadata={"source": "reconstructed", "id": idx}
+                                )
+                                documents.append(doc)
+                        except:
                             continue
                     
-                    logger.info(f"   ✅ Reconstructed {len(new_docstore_dict)} documents from raw data")
+                    if len(documents) < 100:
+                        raise Exception(f"Only extracted {len(documents)} documents, need at least 100")
                     
-                    docstore = InMemoryDocstore(new_docstore_dict)
+                    logger.info(f"   ✅ Extracted {len(documents)} high-quality documents")
+                    logger.info(f"   🔄 Rebuilding FAISS index from scratch...")
                     
-                    vectorstore = FAISS(
-                        embedding_function=embeddings,
-                        index=index,
-                        docstore=docstore,
-                        index_to_docstore_id=index_to_docstore_id
+                    # Create NEW FAISS index from documents (ignore old corrupted index)
+                    vectorstore = FAISS.from_documents(
+                        documents=documents,
+                        embedding=embeddings
                     )
                     
-                    logger.info(f"✅ FAISS vector store reconstructed from raw data")
+                    logger.info(f"✅ FAISS vector store rebuilt from {len(documents)} documents")
                     return vectorstore
                 else:
                     raise Exception("Could not extract enough document content from pickle")