demeter

Autonomous Hydroponic Intelligence
git clone https://github.com/maydayv7/demeter

rag_brain.py (4023B)

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
import sys
import os
import uuid
import pypdf
from qdrant_client import models
from fastembed import TextEmbedding

current_dir = os.path.dirname(os.path.abspath(__file__))
project_root = os.path.abspath(os.path.join(current_dir, '../../'))
sys.path.append(project_root)

from Qdrant.Client import client  

# --- CONFIGURATION ---
COLLECTION_NAME = "Knowledge_Base"
VECTOR_SIZE = 384  
DOCS_FOLDER = os.path.join(project_root, "Knowledge_Base") 

def init_collection():
    """
    Creates the collection if it doesn't exist.
    """
    if client.collection_exists(COLLECTION_NAME):
        print(f"ℹī¸  Collection '{COLLECTION_NAME}' already exists. Appending data...")
    else:
        print(f"🔨 Creating new collection: {COLLECTION_NAME}")
        client.create_collection(
            collection_name=COLLECTION_NAME,
            vectors_config=models.VectorParams(
                size=VECTOR_SIZE, 
                distance=models.Distance.COSINE
            )
        )
        print("✅ Collection created.")

def extract_text_from_pdf(pdf_path):
    """
    Reads a PDF file page by page and returns the full text.
    """
    text = ""
    try:
        reader = pypdf.PdfReader(pdf_path)
        for page in reader.pages:
            page_text = page.extract_text()
            if page_text:
                text += page_text + "\n"
    except Exception as e:
        print(f"❌ Error reading PDF {pdf_path}: {e}")
    return text

def chunk_text(text, chunk_size=500, overlap=50):
    """
    Splits long text into smaller overlapping pieces.
    Overlap helps preserve context between chunks.
    """
    if not text:
        return []
    return [text[i:i+chunk_size] for i in range(0, len(text), chunk_size - overlap)]

def ingest_docs():
    # 1. Setup Collection & Model
    init_collection()
    
    print("🧠 Loading Embedding Model (bge-small-en)...")
    model = TextEmbedding(model_name="BAAI/bge-small-en-v1.5")
    
    # 2. Check if folder exists
    if not os.path.exists(DOCS_FOLDER):
        os.makedirs(DOCS_FOLDER)
        print(f"⚠ī¸ Created folder '{DOCS_FOLDER}'. Please put your PDFs there and run this script again!")
        return

    # 3. Scan for files
    files = [f for f in os.listdir(DOCS_FOLDER) if f.endswith(('.pdf', '.txt'))]
    if not files:
        print(f"📭 No files found in '{DOCS_FOLDER}'. Add some PDFs!")
        return

    print(f"📚 Found {len(files)} documents. Starting ingestion...")
    total_chunks = 0

    for file_name in files:
        file_path = os.path.join(DOCS_FOLDER, file_name)
        print(f"   📄 Processing: {file_name}")

        # A. Extract Text
        content = ""
        if file_name.endswith('.pdf'):
            content = extract_text_from_pdf(file_path)
        else:
            with open(file_path, 'r', encoding='utf-8') as f:
                content = f.read()

        if not content.strip():
            print(f"      ⚠ī¸ Skipping empty file.")
            continue

        # B. Chunk Text
        chunks = chunk_text(content)
        if not chunks:
            continue
            
        # C. Convert to Vectors (Embed)
        # FastEmbed handles the list of strings automatically
        embeddings = list(model.embed(chunks))

        # D. Prepare Points for Qdrant
        points = []
        for i, (text_chunk, vector) in enumerate(zip(chunks, embeddings)):
            points.append(models.PointStruct(
                id=str(uuid.uuid4()),  
                vector=vector.tolist(),
                payload={
                    "text": text_chunk,
                    "source": file_name,
                    "chunk_id": i
                }
            ))

        # E. Upload Batch
        client.upsert(collection_name=COLLECTION_NAME, points=points)
        total_chunks += len(points)
        print(f"      ✅ Uploaded {len(points)} chunks.")

    print(f"\n🎉 Success! Knowledge Base now contains {total_chunks} searchable segments.")

if __name__ == "__main__":
    ingest_docs()