commit 0e867fc4c1f00e6e5476c4fffcf6addf4e60a227
parent 30f0e834ff7c95d89f335b3ab21436f2bba13c9d
Author: Debarghya Das <debarghya1108@gmail.com>
Date: Sun, 1 Mar 2026 02:01:28 +0000
Merge PR
Diffstat:
6 files changed, 222 insertions(+), 49 deletions(-)
diff --git a/Sentinel/Encoders/Vision.py b/Sentinel/Encoders/Vision.py
@@ -2,6 +2,9 @@
import torch
import clip
from PIL import Image
+import io
+import base64
+from pathlib import Path
class VisionEncoder:
def __init__(self, model_name="ViT-B/32"):
@@ -9,12 +12,91 @@ class VisionEncoder:
self.model, self.preprocess = clip.load(model_name, device=self.device)
self.model.eval()
- def encode(self, image_path: str):
- image = self.preprocess(Image.open(image_path).convert("RGB")) \
+ def encode(self, image_input):
+ """
+ Encode an image from multiple input types:
+ - File path (str or Path)
+ - Base64 string
+ - BytesIO object
+ - PIL Image object
+
+ Args:
+ image_input: File path, base64 string, BytesIO, or PIL Image
+
+ Returns:
+ numpy array: Normalized image embedding vector
+ """
+ # Convert input to PIL Image
+ pil_image = self._to_pil_image(image_input)
+
+ # Preprocess and encode
+ image = self.preprocess(pil_image.convert("RGB")) \
.unsqueeze(0).to(self.device)
with torch.no_grad():
vec = self.model.encode_image(image)
vec = vec / vec.norm(dim=-1, keepdim=True)
- return vec.cpu().numpy().flatten()
-\ No newline at end of file
+ return vec.cpu().numpy().flatten()
+
+ def _to_pil_image(self, image_input):
+ """
+ Convert various input types to PIL Image.
+ """
+ # If already a PIL Image
+ if isinstance(image_input, Image.Image):
+ return image_input
+
+ # If BytesIO object
+ if isinstance(image_input, io.BytesIO):
+ image_input.seek(0) # Reset to beginning
+ return Image.open(image_input)
+
+ # If it's a string, determine if it's a path or base64
+ if isinstance(image_input, (str, Path)):
+ # Check if it's a file path
+ if isinstance(image_input, Path) or Path(image_input).exists():
+ return Image.open(image_input)
+
+ # Otherwise, treat as base64
+ return self._base64_to_pil(image_input)
+
+ # If bytes object
+ if isinstance(image_input, bytes):
+ return Image.open(io.BytesIO(image_input))
+
+ raise TypeError(f"Unsupported image input type: {type(image_input)}")
+
+ def _base64_to_pil(self, base64_string):
+ """
+ Convert base64 string to PIL Image.
+ """
+ # Remove header if present (e.g., "data:image/png;base64,...")
+ if "," in base64_string:
+ base64_string = base64_string.split(",")[1]
+
+ # Add padding if necessary
+ missing_padding = len(base64_string) % 4
+ if missing_padding:
+ base64_string += '=' * (4 - missing_padding)
+
+ # Decode and open
+ image_bytes = base64.b64decode(base64_string)
+ return Image.open(io.BytesIO(image_bytes))
+
+
+# Example usage:
+if __name__ == "__main__":
+ encoder = VisionEncoder()
+
+ # Test with file path
+ # vec1 = encoder.encode("path/to/image.jpg")
+
+ # Test with base64
+ sample_base64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+ip1sAAAAASUVORK5CYII="
+ vec2 = encoder.encode(sample_base64)
+ print(f"✅ Encoded base64 image. Vector shape: {vec2.shape}")
+
+ # Test with BytesIO
+ # image_stream = io.BytesIO(image_bytes)
+ # vec3 = encoder.encode(image_stream)
+\ No newline at end of file
diff --git a/Sentinel/Encoders/__pycache__/Vision.cpython-311.pyc b/Sentinel/Encoders/__pycache__/Vision.cpython-311.pyc
Binary files differ.
diff --git a/Sentinel/__pycache__/agent.cpython-311.pyc b/Sentinel/__pycache__/agent.cpython-311.pyc
Binary files differ.
diff --git a/Sentinel/agent.py b/Sentinel/agent.py
@@ -1,7 +1,11 @@
import uuid
+import base64
+import io
from datetime import datetime
import numpy as np
+from pathlib import Path
+# Ensure these imports match your project structure
from Sentinel.Encoders.Vision import VisionEncoder
from Sentinel.Encoders.TimeSeries import SensorEncoder
from Sentinel.fmu import FMU
@@ -12,10 +16,30 @@ class FMUBuilder:
self.vision = VisionEncoder()
self.sensors = SensorEncoder()
- def create_fmu(self, image_path, sensor_data, metadata=None):
- img_vec = self.vision.encode(image_path)
+ def create_fmu(self, image_input, sensor_data, metadata=None):
+ """
+ Creates an FMU from either:
+ - A file path (str/Path)
+ - A Base64 encoded image string
+
+ Args:
+ image_input: Either a file path string or base64 string
+ sensor_data: Dictionary of sensor readings
+ metadata: Optional metadata dictionary
+ """
+
+ # Detect if input is base64 or file path
+ if self._is_base64(image_input):
+ # Handle Base64 input
+ img_vec = self._encode_from_base64(image_input)
+ else:
+ # Handle file path input (original behavior)
+ img_vec = self.vision.encode(image_input)
+
+ # Encode sensor data
sensor_vec = self.sensors.encode(sensor_data)
+ # Combine vectors
fmu_vector = np.concatenate([img_vec, sensor_vec]).tolist()
return FMU(
@@ -23,9 +47,63 @@ class FMUBuilder:
vector=fmu_vector,
metadata={
**(metadata or {}),
- "timestamp": datetime.utcnow().isoformat()
+ "timestamp": datetime.utcnow().isoformat(),
}
)
+
+ def _is_base64(self, s):
+ """
+ Detect if string is base64 or a file path.
+ Returns True if it looks like base64, False if it looks like a path.
+ """
+ if not isinstance(s, str):
+ return False
+
+ # If it has path separators, it's probably a path
+ if '/' in s or '\\' in s or Path(s).exists():
+ return False
+
+ # If it has base64 header, it's definitely base64
+ if s.startswith('data:image'):
+ return True
+
+ # Check if it's valid base64 (after removing potential header)
+ test_str = s.split(',')[-1] if ',' in s else s
+
+ # Base64 strings are typically very long and only contain valid b64 chars
+ if len(test_str) > 100: # Arbitrary threshold
+ try:
+ base64.b64decode(test_str, validate=True)
+ return True
+ except Exception:
+ return False
+
+ return False
+
+ def _encode_from_base64(self, image_base64):
+ """
+ Decode base64 string and encode the image.
+ """
+ # Remove header if present (e.g., "data:image/png;base64,...")
+ if "," in image_base64:
+ image_base64 = image_base64.split(",")[1]
+
+ # Add padding if necessary (fix the "multiple of 4" error)
+ missing_padding = len(image_base64) % 4
+ if missing_padding:
+ image_base64 += '=' * (4 - missing_padding)
+
+ # Decode to bytes
+ image_bytes = base64.b64decode(image_base64)
+
+ # Create file-like object
+ image_stream = io.BytesIO(image_bytes)
+
+ # Encode using VisionEncoder
+ # If VisionEncoder only accepts paths, you may need to update it
+ # to also accept BytesIO objects or PIL Images
+ return self.vision.encode(image_stream)
+
if __name__ == "__main__":
builder = FMUBuilder()
@@ -37,13 +115,17 @@ if __name__ == "__main__":
"humidity": 72.0
}
- fmu = builder.create_fmu("Sentinel/Sample.png", sensors, {
+ # Test with base64
+ sample_base64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+ip1sAAAAASUVORK5CYII="
+
+ fmu = builder.create_fmu(sample_base64, sensors, {
"crop": "lettuce",
"stage": "vegetative"
})
- print("FMU ID:", fmu.id)
- print("Vector length:", len(fmu.vector))
- print("Metadata:", fmu.metadata)
+ print("✅ FMU ID:", fmu.id)
+ print("✅ Vector length:", len(fmu.vector))
+ print("✅ Metadata:", fmu.metadata)
- store_fmu(fmu)
-\ No newline at end of file
+ # Test with file path
+ # fmu2 = builder.create_fmu("path/to/image.png", sensors, {"crop": "basil"})
+\ No newline at end of file
diff --git a/backend/server/functions.py b/backend/server/functions.py
@@ -1,48 +1,38 @@
-import os
-import shutil
import json
-from fastapi import UploadFile
+from qdrant_client.http import models
from Qdrant.Store import store_fmu, COLLECTION_NAME
from Qdrant.Client import client
-from qdrant_client.http import models
-async def process_ingest(file: UploadFile, sensors_str: str, metadata_str: str, builder):
+async def process_ingest(image_base64: str, sensors_str: str, metadata_str: str, builder):
"""
- Handles file saving, FMU creation, and storage logic.
+ Handles FMU creation and storage logic using base64 image.
+ No more temporary files!
"""
- # 1. Save Image Temporarily
- temp_filename = f"temp_{file.filename}"
- with open(temp_filename, "wb") as buffer:
- shutil.copyfileobj(file.file, buffer)
-
try:
- # 2. Parse Data
+ # 1. Parse Data
sensor_data = json.loads(sensors_str)
meta_data = json.loads(metadata_str)
- # 3. Create FMU
- abs_image_path = os.path.abspath(temp_filename)
- fmu = builder.create_fmu(abs_image_path, sensor_data, meta_data)
+ # 2. Create FMU directly from base64
+ print(f"📡 Creating FMU from base64 image...")
+ fmu = builder.create_fmu(image_base64, sensor_data, meta_data)
- # 4. Store in Cloud
+ # 3. Store in Cloud
store_fmu(fmu)
+ print(f"✅ FMU stored successfully: {fmu.id}")
return {"status": "success", "fmu_id": fmu.id}
- finally:
- if os.path.exists(temp_filename):
- os.remove(temp_filename)
+ except Exception as e:
+ print(f"❌ Ingest processing error: {e}")
+ raise
-async def process_search(file: UploadFile, sensors_str: str, builder):
+async def process_search(image_base64: str, sensors_str: str, builder):
"""
Handles image processing, context extraction, and filtered Qdrant search.
+ Uses base64 image instead of temporary files.
"""
- temp_filename = f"temp_search_{file.filename}"
- with open(temp_filename, "wb") as buffer:
- shutil.copyfileobj(file.file, buffer)
-
try:
sensor_data = json.loads(sensors_str)
- abs_image_path = os.path.abspath(temp_filename)
# --- STEP 1: Context Extraction ---
target_crop = sensor_data.get("crop")
@@ -70,12 +60,14 @@ async def process_search(file: UploadFile, sensors_str: str, builder):
]
)
- # --- STEP 4: Generate Vector ---
- query_fmu = builder.create_fmu(abs_image_path, numeric_sensors, metadata=metadata)
+ # --- STEP 4: Generate Vector from base64 ---
+ print(f"🧠 Generating query vector from base64 image...")
+ query_fmu = builder.create_fmu(image_base64, numeric_sensors, metadata=metadata)
query_vector = query_fmu.vector.tolist() if hasattr(query_fmu.vector, 'tolist') else query_fmu.vector
# --- STEP 5: Search ---
try:
+ print(f"🔍 Searching Qdrant with filters...")
response = client.query_points(
collection_name=COLLECTION_NAME,
query=query_vector,
@@ -84,17 +76,19 @@ async def process_search(file: UploadFile, sensors_str: str, builder):
with_payload=True
)
hits = response.points
+ print(f"✅ Found {len(hits)} matches")
except Exception as filter_error:
# Fallback for missing indexes
if "Index required" in str(filter_error):
- print("⚠️ Index missing. Falling back to unfiltered search.")
+ print("⚠️ Payload indexes missing. Falling back to unfiltered search.")
+ print("💡 Run 'python create_indexes.py' to enable filtered searches.")
response = client.search(
collection_name=COLLECTION_NAME,
query_vector=query_vector,
limit=5,
with_payload=True
)
- hits = response
+ hits = response.points
else:
raise filter_error
@@ -105,6 +99,6 @@ async def process_search(file: UploadFile, sensors_str: str, builder):
]
return {"results": results}
- finally:
- if os.path.exists(temp_filename):
- os.remove(temp_filename)
-\ No newline at end of file
+ except Exception as e:
+ print(f"❌ Search processing error: {e}")
+ raise
+\ No newline at end of file
diff --git a/backend/server/main.py b/backend/server/main.py
@@ -1,5 +1,6 @@
import sys
import os
+import base64
# --- PATH FIX ---
current_dir = os.path.dirname(os.path.abspath(__file__))
@@ -29,6 +30,13 @@ print("🌱 Initializing Demeter Agents...")
builder = FMUBuilder()
print("✅ Agents Ready.")
+async def file_to_base64(file: UploadFile) -> str:
+ """Convert uploaded file to base64 string"""
+ contents = await file.read()
+ base64_string = base64.b64encode(contents).decode('utf-8')
+ await file.seek(0) # Reset file pointer in case it's needed again
+ return base64_string
+
@app.post("/ingest")
async def ingest_endpoint(
file: UploadFile = File(...),
@@ -36,10 +44,14 @@ async def ingest_endpoint(
metadata: str = Form(...)
):
try:
- # Pass the builder instance to the route handler
- return await process_ingest(file, sensors, metadata, builder)
+ # Convert file to base64
+ image_base64 = await file_to_base64(file)
+ # Pass the base64 string to the process function
+ return await process_ingest(image_base64, sensors, metadata, builder)
except Exception as e:
print(f"❌ Ingest Error: {e}")
+ import traceback
+ traceback.print_exc()
return {"status": "error", "message": str(e)}
@app.post("/search")
@@ -48,7 +60,10 @@ async def search_endpoint(
sensors: str = Form(...)
):
try:
- return await process_search(file, sensors, builder)
+ # Convert file to base64
+ image_base64 = await file_to_base64(file)
+ # Pass the base64 string to the process function
+ return await process_search(image_base64, sensors, builder)
except Exception as e:
print(f"❌ Search Error: {e}")
import traceback