commit 310ecc90900a22d1e6abd476230bd82df5231a18
parent 15201611237d6da7fd58a87e7015080821dc9727
Author: Debarghya Das <debarghya1108@gmail.com>
Date: Mon, 2 Mar 2026 06:10:48 +0000
Merge PR
Diffstat:
5 files changed, 201 insertions(+), 33 deletions(-)
diff --git a/agent/Sentinel/agent.py b/agent/Sentinel/agent.py
@@ -120,29 +120,29 @@ class FMUBuilder:
return self.vision.encode(image_stream)
-if __name__ == "__main__":
- builder = FMUBuilder()
+# if __name__ == "__main__":
+# builder = FMUBuilder()
- sensors = {
- "pH": 5.9,
- "EC": 1.3,
- "temp": 25.0,
- "humidity": 72.0
- }
+# sensors = {
+# "pH": 5.9,
+# "EC": 1.3,
+# "temp": 25.0,
+# "humidity": 72.0
+# }
- # Test with base64
- sample_base64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+ip1sAAAAASUVORK5CYII="
+# # Test with base64
+# sample_base64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+ip1sAAAAASUVORK5CYII="
- fmu = builder.create_fmu(sample_base64, sensors, {
- "crop": "lettuce",
- "stage": "vegetative"
- })
+# fmu = builder.create_fmu(sample_base64, sensors, {
+# "crop": "lettuce",
+# "stage": "vegetative"
+# })
- print("ā
FMU ID:", fmu.id)
- print("ā
Vector length:", len(fmu.vector))
+# print("ā
FMU ID:", fmu.id)
+# print("ā
Vector length:", len(fmu.vector))
- # Check for the new fields in the output
- print("\nš Checking Schema:")
- print(f" - Action: {fmu.metadata.get('action_taken')}")
- print(f" - Outcome: {fmu.metadata.get('outcome')}")
- print(f" - Sensors Saved: {'sensors' in fmu.metadata}")
-\ No newline at end of file
+# # Check for the new fields in the output
+# print("\nš Checking Schema:")
+# print(f" - Action: {fmu.metadata.get('action_taken')}")
+# print(f" - Outcome: {fmu.metadata.get('outcome')}")
+# print(f" - Sensors Saved: {'sensors' in fmu.metadata}")
+\ No newline at end of file
diff --git a/agent/tools/db_tools.py b/agent/tools/db_tools.py
@@ -3,7 +3,7 @@ from qdrant_client import QdrantClient, models
from qdrant_client.models import PointStruct
class DBTools:
- def __init__(self, host="https://2a9e6ab0-e572-4bfa-a50f-0a169f9753d3.europe-west3-0.gcp.cloud.qdrant.io", collection_name="farm_memory"):
+ def __init__(self, host="https://2a9e6ab0-e572-4bfa-a50f-0a169f9753d3.europe-west3-0.gcp.cloud.qdrant.io", collection_name="Farm_Memory"):
self.client = QdrantClient(url=host)
self.collection_name = collection_name
self.vector_size = 516 # As seen in Qdrant/Setup.py
diff --git a/backend/server/functions.py b/backend/server/functions.py
@@ -86,8 +86,32 @@ async def process_ingest(file: UploadFile, sensors_str: str, metadata_str: str,
meta_data = json.loads(metadata_str)
abs_image_path = os.path.abspath(temp_filename)
+ # --- š¢ NEW: Add Sequence & ID Logic (Same as process_search) ---
+ target_crop = meta_data.get("crop", "Unknown")
+
+ # 1. Get Crop ID (Prefer metadata, fall back to sensor data, then auto-generate)
+ target_crop_id = meta_data.get("crop_id") or sensor_data.get("crop_id")
+ if not target_crop_id:
+ target_crop_id = f"Batch_{target_crop}_{datetime.now().strftime('%Y%m')}"
+
+ # 2. Calculate Sequence Number automatically
+ seq_num = get_next_sequence_number(target_crop_id)
+
+ print(f"š„ Ingesting {target_crop_id} | Snapshot #{seq_num}")
+
+ # 3. Inject into Metadata BEFORE creating FMU
+ meta_data.update({
+ "crop_id": target_crop_id,
+ "sequence_number": seq_num,
+ # Ensure placeholders exist if not provided
+ "action_taken": meta_data.get("action_taken", "PENDING_ACTION"),
+ "outcome": meta_data.get("outcome", "PENDING_OBSERVATION")
+ })
+ # -------------------------------------------------------------
+
fmu = builder.create_fmu(abs_image_path, sensor_data, meta_data)
store_fmu(fmu)
+
return {"status": "success", "fmu_id": fmu.id}
finally:
@@ -206,7 +230,7 @@ async def parse_natural_language_query(query_text: str):
"""
system_prompt = """
You are a Database Translator.
- Your goal: Convert natural language queries into a JSON filter object for a Hydroponic Database.
+ Your goal: Convert natural language queries (English, Hindi, Hinglish, etc.) into a JSON filter object for a Hydroponic Database.
AVAILABLE FIELDS:
- crop (e.g., Lettuce, Basil, Tomato)
@@ -216,15 +240,20 @@ async def parse_natural_language_query(query_text: str):
- crop_id (e.g., "Batch_Lettuce_2026")
RULES:
- 1. If user says "poor health", "bad", "failed" or similar negative words, map to outcome="Negative".
- 2. If user says "good", "healthy" or other positive words, map to outcome="Positive".
- 3. Output strictly JSON matching this structure:
+ 1. TRANSLATION: The user may speak Hindi or mixed "Hinglish". You must map these to the standard English tags.
+ - "Tamatar" -> crop: "Tomato"
+ - "Kharab" / "Sadd gaya" / "Bekar" -> outcome: "Negative"
+ - "Accha hai" / "Badhiya" -> outcome: "Positive"
+ - "Paani" / "Water" -> (No direct filter unless context implies outcome)
+ 2. If user says "poor health", "bad", "failed" or similar negative words, map to outcome="Negative".
+ 3. If user says "good", "healthy" or other positive words, map to outcome="Positive".
+ 4. Output strictly JSON matching this structure:
{
"must": [
{"key": "field_name", "match": "value"}
]
}
- 4. Return empty list [] if no specific filters apply.
+ 5. Return empty list [] if no specific filters apply.
"""
try:
@@ -289,4 +318,48 @@ async def process_text_query(text: str):
}
except Exception as e:
- return {"status": "error", "message": str(e)}
-\ No newline at end of file
+ return {"status": "error", "message": str(e)}
+
+async def process_audio_search(file: UploadFile):
+ """
+ 1. Transcribe Audio (Whisper) -> Text
+ 2. Run Text Search (LLM -> Filters)
+ """
+ temp_filename = f"temp_audio_{file.filename}"
+
+ # Save audio temporarily
+ with open(temp_filename, "wb") as buffer:
+ shutil.copyfileobj(file.file, buffer)
+
+ try:
+ print("šļø Transcribing audio (Multilingual)...")
+ audio_file = open(temp_filename, "rb")
+
+ # š CHANGE THIS MODEL
+ transcription = supervisor.llm.audio.transcriptions.create(
+ file=audio_file,
+ model="whisper-large-v3", # š Use the Multilingual Model (No "-en" suffix)
+ response_format="json",
+ prompt="The audio may contain English or Hindi technical terms about farming." # Optional hint
+ )
+
+ detected_text = transcription.text
+ print(f"š Heard ({transcription.language if hasattr(transcription, 'language') else 'auto'}): '{detected_text}'")
+
+ # 1. Get the standard search results
+ response_data = await process_text_query(detected_text)
+ # 2. š INJECT the transcription into the response
+ response_data["transcription"] = detected_text
+
+ return response_data
+
+ except Exception as e:
+ print(f"ā Audio Search Error: {e}")
+ return {"status": "error", "message": str(e)}
+
+ finally:
+ # Cleanup
+ if 'audio_file' in locals():
+ audio_file.close()
+ if os.path.exists(temp_filename):
+ os.remove(temp_filename)
+\ No newline at end of file
diff --git a/backend/server/main.py b/backend/server/main.py
@@ -13,7 +13,7 @@ from fastapi.middleware.cors import CORSMiddleware
from Sentinel.agent import FMUBuilder
# Import the logic functions
-from backend.server.functions import process_ingest, process_search, process_text_query
+from backend.server.functions import process_ingest, process_search, process_text_query, process_audio_search
app = FastAPI()
app.add_middleware(
@@ -64,6 +64,17 @@ async def search_endpoint(
async def text_query_endpoint(query: str = Form(...)):
return await process_text_query(query)
+@app.post("/query-audio")
+async def audio_query_endpoint(file: UploadFile = File(...)):
+ """
+ Accepts an audio file (webm/wav), transcribes it, and runs a search.
+ """
+ try:
+ return await process_audio_search(file)
+ except Exception as e:
+ print(f"ā Route Error: {e}")
+ return {"status": "error", "message": str(e)}
+
if __name__ == "__main__":
import uvicorn
uvicorn.run(app, host="0.0.0.0", port=8000)
\ No newline at end of file
diff --git a/web/app/upload/page.tsx b/web/app/upload/page.tsx
@@ -1,10 +1,10 @@
"use client";
-import { useState } from "react";
+import { useRef, useState } from "react";
import {
Upload, Save, Activity, Droplets, Thermometer, Wind, Search,
Sprout, Calendar, BarChart3, ArrowRight, Brain, ShieldCheck,
- CheckCircle, AlertTriangle
+ CheckCircle, AlertTriangle, Mic, Square
} from "lucide-react";
import { SensorData, SearchResult, AgentDecision } from "@/models"; // Ensure AgentDecision is exported in models
import { IngestService } from "@/services/api";
@@ -18,6 +18,10 @@ export default function UnifiedPage() {
const [searchResults, setSearchResults] = useState<SearchResult[]>([]);
const [textQuery, setTextQuery] = useState("");
+
+ const [isRecording, setIsRecording] = useState(false);
+ const mediaRecorderRef = useRef<MediaRecorder | null>(null);
+ const chunksRef = useRef<Blob[]>([]);
// š§ State for the Supervisor's Output
const [decision, setDecision] = useState<AgentDecision | null>(null);
@@ -127,6 +131,74 @@ export default function UnifiedPage() {
}
};
+ const startRecording = async () => {
+ try {
+ const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
+ mediaRecorderRef.current = new MediaRecorder(stream);
+ chunksRef.current = [];
+
+ mediaRecorderRef.current.ondataavailable = (e) => {
+ if (e.data.size > 0) chunksRef.current.push(e.data);
+ };
+
+ mediaRecorderRef.current.onstop = async () => {
+ const audioBlob = new Blob(chunksRef.current, { type: "audio/webm" });
+ await handleAudioUpload(audioBlob);
+
+ // Stop all tracks to release microphone
+ stream.getTracks().forEach(track => track.stop());
+ };
+
+ mediaRecorderRef.current.start();
+ setIsRecording(true);
+ } catch (err) {
+ console.error("Mic Error:", err);
+ alert("Microphone access denied.");
+ }
+ };
+
+ const stopRecording = () => {
+ if (mediaRecorderRef.current && isRecording) {
+ mediaRecorderRef.current.stop();
+ setIsRecording(false);
+ }
+ };
+
+ const handleAudioUpload = async (audioBlob: Blob) => {
+ setLoadingSearch(true);
+ setSearchResults([]);
+
+ try {
+ const formData = new FormData();
+ // Rename file to .webm so backend recognizes it
+ formData.append("file", audioBlob, "recording.webm");
+
+ const res = await fetch("http://localhost:8000/query-audio", {
+ method: "POST",
+ body: formData
+ });
+ const data = await res.json();
+ if (data.transcription) {
+ setTextQuery(data.transcription);
+ }
+ // (Reuse the same result mapping logic as Text Search)
+ if (data.results) {
+ // @ts-ignore
+ const mappedResults = data.results.map(r => ({
+ id: r.id,
+ score: 1.0,
+ payload: r.payload
+ }));
+ setSearchResults(mappedResults);
+ if (mappedResults.length === 0) alert("No records found for that audio query.");
+ }
+ } catch (e) {
+ console.error(e);
+ alert("Audio Query Failed");
+ } finally {
+ setLoadingSearch(false);
+ }
+ };
// --- UI Render ---
return (
@@ -160,6 +232,18 @@ export default function UnifiedPage() {
{/* š TEXT SEARCH BAR */}
<div className="max-w-6xl w-full mb-8">
<div className="bg-slate-900/50 p-4 rounded-xl border border-slate-700 flex gap-4">
+ {/* šļø MICROPHONE BUTTON */}
+ <button
+ onClick={isRecording ? stopRecording : startRecording}
+ className={`p-3 rounded-full transition-all ${
+ isRecording
+ ? "bg-red-500 animate-pulse text-white shadow-[0_0_15px_rgba(239,68,68,0.5)]"
+ : "bg-slate-800 text-slate-400 hover:bg-slate-700 hover:text-white"
+ }`}
+ title="Search by Voice"
+ >
+ {isRecording ? <Square className="w-5 h-5" /> : <Mic className="w-5 h-5" />}
+ </button>
<input
type="text"
value={textQuery}