commit 8455160627a3e337bd1322af04f9fd8e0721f322
parent ffa4ad93da86c35247a5165c40f6772f094ee714
Author: Debarghya Das <debarghya1108@gmail.com>
Date: Tue, 10 Mar 2026 17:36:07 +0530
Merge pull request #4 from maydayv7/Deb
Azure CV
Diffstat:
7 files changed, 161 insertions(+), 177 deletions(-)
diff --git a/agent/Marl/bandit.py b/agent/Marl/bandit.py
@@ -5,7 +5,7 @@ import pickle
import os
class ContextualBandit:
- def __init__(self, n_actions=15, feature_dim=519):
+ def __init__(self, n_actions=15, feature_dim=515):
"""
LinGreedy Implementation (Pure Exploitation).
We removed 'alpha' because we do not want to explore.
diff --git a/agent/sub_agents/Doctor.py b/agent/sub_agents/Doctor.py
@@ -1,103 +1,93 @@
-from ultralytics import YOLO
-import cv2
-import json
import os
+import requests
import logging
-import numpy as np
import base64
-from io import BytesIO
-from PIL import Image
+from dotenv import load_dotenv
-# Setup basic logging
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
class VisionAgent:
- def __init__(self, model_path=None):
- logger.info("👁️ Initializing Vision Agent (Doctor)...")
+ def __init__(self):
+ load_dotenv()
+ self.endpoint = os.getenv("AZURE_ENDPOINT", "").rstrip('/')
+ self.prediction_key = os.getenv("AZURE_PREDICTION_KEY")
+ self.project_id = os.getenv("AZURE_PROJECT_ID")
+ self.iteration_name = os.getenv("AZURE_ITERATION_NAME")
+ self.model_name = "azure_custom_vision"
- # 1. Find the project root
- if model_path:
- default_model = model_path
+ if not all([self.endpoint, self.prediction_key, self.project_id, self.iteration_name]):
+ logger.error("Missing Azure Custom Vision environment variables.")
+
+ def analyze_frame(self, image_input, threshold=0.25):
+ if not image_input:
+ return {"error": "No image input provided"}
+
+ image_data_bytes = None
+
+ if isinstance(image_input, str) and len(image_input) < 1000 and os.path.exists(image_input):
+ try:
+ with open(image_input, "rb") as f:
+ image_data_bytes = f.read()
+ except Exception as e:
+ return {"error": str(e)}
else:
- current_file = os.path.abspath(__file__)
- agent_dir = os.path.dirname(os.path.dirname(current_file)) # agent/
- default_model = os.path.join(agent_dir, "model", "plant_disease_model.pt")
-
- self.model_name = default_model
-
- # 2. Load Model with Fallback
+ try:
+ if ',' in image_input:
+ image_input = image_input.split(',')[1]
+ image_data_bytes = base64.b64decode(image_input)
+ except Exception as e:
+ return {
+ "status": "Error",
+ "model_used": self.model_name,
+ "health_assessment": "UNKNOWN",
+ "visual_alert": False,
+ "object_counts": {},
+ "detailed_detections": [],
+ "error": str(e)
+ }
+
try:
- if os.path.exists(self.model_name):
- logger.info(f"✅ Found plant disease model at: {self.model_name}")
- self.model = YOLO(self.model_name)
- else:
- logger.warning(f"⚠️ Custom model not found. Using generic YOLOv8n.")
- self.model = YOLO("yolov8n.pt")
- self.model_name = "yolov8n.pt"
+ url = f"{self.endpoint}/customvision/v3.0/Prediction/{self.project_id}/detect/iterations/{self.iteration_name}/image"
- # Optimization
- self.model.to('cpu')
-
- except Exception as e:
- logger.error(f"❌ Critical Error loading model: {e}")
- self.model = None
-
- def analyze_frame(self, image_b64):
- """
- Scans a base64 encoded image for pests, diseases, or growth stages.
- """
- if not self.model:
- return {"error": "Model not initialized"}
+ headers = {
+ "Prediction-Key": self.prediction_key,
+ "Content-Type": "application/octet-stream"
+ }
- if not image_b64:
- return {"error": "No image data provided"}
-
- try:
- # 3. Decode Base64 to Image
- # Handle data URI scheme if present (e.g., "data:image/png;base64,...")
- if "," in image_b64:
- image_b64 = image_b64.split(",")[1]
+ response = requests.post(url, headers=headers, data=image_data_bytes)
+ response.raise_for_status()
- image_data = base64.b64decode(image_b64)
- image = Image.open(BytesIO(image_data))
-
- # 4. Run Inference
- # YOLO can accept PIL Images directly
- results = self.model.predict(image, conf=0.25, save=False, verbose=False)
- result = results[0]
+ predictions = response.json().get("predictions", [])
detections = []
summary_counts = {}
- for box in result.boxes:
- class_id = int(box.cls[0])
- label = self.model.names[class_id]
- confidence = float(box.conf[0])
-
- detections.append({
- "object": label,
- "confidence": round(confidence, 2),
- "box": [round(x, 2) for x in box.xywhn[0].tolist()]
- })
- summary_counts[label] = summary_counts.get(label, 0) + 1
+ for p in predictions:
+ confidence = p["probability"]
+ if confidence >= threshold:
+ label = p["tagName"]
+ box = p["boundingBox"]
+
+ detections.append({
+ "object": label,
+ "confidence": round(confidence, 2),
+ "box": [
+ round(box["left"], 2),
+ round(box["top"], 2),
+ round(box["width"], 2),
+ round(box["height"], 2)
+ ]
+ })
+ summary_counts[label] = summary_counts.get(label, 0) + 1
- # 5. Health Logic
health_status = "HEALTHY"
visual_alert = False
- if not detections:
- # If generic model, it might just see nothing.
- # If disease model, empty usually means healthy.
- if "yolov8n" in self.model_name:
- health_status = "NO_OBJECTS_DETECTED"
- else:
- health_status = "HEALTHY"
- else:
+ if detections:
for label in summary_counts:
label_lower = label.lower()
- # Keywords that imply sickness
- sick_keywords = ['spot', 'rot', 'blight', 'mildew', 'rust', 'virus', 'miner', 'mite', 'wilt']
+ sick_keywords = ['spot', 'rot', 'blight', 'mildew', 'rust', 'virus', 'miner', 'mite', 'wilt', 'aphid']
if any(x in label_lower for x in sick_keywords) and "healthy" not in label_lower:
health_status = "DISEASE_DETECTED"
visual_alert = True
@@ -113,6 +103,13 @@ class VisionAgent:
}
except Exception as e:
-
- logger.error(f"Error during analysis: {e}")
- return {"error": str(e)}
-\ No newline at end of file
+ logger.error(f"Error during Azure analysis: {e}")
+ return {
+ "status": "Error",
+ "model_used": self.model_name,
+ "health_assessment": "UNKNOWN",
+ "visual_alert": False,
+ "object_counts": {},
+ "detailed_detections": [],
+ "error": str(e)
+ }
+\ No newline at end of file
diff --git a/agent/sub_agents/Supervisor.py b/agent/sub_agents/Supervisor.py
@@ -63,13 +63,13 @@ API_KEY = os.environ.get("GROQ_API_KEY")
class SupervisorAgent:
def __init__(self, researcher_agent=None):
self.name = "Supervisor"
- self.bandit = ContextualBandit(n_actions=NUM_ACTIONS, feature_dim=519)
+ self.bandit = ContextualBandit(n_actions=NUM_ACTIONS, feature_dim=515)
if API_KEY:
self.model = ChatOpenAI(
base_url="https://api.groq.com/openai/v1",
api_key=API_KEY,
- model="qwen/qwen3-32b",
+ model="llama-3.3-70b-versatile",
temperature=0.0 # Zero temp for strict judging
)
@@ -149,14 +149,11 @@ class SupervisorAgent:
ADVISORY STRATEGY: {state['strategy_advice']}
TASK:
- 1. BIAS: You should almost always APPROVE.
- 2. ONLY 'REJECT' if the plan is physically impossible or immediately fatal (e.g., pH < 3.0, Water Temp > 40°C).
- 3. IGNORE 'Simulation Fail' warnings if the Strategy justifies the extreme values (e.g., 'Flush' requires low EC).
- 4. Treat "Risk" warnings as acceptable trade-offs for the strategy.
+ 1. If the failures are dangerous (Toxic pH, Thermal Shock, Low Health), REJECT the plan.
+ 2. If the failures are minor or necessary for the Strategy (e.g., Low Humidity required for 'Fungal Treatment'), APPROVE it.
+
OUTPUT JSON: {{ "verdict": "APPROVE" or "REJECT", "critique": "Explanation..." }}
"""
-
- print("Supervisor Prompt:\n", prompt)
try:
response = self.model.invoke([HumanMessage(content=prompt)])
@@ -171,9 +168,6 @@ class SupervisorAgent:
# Default to reject if unsafe
return {"final_decision": "REJECT", "critique": "Plan failed automated safety checks."}
-
-
-
# --- ENTRY POINT ---
def synthesize_plan(self, atmos_plan, water_plan, fmu, history, strategy_info):
@@ -193,7 +187,7 @@ class SupervisorAgent:
result = self.app.invoke(initial_state)
final_targets = result.get("merged_plan", {})
- current_sensors = fmu.metadata.get('sensors', {})
+ current_sensors = fmu.metadata.get('sensor_data', {})
print(f"[{self.name}] ⚙️ Converting Targets to Actuator Commands...")
diff --git a/agent/sub_agents/fetching_agent.py b/agent/sub_agents/fetching_agent.py
@@ -13,7 +13,7 @@ from Qdrant.Store import COLLECTION_NAME
from Qdrant.Client import client
class FetchingAgent:
- def __init__(self, simulator_url="https://unexhumed-melaine-bouncingly.ngrok-free.dev/simulation/state"):
+ def __init__(self, simulator_url="https://unexhumed-melaine-bouncingly.ngrok-free.dev/azure/state"):
self.sim_url = simulator_url
self.builder = FMUBuilder()
@@ -26,12 +26,19 @@ class FetchingAgent:
if response.status_code == 200:
data = response.json()
- # 1. Extract Data
window_data = data.get("sensor_window", {})
image_b64 = data.get("image", "")
raw_meta = data.get("metadata", {})
- # 2. Filter Sensors
+ if not image_b64:
+ from PIL import Image
+ import base64
+ from io import BytesIO
+ img = Image.new('RGB', (512, 512), (50, 50, 50))
+ buf = BytesIO()
+ img.save(buf, format="PNG")
+ image_b64 = base64.b64encode(buf.getvalue()).decode("utf-8")
+
wanted_keys = {"ph": "pH", "ec": "EC", "humidity": "humidity", "temp": "temp", "air_temp": "temp"}
sensor_snapshot = {}
for key, value_list in window_data.items():
@@ -41,7 +48,6 @@ class FetchingAgent:
val = value_list[-1] if isinstance(value_list, list) and value_list else 0.0
sensor_snapshot[out_name] = val
- # 3. Calculate Sequence & Prepare Metadata
crop_id = raw_meta.get("crop_id", "UNKNOWN_CROP")
next_seq = self._get_next_sequence(crop_id)
print(f"[Fetcher] 🔢 Sequence for {crop_id}: {next_seq}")
@@ -51,26 +57,23 @@ class FetchingAgent:
"stage": raw_meta.get("stage", "unknown"),
"crop_id": crop_id,
"sequence_number": next_seq,
- # Store raw image for JudgeAgent (since we don't save to disk)
"image_b64": image_b64
}
- # 4. Create FMU (BUT DO NOT STORE)
fmu = self.builder.create_fmu(image_b64, sensor_snapshot, filtered_metadata)
print(f"[Fetcher] 🧠 FMU Created (ID: {fmu.id}) - Handing off to Judge.")
- # 5. Historian Search (Optional context for Researcher)
search_results = self.find_similar_instances(fmu)
return fmu, sensor_snapshot, search_results, image_b64
else:
print(f"[Fetcher] ❌ Error: Simulator returned {response.status_code}")
- return None, None, None
+ return None, None, None, None
except Exception as e:
print(f"[Fetcher] ❌ Critical Error: {e}")
- return None, None, None
+ return None, None, None, None
def _get_next_sequence(self, crop_id):
"""Queries Qdrant for count of existing points for this crop_id."""
diff --git a/agent/sub_agents/judge_agent.py b/agent/sub_agents/judge_agent.py
@@ -1,7 +1,6 @@
import os
import json
import base64
-import re
import tempfile
from typing import TypedDict, Dict, Any, Optional
@@ -15,14 +14,12 @@ from agent.sub_agents.base_agent import BaseReasoningAgent
from Qdrant.Client import client
from Qdrant.Store import COLLECTION_NAME
-# Import farm_memory to allow writing verdicts
-from agent.sub_agents.water_and_atmospheric_dependencies.retrieval import diagnose_plant, ask_memory, farm_memory
+from agent.sub_agents.water_and_atmospheric_dependencies.retrieval import diagnose_plant, ask_memory
# --- STATE DEFINITION ---
class JudgeState(TypedDict):
# Inputs
current_fmu: Any
- image_b64: str # Added to State
# Internal Context
prev_point: Any
@@ -49,7 +46,7 @@ class JudgeAgent(BaseReasoningAgent):
self.llm = ChatOpenAI(
base_url="https://api.groq.com/openai/v1",
api_key=os.environ.get("GROQ_API_KEY"),
- model="qwen/qwen3-32b",
+ model="llama-3.3-70b-versatile",
temperature=0.1
)
@@ -57,13 +54,23 @@ class JudgeAgent(BaseReasoningAgent):
def _build_graph(self):
workflow = StateGraph(JudgeState)
+
+ # 1. Retrieve: Get N-1 state from Qdrant
workflow.add_node("retrieve_evidence", self.node_retrieve_evidence)
+
+ # 2. Investigate: Run Tools (Vision & Memory)
workflow.add_node("run_forensics", self.node_run_forensics)
+
+ # 3. Deliberate: LLM Synthesis
workflow.add_node("deliberate", self.node_deliberate)
+
+ # 4. Update: Write to DBs
workflow.add_node("file_verdict", self.node_file_verdict)
+ # Flow
workflow.set_entry_point("retrieve_evidence")
+ # Conditional: If no history, skip to end
workflow.add_conditional_edges(
"retrieve_evidence",
lambda x: "run_forensics" if x.get("prev_point") else "end_no_history",
@@ -82,6 +89,9 @@ class JudgeAgent(BaseReasoningAgent):
# --- NODES ---
def node_retrieve_evidence(self, state: JudgeState):
+ """
+ Finds the previous cycle (N-1) to compare against.
+ """
print(f"[{self.name}] 🕵️ Retrieve Evidence...")
fmu = state["current_fmu"]
crop_id = fmu.metadata.get("crop_id")
@@ -92,6 +102,7 @@ class JudgeAgent(BaseReasoningAgent):
return {"prev_point": None}
prev_seq = current_seq - 1
+
try:
s_filter = models.Filter(
must=[
@@ -106,6 +117,7 @@ class JudgeAgent(BaseReasoningAgent):
with_vectors=True
)
return {"prev_point": res[0] if res else None, "crop_id": crop_id}
+
except Exception as e:
print(f" -> DB Error: {e}")
return {"prev_point": None}
@@ -113,43 +125,49 @@ class JudgeAgent(BaseReasoningAgent):
def node_run_forensics(self, state: JudgeState):
"""
Executes the TWO mandated tools: diagnose_plant and ask_memory.
+ Added robust error handling for the new Azure API dependency.
"""
print(f"[{self.name}] 🔎 Running Forensics...")
+ prev_point = state["prev_point"]
crop_id = state["crop_id"]
- image_b64 = state.get("image_b64")
-
- # --- TOOL 1: diagnose_plant ---
- visual_data = {"status": "No Image"}
+
+ # --- TOOL 1: diagnose_plant (Azure Custom Vision) ---
+ visual_data = {"status": "No Image", "health_assessment": "UNKNOWN"}
+ image_b64 = prev_point.payload.get("image_b64")
+
if image_b64:
+ temp_path = None
try:
+ # Create temp file for the tool
with tempfile.NamedTemporaryFile(suffix=".jpg", delete=False) as temp:
temp.write(base64.b64decode(image_b64))
temp_path = temp.name
- print(f" -> Invoking Tool inside: diagnose_plant")
- visual_data = diagnose_plant.invoke({"image_b64": image_b64})
- os.remove(temp_path)
+ print(f" -> Invoking Tool: diagnose_plant (Azure)")
+ visual_data = diagnose_plant.invoke({"image_path": temp_path})
+
+ # Handle unexpected API failure in tool response
+ if "error" in visual_data:
+ print(f" -> Warning: Azure diagnosis failed: {visual_data['error']}")
+ visual_data = {"status": "Error", "health_assessment": "UNKNOWN"}
+
except Exception as e:
- visual_data = {"error": str(e)}
- else:
- print(" -> No image found for diagnosis.")
+ print(f" -> Critical Tool Error: {e}")
+ visual_data = {"status": "Error", "health_assessment": "UNKNOWN"}
+
+ finally:
+ if temp_path and os.path.exists(temp_path):
+ os.remove(temp_path)
# --- TOOL 2: ask_memory ---
- # print(f" -> Invoking Tool: ask_memory for '{crop_id}'")
+ query = f"What is the health history and past treatments for {crop_id}?"
+ print(f" -> Invoking Tool: ask_memory")
try:
- # We updated the tool to expect 'plant_id', so we must pass that key
- raw_memory = ask_memory.invoke({"plant_id": crop_id})
- # FIX: Force conversion to string to ensure it renders in prompt
- # print(f" -> Memory Retrieved: '{raw_memory}'")
- memory_data = str(raw_memory)
-
- # print(f" -> Memory Retrieved {memory_data}")
- self
+ memory_data = ask_memory.invoke({"query": query})
except Exception as e:
- print(f" -> Memory Tool Error: {e}")
+ print(f" -> Memory Retrieval Failed: {e}")
memory_data = "Memory unavailable."
- # Explicitly return the dict to update state keys
return {
"visual_report": visual_data,
"biography": memory_data
@@ -160,16 +178,11 @@ class JudgeAgent(BaseReasoningAgent):
LLM synthesizes Visual + History + Sensor Delta to form a verdict.
"""
print(f"[{self.name}] ⚖️ Deliberating...")
-
- # --- DEBUG: Verify State Content ---
- # print(f"DEBUG CHECK -> Biography Content: '{state.get('biography')}'")
- prev_sensors = state["prev_point"].payload.get("sensors", {})
- curr_sensors = state["current_fmu"].metadata.get("sensors", {})
+ prev_sensors = state["prev_point"].payload.get("sensor_data", {})
+ curr_sensors = state["current_fmu"].metadata.get("sensor_data", {})
visual = state["visual_report"]
-
- # Ensure history is never None
- history = state.get("biography", "No history available.")
+ history = state["biography"]
prompt = f"""
You are the Chief Judge of an Automated Farm.
@@ -194,30 +207,12 @@ class JudgeAgent(BaseReasoningAgent):
Output JSON: {{ "outcome": "IMPROVED"|"DETERIORATED"|"STABLE", "reward": float(-1.0 to 1.0), "reason": "Short explanation" }}
"""
- # print("Judge Prompt:\n", prompt)
try:
response = self.llm.invoke([HumanMessage(content=prompt)])
- # print(f" -> LLM Response for Judge Deliberation: {response.content}")
- content = response.content.strip()
-
- # print(f" -> Raw LLM Output: '{content}'")
- code_block_match = re.search(r"```json\s*(\{.*?\})\s*```", content, re.DOTALL)
+ content = response.content.replace("```json", "").replace("```", "").strip()
+ verdict = json.loads(content)
- if code_block_match:
- json_str = code_block_match.group(1)
- else:
- # 2. Fallback: Find the first '{' and the last '}'
- # This handles cases where the LLM forgets the ```json tags
- json_match = re.search(r"\{.*\}", content, re.DOTALL)
- if json_match:
- json_str = json_match.group(0)
- else:
- raise ValueError(f"No JSON found in response: {content[:50]}...")
-
- # 3. Parse
- verdict = json.loads(json_str)
-
- print(f" -> Verdict: {verdict.get('outcome')} ({verdict.get('reward')})")
+ print(f" -> Verdict: {verdict['outcome']} ({verdict['reward']})")
return {
"outcome": verdict.get("outcome", "STABLE"),
"reward": verdict.get("reward", 0.0),
@@ -229,14 +224,13 @@ class JudgeAgent(BaseReasoningAgent):
def node_file_verdict(self, state: JudgeState):
"""
- Writes the final judgment to Qdrant AND FarmMemory.
+ Writes the final judgment to Qdrant.
"""
print(f"[{self.name}] 📝 Filing Verdict...")
prev_id = state["prev_point"].id
- crop_id = state["crop_id"]
- # 1. Update Qdrant Snapshot
+ # Update Qdrant Snapshot
self.qdrant.set_payload(
collection_name=COLLECTION_NAME,
points=[prev_id],
@@ -247,16 +241,6 @@ class JudgeAgent(BaseReasoningAgent):
"visual_diagnosis": str(state["visual_report"].get("health_assessment", "N/A"))
}
)
-
- # 2. Write to FarmMemory (Text/Biography Store)
- try:
- verdict_summary = (
- f"Cycle Review for {crop_id}: Result was {state['outcome']} "
- f"(Reward: {state['reward']}). Judge's Note: {state['explanation']}"
- )
- farm_memory.log_event(crop_id, verdict_summary)
- except Exception as e:
- print(f" -> ⚠️ Failed to log to FarmMemory: {e}")
# Prepare Training Data Bundle
training_data = {
@@ -269,14 +253,16 @@ class JudgeAgent(BaseReasoningAgent):
return {"training_data": training_data}
# --- ENTRY POINT ---
- def review_previous_cycle(self, current_fmu: FMU, image_b64: str):
+ def review_previous_cycle(self, current_fmu: FMU):
+ """
+ The public API called by the main system.
+ """
initial_state = {
"current_fmu": current_fmu,
- "image_b64": image_b64,
"prev_point": None,
"crop_id": "",
"visual_report": {},
- "biography": "", # Starts empty
+ "biography": "",
"reward": 0.0,
"outcome": "",
"explanation": "",
diff --git a/agent/sub_agents/water_and_atmospheric_dependencies/physics_engine.py b/agent/sub_agents/water_and_atmospheric_dependencies/physics_engine.py
@@ -4,7 +4,7 @@ from langchain_openai import ChatOpenAI
from langchain_core.messages import SystemMessage, HumanMessage
# Configuration
-API_KEY = os.environ.get("GROQ_API_KEY1")
+API_KEY = os.environ.get("GROQ_API_KEY")
MODEL_ID = "qwen/qwen3-32b" # Using the latest supported Groq model
def predict_outcome(current_state: dict, proposed_action: dict) -> dict:
@@ -21,8 +21,9 @@ def predict_outcome(current_state: dict, proposed_action: dict) -> dict:
base_url="https://api.groq.com/openai/v1",
api_key=API_KEY,
model=MODEL_ID,
- temperature=0.1, # Low temp for consistent physics logic
- max_tokens=1024
+ temperature=0.1,
+ max_tokens=1024,
+ model_kwargs={"reasoning_effort": "none"}
)
system_prompt = (
@@ -50,6 +51,9 @@ def predict_outcome(current_state: dict, proposed_action: dict) -> dict:
# Clean and Parse JSON
content = response.content.replace("```json", "").replace("```", "").strip()
+ if not content:
+ raise ValueError("Empty response from model")
+
result = json.loads(content)
# Default fallback keys if the LLM misses them
diff --git a/backend/server/functions.py b/backend/server/functions.py
@@ -22,7 +22,7 @@ from Qdrant.Store import store_fmu, COLLECTION_NAME
from Qdrant.Client import client
# --- INITIALIZE COGNITIVE STACK ---
-print("🌱 Initializing Demeter Cognitive Stack (Bandit Disabled)...")
+print("🌱 Initializing Demeter Cognitive Stack....")
researcher = ResearcherAgent()
atmos_agent = AtmosphericAgent()