creek
The AI Image Editor of 2030git clone https://github.com/maydayv7/creek
commit e2ec9d7abe9bce68ba1ef3d00a42604c75c12930 parent 1231e4808273b1708a4943de17ef29609ab14b15 Author: Nilotpal Gupta <nilotpalgupta0701@gmail.com> Date: Thu, 27 Nov 2025 19:29:24 +0530 Merge pull request #9 from nilotpal-n7/dev hybrid parallelism + unified json result structure
Diffstat:
19 files changed, 506 insertions(+), 309 deletions(-)
diff --git a/android/app/src/main/python/analyze_layout.py b/android/app/src/main/python/analyze_layout.py @@ -229,13 +229,15 @@ def analyze_single_image(img_path): # 5. Sort and get top 5 sorted_features = sorted(scores.items(), key=lambda x: x[1], reverse=True) - top5 = [{"name": k, "score": v} for k, v in sorted_features[:5]] + top5 = {} + for k, v in sorted_features[:5]: + top5[k] = float(v) # 6. Return JSON result result = { "success": True, - "scores": {k: float(v) for k, v in scores.items()}, - "top5": top5, + "predictions": {k: float(v) for k, v in scores.items()}, + "best": top5, "image_name": os.path.basename(img_path) } diff --git a/android/app/src/main/python/color_style_infer.py b/android/app/src/main/python/color_style_infer.py @@ -181,21 +181,15 @@ def analyze_color_style(image_path): probs = clf.predict_proba([flat_vector])[0] classes = le.classes_ - # Format Results - prediction = classes[np.argmax(probs)] - score = max(probs) - - results = [] + results = {} for c, p in zip(classes, probs): - results.append({"label": c, "score": p}) - - results = sorted(results, key=lambda x: x["score"], reverse=True)[:3] + results[c] = float(p) + sorted_features = sorted(results.items(), key=lambda x: x[1], reverse=True) response = { "success": True, - "predictions": results, - "top_label": prediction, - "top_score": score + "predictions": {k: float(v) for k, v in results.items()}, + "best": {sorted_features[0][0]: float(sorted_features[0][1])}, } return json.dumps(response, cls=NumpyEncoder) diff --git a/devtools_options.yaml b/devtools_options.yaml @@ -0,0 +1,3 @@ +description: This file stores settings for Dart & Flutter DevTools. +documentation: https://docs.flutter.dev/tools/devtools/extensions#configure-extension-enablement-states +extensions: diff --git a/lib/services/color_analyzer_service.dart b/lib/services/color_analyzer_service.dart @@ -1,3 +1,5 @@ +// lib/services/color_analyzer_service.dart + import 'package:flutter/foundation.dart'; import 'python_service.dart'; diff --git a/lib/services/embedding_analyzer_service.dart b/lib/services/embedding_analyzer_service.dart @@ -1,36 +1,32 @@ +// lib/services/embedding_analyzer_service.dart + import 'dart:convert'; -import 'dart:math'; import 'dart:io'; +import 'package:adobe/utils/image_utils.dart'; import 'package:flutter/foundation.dart'; -import 'package:flutter/services.dart'; -import 'package:flutter_onnxruntime/flutter_onnxruntime.dart'; -import 'package:path_provider/path_provider.dart'; +import 'package:onnxruntime/onnxruntime.dart'; import '../utils/clip_image_processor.dart'; class EmbeddingAnalyzerService { - static OrtSession? _session; - static List<List<double>>? _centroids; - static List<String>? _classes; + OrtSession? _session; + List<List<double>>? _centroids; + List<String>? _classes; - static Future<void> initialize() async { + Future<void> initialize({required String modelPath, required String jsonPath}) async { if (_session != null) return; try { debugPrint("Initializing Embeddings (Dino/CLIP)..."); - final rawAsset = await rootBundle.load('assets/clip_image_encoder.onnx'); - final dir = await getTemporaryDirectory(); - final modelFile = File('${dir.path}/clip_model.onnx'); - - if (!await modelFile.exists()) { - await modelFile.writeAsBytes(rawAsset.buffer.asUint8List()); - } + // Initialize FFI Env + OrtEnv.instance.init(); - // Initialize runtime - final ort = OnnxRuntime(); - _session = await ort.createSession(modelFile.path); + // 1. Load Model from passed PATH (No rootBundle!) + final sessionOptions = OrtSessionOptions(); + _session = OrtSession.fromFile(File(modelPath), sessionOptions); + sessionOptions.release(); - final jsonString = await rootBundle.loadString('assets/style_embeddings.json'); + final jsonString = await File(jsonPath).readAsString(); final jsonData = json.decode(jsonString); _classes = List<String>.from(jsonData['classes']); @@ -40,11 +36,18 @@ class EmbeddingAnalyzerService { } } - static Future<Map<String, dynamic>?> analyze(String imagePath) async { - if (_session == null) await initialize(); + Future<Map<String, dynamic>?> analyze(String imagePath, { + String? modelPath, + String? jsonPath + }) async { + // Safety check + if (modelPath == null || jsonPath == null) return null; + await initialize(modelPath: modelPath, jsonPath: jsonPath); + if (_session == null || _centroids == null) return null; - OrtValue? inputOrt; - OrtValue? outputOrt; + OrtValueTensor? inputOrt; + OrtRunOptions? runOptions; + List<OrtValue?>? outputs; try { final float32Input = await ClipImageProcessor.preprocess(imagePath); @@ -52,24 +55,24 @@ class EmbeddingAnalyzerService { // Create Tensor // Note: ensure ClipImageProcessor returns a flat List<double> - inputOrt = await OrtValue.fromList( + inputOrt = OrtValueTensor.createTensorWithDataList( float32Input, [1, 3, 224, 224] ); + runOptions = OrtRunOptions(); + // Run Inference - // 'image' is the input name for CLIP. Check your model if it differs (e.g. 'input') - final outputs = await _session!.run({"image": inputOrt}); + // 'image' is the input name for CLIP. + outputs = _session!.run(runOptions, {"image": inputOrt}); - // Get Output - outputOrt = outputs['output']; // or outputs['image_features'] depending on model - // If outputOrt is null, try iterating outputs.values.first - final resultOrt = outputOrt ?? outputs.values.first; + if (outputs.isEmpty) throw Exception("No output from model"); - if (resultOrt == null) throw Exception("No output from model"); + // Get Output + // FFI returns list of outputs. Usually index 0. + final dynamic outputRaw = outputs[0]?.value; // Flatten Output - final List<dynamic> rawList = await resultOrt.asList(); final List<double> imgFeat = []; void flatten(dynamic data) { @@ -81,48 +84,39 @@ class EmbeddingAnalyzerService { } } } - flatten(rawList); + flatten(outputRaw); // Normalize & Compare - final normFeat = _l2Normalize(imgFeat); - List<Map<String, dynamic>> scores = []; + final normFeat = l2Normalize(imgFeat); + Map<String, double> scores = {}; for (int i = 0; i < _classes!.length; i++) { - double score = _dotProduct(normFeat, _centroids![i]); - scores.add({"name": _classes![i], "score": score * 100.0}); + double score = dotProduct(normFeat, _centroids![i]); + scores[_classes![i]] = score * 100.0; } - scores.sort((a, b) => (b['score'] as double).compareTo(a['score'] as double)); + scores = Map.fromEntries( + scores.entries.toList() + ..sort((a, b) => b.value.compareTo(a.value)) + ); return { "success": true, - "label": scores.first['name'], - "top5": scores.take(5).toList(), + "predictions": scores, + "best": {scores.keys.first: scores.values.first}, }; } catch (e) { debugPrint("Embeddings Analysis Error: $e"); return null; } finally { - inputOrt?.dispose(); - outputOrt?.dispose(); + inputOrt?.release(); + runOptions?.release(); + outputs?.forEach((element) => element?.release()); } } - static List<double> _l2Normalize(List<double> vec) { - double sum = vec.fold(0, (p, c) => p + c * c); - double norm = sqrt(sum); - if (norm == 0) return vec; - return vec.map((v) => v / norm).toList(); - } - - static double _dotProduct(List<double> a, List<double> b) { - double sum = 0; - // Ensure lengths match to avoid range errors - int len = min(a.length, b.length); - for (int i = 0; i < len; i++) { - sum += a[i] * b[i]; - } - return sum; + void dispose() { + _session?.release(); } } diff --git a/lib/services/emotional_embeddings_service.dart b/lib/services/emotional_embeddings_service.dart @@ -1,38 +1,31 @@ +// lib/services/emotional_embeddings_service.dart + import 'dart:convert'; -import 'dart:math'; import 'dart:io'; +import 'package:adobe/utils/image_utils.dart'; import 'package:flutter/foundation.dart'; -import 'package:flutter/services.dart'; -import 'package:flutter_onnxruntime/flutter_onnxruntime.dart'; -import 'package:path_provider/path_provider.dart'; +import 'package:onnxruntime/onnxruntime.dart'; import '../utils/clip_image_processor.dart'; class EmotionalEmbeddingsService { - static OrtSession? _session; - static List<List<double>>? _centroids; - static List<String>? _classes; + OrtSession? _session; + List<List<double>>? _centroids; + List<String>? _classes; - static Future<void> initialize() async { + Future<void> initialize({required String modelPath, required String jsonPath}) async { if (_session != null) return; try { debugPrint("Initializing Embeddings (Dino/CLIP)..."); + + OrtEnv.instance.init(); - final rawAsset = await rootBundle.load('assets/clip_image_encoder.onnx'); - final dir = await getTemporaryDirectory(); - final modelFile = File('${dir.path}/clip_model.onnx'); - - if (!await modelFile.exists()) { - await modelFile.writeAsBytes(rawAsset.buffer.asUint8List()); - } - - // Initialize runtime - final ort = OnnxRuntime(); - _session = await ort.createSession(modelFile.path); + // 1. Load Model from passed PATH (No rootBundle!) + final sessionOptions = OrtSessionOptions(); + _session = OrtSession.fromFile(File(modelPath), sessionOptions); + sessionOptions.release(); - final jsonString = await rootBundle.loadString( - 'assets/emotion_centroids.json', - ); + final jsonString = await File(jsonPath).readAsString(); final jsonData = json.decode(jsonString); _classes = List<String>.from(jsonData['classes']); @@ -45,11 +38,18 @@ class EmotionalEmbeddingsService { } } - static Future<Map<String, dynamic>?> analyze(String imagePath) async { - if (_session == null) await initialize(); + Future<Map<String, dynamic>?> analyze(String imagePath, { + String? modelPath, + String? jsonPath + }) async { + // Safety check + if (modelPath == null || jsonPath == null) return null; + await initialize(modelPath: modelPath, jsonPath: jsonPath); + if (_session == null || _centroids == null) return null; - OrtValue? inputOrt; - OrtValue? outputOrt; + OrtValueTensor? inputOrt; + OrtRunOptions? runOptions; + List<OrtValue?>? outputs; try { final float32Input = await ClipImageProcessor.preprocess(imagePath); @@ -58,23 +58,22 @@ class EmotionalEmbeddingsService { // Create Tensor // Note: ensure ClipImageProcessor returns a flat List<double> - inputOrt = await OrtValue.fromList(float32Input, [1, 3, 224, 224]); + inputOrt = OrtValueTensor.createTensorWithDataList( + float32Input, [1, 3, 224, 224]); + + runOptions = OrtRunOptions(); // Run Inference - // 'image' is the input name for CLIP. Check your model if it differs (e.g. 'input') - final outputs = await _session!.run({"image": inputOrt}); + // 'image' is the input name for CLIP. + outputs = _session!.run(runOptions, {"image": inputOrt}); //debugPrint("[EMOTION] Output keys: ${outputs.keys}"); - // Get Output - outputOrt = - outputs['output']; // or outputs['image_features'] depending on model - // If outputOrt is null, try iterating outputs.values.first - final resultOrt = outputOrt ?? outputs.values.first; + if (outputs.isEmpty) throw Exception("No output from model"); - if (resultOrt == null) throw Exception("No output from model"); + // Get Output + final dynamic outputRaw = outputs[0]?.value; // Flatten Output - final List<dynamic> rawList = await resultOrt.asList(); final List<double> imgFeat = []; void flatten(dynamic data) { @@ -87,52 +86,41 @@ class EmotionalEmbeddingsService { } } - flatten(rawList); + flatten(outputRaw); // debugPrint("[EMOTION] Embedding length: ${imgFeat.length}"); // debugPrint("[EMOTION] Centroid length: ${_centroids?[0].length}"); // Normalize & Compare - final normFeat = _l2Normalize(imgFeat); - List<Map<String, dynamic>> scores = []; + final normFeat = l2Normalize(imgFeat); + Map<String, dynamic> scores = {}; for (int i = 0; i < _classes!.length; i++) { - double score = _dotProduct(normFeat, _centroids![i]); - scores.add({"name": _classes![i], "score": score * 100.0}); + double score = dotProduct(normFeat, _centroids![i]); + scores[_classes![i]] = score * 100.0; } - scores.sort( - (a, b) => (b['score'] as double).compareTo(a['score'] as double), + scores = Map.fromEntries( + scores.entries.toList() + ..sort((a, b) => b.value.compareTo(a.value)) ); return { "success": true, - "label": scores.first['name'], - "top5": scores.take(5).toList(), + "predictions": scores, + "best": {scores.keys.first: scores.values.first}, }; } catch (e) { debugPrint("Embeddings Analysis Error: $e"); return null; } finally { - inputOrt?.dispose(); - outputOrt?.dispose(); + inputOrt?.release(); + runOptions?.release(); + outputs?.forEach((element) => element?.release()); } } - static List<double> _l2Normalize(List<double> vec) { - double sum = vec.fold(0, (p, c) => p + c * c); - double norm = sqrt(sum); - if (norm == 0) return vec; - return vec.map((v) => v / norm).toList(); - } - - static double _dotProduct(List<double> a, List<double> b) { - double sum = 0; - // Ensure lengths match to avoid range errors - int len = min(a.length, b.length); - for (int i = 0; i < len; i++) { - sum += a[i] * b[i]; - } - return sum; + void dispose() { + _session?.release(); } } diff --git a/lib/services/image_analyzer_service.dart b/lib/services/image_analyzer_service.dart @@ -1,60 +1,255 @@ -import 'dart:convert'; +// lib/services/image_analyzer_service.dart + +import 'dart:async'; +import 'dart:io'; +import 'dart:isolate'; +import 'dart:ui'; +import 'dart:developer' as dev; import 'package:flutter/foundation.dart'; +import 'package:flutter/services.dart'; +import 'package:path_provider/path_provider.dart'; -// Import the segregated services +// Import analysis services import 'layout_analyzer_service.dart'; import 'color_analyzer_service.dart'; import 'texture_analyzer_service.dart'; import 'embedding_analyzer_service.dart'; -import 'emotional_embeddings_service.dart'; +import 'emotional_embeddings_service.dart'; import 'lighting_service.dart'; import 'era_service.dart'; class ImageAnalyzerService { - /// Runs all 4 analysis tools in parallel and returns a combined JSON Map. + + /// Prepare ALL assets (Models + JSONs) on Main Thread. + /// Returns a map of absolute paths. + static Future<Map<String, String>> _prepareAssets() async { + final dir = await getApplicationDocumentsDirectory(); + + Future<String> copy(String assetName, {String? filename}) async { + final name = filename ?? assetName; + final file = File('${dir.path}/$name'); + // Only copy if it doesn't exist or is empty + if (!await file.exists() || await file.length() == 0) { + final data = await rootBundle.load('assets/$assetName'); + await file.writeAsBytes(data.buffer.asUint8List(), flush: true); + } + return file.path; + } + + // Copy everything needed by the services + // Make sure your assets/ folder actually contains these files! + final paths = await Future.wait([ + // Texture + copy('dinov2.onnx'), + copy('dinov2.onnx.data'), + copy('texture_centroids.json'), + + // CLIP (Embeddings/Emotion/Lighting) + copy('clip_image_encoder.onnx'), + copy('style_embeddings.json'), + copy('emotion_centroids.json'), + copy('lighting_centroids.json'), + ]); + + return { + 'texture_model': paths[0], + // paths[1] is .data, implicit + 'texture_json': paths[2], + 'clip_model': paths[3], + 'embedding_json': paths[4], + 'emotion_json': paths[5], + 'lighting_json': paths[6], + }; + } + + static Future<Map<String, dynamic>> _runProfiledJob({ + required String name, + required String imagePath, + required RootIsolateToken rootToken, + required bool runInIsolate, + // Pass config map to the task + required Map<String, String> assetPaths, + required Future<dynamic> Function(String, Map<String, String>) task, + }) async { + final stopwatch = Stopwatch()..start(); + final dev.TimelineTask timelineTask = dev.TimelineTask(); + timelineTask.start('Analyze: $name'); + + try { + dynamic result; + + if (runInIsolate) { + // PARALLEL (Isolate) + result = await Isolate.run(() async { + BackgroundIsolateBinaryMessenger.ensureInitialized(rootToken); + DartPluginRegistrant.ensureInitialized(); + return await task(imagePath, assetPaths); + }); + } else { + // CONCURRENT (Main Thread) + result = await task(imagePath, assetPaths); + } + + stopwatch.stop(); + timelineTask.finish(); + + return { + 'data': result, + 'stats': {'execution_time_ms': stopwatch.elapsedMilliseconds} + }; + } catch (e) { + stopwatch.stop(); + timelineTask.finish(); + debugPrint("[$name] Failed: $e"); + return {'error': e.toString()}; + } + } + static Future<Map<String, dynamic>> analyzeFullSuite(String imagePath) async { - // 1. Instantiate services - final layoutService = LayoutAnalyzerService(); - final colorService = ColorAnalyzerService(); - final textureService = TextureAnalyzerService(); - // EmbeddingService is static + final totalSw = Stopwatch()..start(); + + final RootIsolateToken? token = RootIsolateToken.instance; + if (token == null) { + throw Exception("RootIsolateToken is null."); + } try { - // 2. Run in parallel for performance + // 1. Prepare Assets on Main Thread (Safe) + final assetPaths = await _prepareAssets(); + + // 2. Run Analysis final results = await Future.wait([ - layoutService.analyze(imagePath), // Index 0: Layout - colorService.analyze(imagePath), // Index 1: Color - textureService.analyze(imagePath), // Index 2: Texture - EmbeddingAnalyzerService.analyze(imagePath), // Index 3: Embeddings - EmotionalEmbeddingsService.analyze(imagePath), // Index 4: Emotional Embeddings - LightingEmbeddingsService.analyze(imagePath), // Index 5: Lighting Embeddings - EraEmbeddingsService.analyze(imagePath), // Index 6: Era Embeddings + + // --- GROUP A: MAIN THREAD (Python) --- + _runProfiledJob( + name: 'Layout', imagePath: imagePath, rootToken: token, runInIsolate: false, + assetPaths: assetPaths, + task: (path, _) => LayoutAnalyzerService().analyze(path) + ), + + _runProfiledJob( + name: 'Color', imagePath: imagePath, rootToken: token, runInIsolate: false, + assetPaths: assetPaths, + task: (path, _) => ColorAnalyzerService().analyze(path) + ), + + // --- GROUP B: PARALLEL (ONNX FFI) --- + // Notice we pass assetPaths to the tasks + + _runProfiledJob( + name: 'Texture', imagePath: imagePath, rootToken: token, runInIsolate: true, + assetPaths: assetPaths, + task: (path, assets) async { + final service = TextureAnalyzerService(); + // Pass specific paths + final res = await service.analyze(path, + modelPath: assets['texture_model'], + jsonPath: assets['texture_json'] + ); + service.dispose(); + return res; + } + ), + + _runProfiledJob( + name: 'Embedding', imagePath: imagePath, rootToken: token, runInIsolate: true, + assetPaths: assetPaths, + task: (path, assets) async { + final service = EmbeddingAnalyzerService(); + final res = await service.analyze(path, + modelPath: assets['clip_model'], + jsonPath: assets['embedding_json'] + ); + service.dispose(); + return res; + } + ), + + _runProfiledJob( + name: 'Emotional', imagePath: imagePath, rootToken: token, runInIsolate: true, + assetPaths: assetPaths, + task: (path, assets) async { + final service = EmotionalEmbeddingsService(); + final res = await service.analyze(path, + modelPath: assets['clip_model'], + jsonPath: assets['emotion_json'] + ); + service.dispose(); + return res; + } + ), + + _runProfiledJob( + name: 'Lighting', imagePath: imagePath, rootToken: token, runInIsolate: true, + assetPaths: assetPaths, + task: (path, assets) async { + final service = LightingEmbeddingsService(); + final res = await service.analyze(path, + modelPath: assets['clip_model'], + jsonPath: assets['lighting_json'] + ); + service.dispose(); + return res; + } + ), ]); - // 3. Construct Unified Result - final Map<String, dynamic> combinedResult = { + totalSw.stop(); + + final finalResult = { 'success': true, + 'total_time_ms': totalSw.elapsedMilliseconds, 'timestamp': DateTime.now().toIso8601String(), - 'layout': results[0], - 'color': results[1], - 'texture': results[2], // This is a List - 'embedding': results[3], - 'emotions': results[4], - 'lighting': results[5], - 'era': results[6], + 'results': { + 'layout': results[0], + 'color': results[1], + 'texture': results[2], + 'embedding': results[3], + 'emotions': results[4], + 'lighting': results[5], + } }; - return combinedResult; + _logSummary(finalResult); + print("\n\n\n layout: ${results[0]} \n\n\n"); + print("color: ${results[1]} \n\n\n"); + print("texture: ${results[2]} \n\n\n"); + print("embedding: ${results[3]} \n\n\n"); + print("emotions: ${results[4]} \n\n\n"); + print("lighting: ${results[5]} \n\n\n"); + return finalResult; } catch (e) { debugPrint("Master Analysis Failed: $e"); - return { - 'success': false, - 'error': e.toString(), - }; - } finally { - // 4. Cleanup heavy resources - textureService.dispose(); + return {'success': false, 'error': e.toString()}; } } + + static void _logSummary(Map<String, dynamic> result) { + // ... (Keep your existing log summary code) ... + final int total = result['total_time_ms']; + final Map<String, dynamic> subTasks = result['results']; + + StringBuffer output = StringBuffer(); + output.writeln('\n══════════ ⚡ HYBRID PARALLEL REPORT ⚡ ══════════'); + output.writeln(' 🕒 Total Wall Time: ${total}ms'); + output.writeln('──────────────────────────────────────────────────────'); + + subTasks.forEach((key, value) { + if (value is Map && value.containsKey('stats')) { + final int time = value['stats']['execution_time_ms']; + final int barLength = (time / 50).ceil().clamp(0, 20); + final String bar = '█' * barLength; + + output.writeln(' ${key.padRight(12)} : ${time.toString().padLeft(4)}ms $bar'); + } else { + String err = value['error'] ?? "Unknown"; + if (err.length > 30) err = "${err.substring(0, 30)}..."; + output.writeln(' ${key.padRight(12)} : FAILED ❌ ($err)'); + } + }); + output.writeln('══════════════════════════════════════════════════════\n'); + + dev.log(output.toString(), name: 'ImageAnalyzer'); + } } diff --git a/lib/services/layout_analyzer_service.dart b/lib/services/layout_analyzer_service.dart @@ -1,3 +1,5 @@ +// lib/services/layout_analyzer_service.dart + import 'package:flutter/foundation.dart'; import 'python_service.dart'; diff --git a/lib/services/lighting_service.dart b/lib/services/lighting_service.dart @@ -1,39 +1,32 @@ +// lib/services/lighting_service.dart + import 'dart:convert'; -import 'dart:math'; import 'dart:io'; +import 'package:adobe/utils/image_utils.dart'; import 'package:flutter/foundation.dart'; -import 'package:flutter/services.dart'; -import 'package:flutter_onnxruntime/flutter_onnxruntime.dart'; -import 'package:path_provider/path_provider.dart'; +import 'package:onnxruntime/onnxruntime.dart'; import '../utils/clip_image_processor.dart'; class LightingEmbeddingsService { - static OrtSession? _session; - static List<List<double>>? _centroids; - static List<String>? _classes; + OrtSession? _session; + List<List<double>>? _centroids; + List<String>? _classes; - static Future<void> initialize() async { + Future<void> initialize({required String modelPath, required String jsonPath}) async { if (_session != null) return; try { debugPrint("Initializing Embeddings (Dino/CLIP)..."); + + OrtEnv.instance.init(); - final rawAsset = await rootBundle.load('assets/clip_image_encoder.onnx'); - final dir = await getTemporaryDirectory(); - final modelFile = File('${dir.path}/clip_model.onnx'); - - if (!await modelFile.exists()) { - await modelFile.writeAsBytes(rawAsset.buffer.asUint8List()); - } - - // Initialize runtime - final ort = OnnxRuntime(); - _session = await ort.createSession(modelFile.path); + // 1. Load Model from passed PATH (No rootBundle!) + final sessionOptions = OrtSessionOptions(); + _session = OrtSession.fromFile(File(modelPath), sessionOptions); + sessionOptions.release(); - final jsonString = await rootBundle.loadString( - 'assets/lighting_centroids.json', - ); - final jsonData = json.decode(jsonString); + final jsonStr = await File(jsonPath).readAsString(); + final jsonData = json.decode(jsonStr); _classes = List<String>.from(jsonData['classes']); _centroids = @@ -45,11 +38,18 @@ class LightingEmbeddingsService { } } - static Future<Map<String, dynamic>?> analyze(String imagePath) async { - if (_session == null) await initialize(); + Future<Map<String, dynamic>?> analyze(String imagePath, { + String? modelPath, + String? jsonPath, + }) async { + // Safety check + if (modelPath == null || jsonPath == null) return null; + await initialize(modelPath: modelPath, jsonPath: jsonPath); + if (_session == null || _centroids == null) return null; - OrtValue? inputOrt; - OrtValue? outputOrt; + OrtValueTensor? inputOrt; + OrtRunOptions? runOptions; + List<OrtValue?>? outputs; try { final float32Input = await ClipImageProcessor.preprocess(imagePath); @@ -57,23 +57,22 @@ class LightingEmbeddingsService { // Create Tensor // Note: ensure ClipImageProcessor returns a flat List<double> - inputOrt = await OrtValue.fromList(float32Input, [1, 3, 224, 224]); + inputOrt = OrtValueTensor.createTensorWithDataList( + float32Input, [1, 3, 224, 224]); + + runOptions = OrtRunOptions(); // Run Inference - // 'image' is the input name for CLIP. Check your model if it differs (e.g. 'input') - final outputs = await _session!.run({"image": inputOrt}); + // 'image' is the input name for CLIP. + outputs = _session!.run(runOptions, {"image": inputOrt}); // <--- UPDATED //debugPrint("[LIGHTING] Output keys: ${outputs.keys}"); - // Get Output - outputOrt = - outputs['output']; // or outputs['image_features'] depending on model - // If outputOrt is null, try iterating outputs.values.first - final resultOrt = outputOrt ?? outputs.values.first; + if (outputs.isEmpty) throw Exception("No output from model"); - if (resultOrt == null) throw Exception("No output from model"); + // Get Output + final dynamic outputRaw = outputs[0]?.value; // Flatten Output - final List<dynamic> rawList = await resultOrt.asList(); final List<double> imgFeat = []; void flatten(dynamic data) { @@ -86,49 +85,38 @@ class LightingEmbeddingsService { } } - flatten(rawList); + flatten(outputRaw); // Normalize & Compare - final normFeat = _l2Normalize(imgFeat); - List<Map<String, dynamic>> scores = []; + final normFeat = l2Normalize(imgFeat); + Map<String, dynamic> scores = {}; for (int i = 0; i < _classes!.length; i++) { - double score = _dotProduct(normFeat, _centroids![i]); - scores.add({"name": _classes![i], "score": score * 100.0}); + double score = dotProduct(normFeat, _centroids![i]); + scores[_classes![i]] = score * 100.0; } - scores.sort( - (a, b) => (b['score'] as double).compareTo(a['score'] as double), + scores = Map.fromEntries( + scores.entries.toList() + ..sort((a, b) => b.value.compareTo(a.value)), ); return { "success": true, - "label": scores.first['name'], - "top5": scores.take(5).toList(), + "predictions": scores, + "best": {scores.keys.first: scores.values.first}, }; } catch (e) { debugPrint("Embeddings Analysis Error: $e"); return null; } finally { - inputOrt?.dispose(); - outputOrt?.dispose(); + inputOrt?.release(); + runOptions?.release(); + outputs?.forEach((element) => element?.release()); } } - static List<double> _l2Normalize(List<double> vec) { - double sum = vec.fold(0, (p, c) => p + c * c); - double norm = sqrt(sum); - if (norm == 0) return vec; - return vec.map((v) => v / norm).toList(); - } - - static double _dotProduct(List<double> a, List<double> b) { - double sum = 0; - // Ensure lengths match to avoid range errors - int len = min(a.length, b.length); - for (int i = 0; i < len; i++) { - sum += a[i] * b[i]; - } - return sum; + void dispose() { + _session?.release(); } } diff --git a/lib/services/texture_analyzer_service.dart b/lib/services/texture_analyzer_service.dart @@ -1,11 +1,11 @@ +// lib/services/texture_analyzer_service.dart + import 'dart:io'; -import 'dart:math'; import 'dart:convert'; import 'package:flutter/foundation.dart'; -import 'package:flutter/services.dart'; import 'package:image/image.dart' as img; -import 'package:flutter_onnxruntime/flutter_onnxruntime.dart'; -import 'package:path_provider/path_provider.dart'; +import 'package:onnxruntime/onnxruntime.dart'; +import '../utils/image_utils.dart'; class TextureAnalyzerService { OrtSession? _session; @@ -17,33 +17,22 @@ class TextureAnalyzerService { static const double RELATIVE_THRESH = 0.85; static const double ABSOLUTE_FLOOR = 0.15; - Future<void> _loadResources() async { + Future<void> initialize({required String modelPath, required String jsonPath}) async { if (_session != null) return; try { debugPrint("Initializing Texture Service..."); - final dir = await getApplicationDocumentsDirectory(); - final modelFile = File('${dir.path}/dinov2.onnx'); - final dataFile = File('${dir.path}/dinov2.onnx.data'); - - if (!await modelFile.exists()) { - final data = await rootBundle.load('assets/dinov2.onnx'); - await modelFile.writeAsBytes(data.buffer.asUint8List(), flush: true); - } - - // Check for external data file - if (!await dataFile.exists()) { - try { - final data = await rootBundle.load('assets/dinov2.onnx.data'); - await dataFile.writeAsBytes(data.buffer.asUint8List(), flush: true); - } catch (_) { - // External data might not exist for all models - } - } + + // Initialize ORT Environment (FFI) + OrtEnv.instance.init(); - _session = await OnnxRuntime().createSession(modelFile.path); + // 1. Load Model from passed PATH + final sessionOptions = OrtSessionOptions(); + _session = OrtSession.fromFile(File(modelPath), sessionOptions); + sessionOptions.release(); - final jsonStr = await rootBundle.loadString('assets/texture_centroids.json'); + // 2. Load JSON from passed PATH + final jsonStr = await File(jsonPath).readAsString(); final Map<String, dynamic> jsonMap = json.decode(jsonStr); _centroids = {}; jsonMap.forEach((k, v) => _centroids![k] = List<double>.from(v)); @@ -53,18 +42,25 @@ class TextureAnalyzerService { } } - Future<List<Map<String, dynamic>>> analyze(String path) async { - await _loadResources(); - if (_session == null || _centroids == null) return []; + // Changed return type to Map to match other services + Future<Map<String, dynamic>> analyze(String path, { + String? modelPath, + String? jsonPath + }) async { + // Safety check + if (modelPath == null || jsonPath == null) return {'success': false, 'error': 'Paths missing'}; + await initialize(modelPath: modelPath, jsonPath: jsonPath); + if (_session == null || _centroids == null) return {'success': false, 'error': 'Init failed'}; - OrtValue? inputOrt; - OrtValue? outputOrt; + OrtValueTensor? inputOrt; + OrtRunOptions? runOptions; + List<OrtValue?>? outputs; try { // 1. Preprocess final bytes = await File(path).readAsBytes(); final image = img.decodeImage(bytes); - if (image == null) return []; + if (image == null) return {'success': false, 'error': 'Image decode failed'}; final resized = img.copyResize(image, width: INPUT_SIZE, height: INPUT_SIZE); @@ -80,16 +76,27 @@ class TextureAnalyzerService { pixelIndex++; } } + + final float32List = Float32List.fromList(inputFloats); // 3. Inference - inputOrt = await OrtValue.fromList(inputFloats, [1, 3, INPUT_SIZE, INPUT_SIZE]); - final outputs = await _session!.run({"input": inputOrt}); - outputOrt = outputs['output']; + // Create Tensor using FFI method with Float32List + inputOrt = OrtValueTensor.createTensorWithDataList( + float32List, [1, 3, INPUT_SIZE, INPUT_SIZE]); + + runOptions = OrtRunOptions(); - if (outputOrt == null) throw Exception("Output tensor missing"); + // Run Inference + // 'input' is the standard name, check your model using Netron if it fails + outputs = _session!.run(runOptions, {"input": inputOrt}); + + if (outputs.isEmpty) throw Exception("Output tensor missing"); + + // Get Output + // FFI returns a List<OrtValue>, usually index 0 is what we want + final dynamic outputRaw = outputs[0]?.value; // Flatten output - final dynamic outputRaw = await outputOrt.asList(); final List<double> rawOutput = []; void flatten(dynamic data) { if (data is num) { @@ -108,17 +115,17 @@ class TextureAnalyzerService { double sumSim = 0.0; int patches = 256; for (int i = 0; i < patches; i++) { - int start = i * EMBED_DIM; - if (start + EMBED_DIM <= rawOutput.length) { - sumSim += _cosineSim(rawOutput.sublist(start, start + EMBED_DIM), centroid); - } + int start = i * EMBED_DIM; + if (start + EMBED_DIM <= rawOutput.length) { + sumSim += cosineSim(rawOutput.sublist(start, start + EMBED_DIM), centroid); + } } rawScores[name] = sumSim / patches; }); // 5. Filter & Sort var sorted = rawScores.entries.toList()..sort((a, b) => b.value.compareTo(a.value)); - List<Map<String, dynamic>> finalResults = []; + Map<String, dynamic> finalResults = {}; if (sorted.isNotEmpty) { double topScore = sorted.first.value; @@ -127,31 +134,29 @@ class TextureAnalyzerService { if (entry.value < ABSOLUTE_FLOOR) continue; if (entry.value < (topScore * RELATIVE_THRESH)) break; - finalResults.add({'name': entry.key, 'score': entry.value}); + finalResults[entry.key] = entry.value; } } - return finalResults; + + // Return Map consistent with other services + return { + 'success': true, + 'predictions': finalResults, + 'best': {finalResults.keys.first: finalResults.values.first}, + }; } catch (e) { debugPrint("Texture Analysis Failed: $e"); - return []; + return {'success': false, 'error': e.toString()}; } finally { - inputOrt?.dispose(); - outputOrt?.dispose(); - } - } - - double _cosineSim(List<double> a, List<double> b) { - double dot = 0.0, nA = 0.0, nB = 0.0; - for (int i = 0; i < a.length; i++) { - dot += a[i] * b[i]; - nA += a[i] * a[i]; - nB += b[i] * b[i]; + // FFI Requires manual release of C++ resources + inputOrt?.release(); + runOptions?.release(); + outputs?.forEach((element) => element?.release()); } - return (nA == 0 || nB == 0) ? 0.0 : dot / (sqrt(nA) * sqrt(nB)); } void dispose() { - _session?.close(); + _session?.release(); } -} +} +\ No newline at end of file diff --git a/lib/utils/clip_image_processor.dart b/lib/utils/clip_image_processor.dart @@ -1,3 +1,5 @@ +// lib/utils/clip_image_processor.dart + import 'dart:typed_data'; import 'dart:io'; import 'package:image/image.dart' as img; diff --git a/lib/utils/image_utils.dart b/lib/utils/image_utils.dart @@ -0,0 +1,30 @@ +// lib/utils/image_utils.dart + +import 'dart:math'; + +List<double> l2Normalize(List<double> vec) { + double sum = vec.fold(0, (p, c) => p + c * c); + double norm = sqrt(sum); + if (norm == 0) return vec; + return vec.map((v) => v / norm).toList(); +} + +double dotProduct(List<double> a, List<double> b) { + double sum = 0; + // Ensure lengths match to avoid range errors + int len = min(a.length, b.length); + for (int i = 0; i < len; i++) { + sum += a[i] * b[i]; + } + return sum; +} + +double cosineSim(List<double> a, List<double> b) { + double dot = 0.0, nA = 0.0, nB = 0.0; + for (int i = 0; i < a.length; i++) { + dot += a[i] * b[i]; + nA += a[i] * a[i]; + nB += b[i] * b[i]; + } + return (nA == 0 || nB == 0) ? 0.0 : dot / (sqrt(nA) * sqrt(nB)); +} +\ No newline at end of file diff --git a/linux/flutter/generated_plugin_registrant.cc b/linux/flutter/generated_plugin_registrant.cc @@ -7,13 +7,9 @@ #include "generated_plugin_registrant.h" #include <file_selector_linux/file_selector_plugin.h> -#include <flutter_onnxruntime/flutter_onnxruntime_plugin.h> void fl_register_plugins(FlPluginRegistry* registry) { g_autoptr(FlPluginRegistrar) file_selector_linux_registrar = fl_plugin_registry_get_registrar_for_plugin(registry, "FileSelectorPlugin"); file_selector_plugin_register_with_registrar(file_selector_linux_registrar); - g_autoptr(FlPluginRegistrar) flutter_onnxruntime_registrar = - fl_plugin_registry_get_registrar_for_plugin(registry, "FlutterOnnxruntimePlugin"); - flutter_onnxruntime_plugin_register_with_registrar(flutter_onnxruntime_registrar); } diff --git a/linux/flutter/generated_plugins.cmake b/linux/flutter/generated_plugins.cmake @@ -4,10 +4,10 @@ list(APPEND FLUTTER_PLUGIN_LIST file_selector_linux - flutter_onnxruntime ) list(APPEND FLUTTER_FFI_PLUGIN_LIST + onnxruntime ) set(PLUGIN_BUNDLED_LIBRARIES) diff --git a/macos/Flutter/GeneratedPluginRegistrant.swift b/macos/Flutter/GeneratedPluginRegistrant.swift @@ -6,14 +6,12 @@ import FlutterMacOS import Foundation import file_selector_macos -import flutter_onnxruntime import path_provider_foundation import shared_preferences_foundation import sqflite_darwin func RegisterGeneratedPlugins(registry: FlutterPluginRegistry) { FileSelectorPlugin.register(with: registry.registrar(forPlugin: "FileSelectorPlugin")) - FlutterOnnxruntimePlugin.register(with: registry.registrar(forPlugin: "FlutterOnnxruntimePlugin")) PathProviderPlugin.register(with: registry.registrar(forPlugin: "PathProviderPlugin")) SharedPreferencesPlugin.register(with: registry.registrar(forPlugin: "SharedPreferencesPlugin")) SqflitePlugin.register(with: registry.registrar(forPlugin: "SqflitePlugin")) diff --git a/pubspec.lock b/pubspec.lock @@ -160,14 +160,6 @@ packages: url: "https://pub.dev" source: hosted version: "5.0.0" - flutter_onnxruntime: - dependency: "direct main" - description: - name: flutter_onnxruntime - sha256: "55842e69293ec52c07f3065049ff7641e94e8a6cca3f659a913d5401a3994424" - url: "https://pub.dev" - source: hosted - version: "1.6.0" flutter_plugin_android_lifecycle: dependency: transitive description: @@ -362,6 +354,14 @@ packages: url: "https://pub.dev" source: hosted version: "1.0.0" + onnxruntime: + dependency: "direct main" + description: + name: onnxruntime + sha256: e77ec05acafc135cc5fe7bcdf11b101b39f06513c9d5e9fa02cb1929f6bac72a + url: "https://pub.dev" + source: hosted + version: "1.4.1" path: dependency: "direct main" description: diff --git a/pubspec.yaml b/pubspec.yaml @@ -21,8 +21,7 @@ dependencies: shared_preferences: ^2.5.3 provider: ^6.1.5+1 image: ^4.5.4 - flutter_onnxruntime: ^1.6.0 - intl: ^0.20.2 + onnxruntime: ^1.4.1 dev_dependencies: flutter_test: diff --git a/windows/flutter/generated_plugin_registrant.cc b/windows/flutter/generated_plugin_registrant.cc @@ -7,11 +7,8 @@ #include "generated_plugin_registrant.h" #include <file_selector_windows/file_selector_windows.h> -#include <flutter_onnxruntime/flutter_onnxruntime_plugin.h> void RegisterPlugins(flutter::PluginRegistry* registry) { FileSelectorWindowsRegisterWithRegistrar( registry->GetRegistrarForPlugin("FileSelectorWindows")); - FlutterOnnxruntimePluginRegisterWithRegistrar( - registry->GetRegistrarForPlugin("FlutterOnnxruntimePlugin")); } diff --git a/windows/flutter/generated_plugins.cmake b/windows/flutter/generated_plugins.cmake @@ -4,10 +4,10 @@ list(APPEND FLUTTER_PLUGIN_LIST file_selector_windows - flutter_onnxruntime ) list(APPEND FLUTTER_FFI_PLUGIN_LIST + onnxruntime ) set(PLUGIN_BUNDLED_LIBRARIES)