Working with tracker, getting unreliable results

src/IronMiner.java

@@ -16,7 +16,10 @@ import org.opencv.core.CvType;
 import org.opencv.core.Mat;
 import org.opencv.core.Rect2d;
 import org.opencv.tracking.Tracker;
+import org.opencv.tracking.TrackerBoosting;
+import org.opencv.tracking.TrackerGOTURN;
 import org.opencv.tracking.TrackerKCF;
+import org.opencv.tracking.TrackerMOSSE;
 
 public class IronMiner {
 	
@@ -33,9 +36,9 @@ public class IronMiner {
 	
 	public IronMiner() throws AWTException, IOException 
 	{
-		cursor = new Cursor();
-		cursorTask = new CursorTask();
-		inventory = new Inventory();
+		//cursor = new Cursor();
+		//cursorTask = new CursorTask();
+		//inventory = new Inventory();
 		objectDetector = new ObjectDetector();
 		robot = new Robot();
 		randomizer = new Randomizer();
@@ -44,47 +47,56 @@ public class IronMiner {
 	public void run() throws Exception {
 		while (true) {
 			BufferedImage screenCapture = objectDetector.captureScreenshotGameWindow();
-			ArrayList<DetectedObject> detectedObjects = objectDetector.getObjectsInImage(screenCapture);
+			ArrayList<DetectedObject> detectedObjects = objectDetector.getObjectsInImage(screenCapture, 0.60);
 			ArrayList<DetectedObject> ironOres = objectDetector.getObjectsOfClassInList(detectedObjects, "ironOre");
 			
 			DetectedObject closestIronOre = getClosestObjectToCharacter(ironOres);
 			if (closestIronOre != null) {
 				System.out.println("Found iron ore! Starting tracking!");
-				Tracker objectTracker = TrackerKCF.create();
+				Tracker objectTracker = TrackerBoosting.create();
 				Rect2d boundingBox = closestIronOre.getBoundingRect2d();
 				objectTracker.init(getMatFromBufferedImage(screenCapture), boundingBox);
 				
-				cursor.moveAndLeftClickAtCoordinatesWithRandomness(closestIronOre.getCenterForClicking(), 10, 10);
+				//cursor.moveAndLeftClickAtCoordinatesWithRandomness(closestIronOre.getCenterForClicking(), 10, 10);
 				
 				long mineStartTime = System.currentTimeMillis();
 				int maxTimeToMine = randomizer.nextGaussianWithinRange(3500, 5000);
 				
 				// track until either we lose the object or too much time passes
 				int lostTrackCounter = 0;
-				while (((System.currentTimeMillis() - mineStartTime) < maxTimeToMine) && lostTrackCounter < 3) {
-					
-					screenCapture = objectDetector.captureScreenshotGameWindow();
-					detectedObjects = objectDetector.getObjectsInImage(screenCapture);
+				//while (((System.currentTimeMillis() - mineStartTime) < maxTimeToMine) && lostTrackCounter < 3) {
+				while (lostTrackCounter < 600) {	
 					
+					//screenCapture = objectDetector.captureScreenshotGameWindow();
+					detectedObjects = objectDetector.getObjectsInImage(screenCapture, 0.3); // Call with lower threshold value here
+					/*
+					 * 600 iterations: 30 seconds
 					boolean ok = objectTracker.update(getMatFromBufferedImage(screenCapture), boundingBox);
-					if (!ok || !objectDetector.isObjectPresentInBoundingBoxInImage(detectedObjects, boundingBox, "ironOre")) {
+					if (!ok) {
 						System.out.println("Lost track for + " + lostTrackCounter + "! Finding new ore soon.");
 						lostTrackCounter++;
 					}
-					else if (ok) {
-						lostTrackCounter = 0;
-						System.out.println("Tracking at " + boundingBox.x + ", " + boundingBox.y + ", " + boundingBox.width + ", " + boundingBox.height);
+					else if (!objectDetector.isObjectPresentInBoundingBoxInImage(detectedObjects, boundingBox, "ironOre")) {
+						System.out.println("Can't find object in bounding box. Error " + lostTrackCounter + "! Finding new ore soon.");
+						lostTrackCounter++;
 					}
-
+					else if (ok) {
+						//lostTrackCounter = 0;
+						System.out.println("Tracking at " + boundingBox.x + ", " + boundingBox.y + ", " + boundingBox.width + ", " + boundingBox.height);
+					}*/
+					System.out.println(lostTrackCounter);
+					lostTrackCounter++;
 				}
+				System.out.println("600 iterations took " + (System.currentTimeMillis() - mineStartTime));
+				break;
 			}
 			
-			dropInventoryIfFull();
+			//dropInventoryIfFull();
 		}
 	}
 	
 	private void dropInventoryIfFull() throws Exception {
-		inventory.update(); // TODO: add iron ore to inventory items
+		inventory.update(); // TODO: add iron ore to indonventory items
 		if (inventory.isInventoryFull()) {
 			cursorTask.optimizedDropAllItemsInInventory(cursor, inventory);
 		}

src/MirrorDisplay.java

@@ -0,0 +1,46 @@
+import java.awt.AWTException;
+import java.awt.Dimension;
+import java.awt.Graphics;
+import java.awt.Rectangle;
+import java.awt.Robot;
+import java.awt.image.BufferedImage;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import javax.swing.JFrame;
+import javax.swing.JPanel;
+
+public class MirrorDisplay extends JPanel{
+
+    private BufferedImage image;
+
+    public MirrorDisplay()  {
+       try {
+           Robot robot = new Robot();
+           image = robot.createScreenCapture(new Rectangle(0, 0, 500, 500));
+        } catch (AWTException ex) {
+           Logger.getLogger(MirrorDisplay.class.getName()).log(Level.SEVERE, null, ex);
+        }
+
+    }
+
+    @Override
+    public void paintComponent(Graphics g) {
+        g.drawImage(image, 0, 0, null); 
+
+    }
+public static void main(String[] args) throws InterruptedException {
+
+        JFrame test = new JFrame();
+        
+        while (true) {
+        	test.add(new MirrorDisplay());
+        	Dimension b = new Dimension(500,500);
+        	test.setMinimumSize(b);
+
+        	test.setVisible(true);
+        	Thread.sleep(50);
+        	test.dispose();
+        }
+
+    }
+}

src/ObjectDetector.java

@@ -61,11 +61,15 @@ public class ObjectDetector {
 		this.detectedObjects = getRecognizedObjectsFromImage(fileName);*/
 	}
 	
-	public ArrayList<DetectedObject> getObjectsInImage(BufferedImage image) throws Exception {
+	public ArrayList<DetectedObject> getObjectsInImage(BufferedImage image, double scoreThreshold) throws Exception {
 		List<Tensor<?>> outputs = null;
 		ArrayList<DetectedObject> detectedObjectsInImage = new ArrayList<DetectedObject>();
 		
+		long timebefore = System.currentTimeMillis();
+		makeImageTensor(image);
+		System.out.println("Conversion took: " + (System.currentTimeMillis() - timebefore));
         try (Tensor<UInt8> input = makeImageTensor(image)) {
+        	long timebefore2 = System.currentTimeMillis();
           outputs =
               model
                   .session()
@@ -75,6 +79,7 @@ public class ObjectDetector {
                   .fetch("detection_classes")
                   .fetch("detection_boxes")
                   .run();
+          System.out.println("Model took: " + (System.currentTimeMillis() - timebefore));
         }
         
         try (Tensor<Float> scoresT = outputs.get(0).expect(Float.class);
@@ -87,7 +92,7 @@ public class ObjectDetector {
             float[][] boxes = boxesT.copyTo(new float[1][maxObjects][4])[0];
 
             for (int i = 0; i < scores.length; ++i) {
-              if (scores[i] > 0.80) {
+              if (scores[i] > scoreThreshold) {
             	  detectedObjectsInImage.add(new DetectedObject(scores[i], classes[i], boxes[i]));
               }
             }

src/ObjectDetectorTest.java

@@ -17,7 +17,7 @@ class ObjectDetectorTest {
 	void testObjectDetection() throws Exception {
 		ObjectDetector objectDetector = new ObjectDetector();
 		BufferedImage testImage1 = ImageIO.read(new File("/home/dpapp/tensorflow-1.5.0/models/raccoon_dataset/test_images/ironOre_test_9.jpg"));
-		ArrayList<DetectedObject> detectedObjects1 = objectDetector.getObjectsInImage(testImage1);
+		ArrayList<DetectedObject> detectedObjects1 = objectDetector.getObjectsInImage(testImage1, 0.8);
 		ArrayList<DetectedObject> detectedIronOres1 = objectDetector.getObjectsOfClassInList(detectedObjects1, "ironOre");
 		ArrayList<DetectedObject> detectedOres1 = objectDetector.getObjectsOfClassInList(detectedObjects1, "ore");
 
@@ -26,6 +26,13 @@ class ObjectDetectorTest {
 		//assertDetectedObjectsAreEqual();
 	}
 	
+	
+	/*@Test
+	void testObjectDetectionInSubImage() throws Exception {
+		ObjectDetector objectDetector = new ObjectDetector();
+		BufferedImage testImage = ImageIO.read(new File("/home/dpapp/tensorflow-1.5.0/models/raccoon_dataset/test_images/ironOre_test_9.jpg"));
+	}*/
+	
 	void assertDetectedObjectsAreEqual(DetectedObject obj1, DetectedObject obj2) {
 		
 	}

src/ObjectTrackerTest.java

@@ -1,6 +1,8 @@
 import static org.junit.jupiter.api.Assertions.*;
 
 import java.awt.AWTException;
+import java.awt.Color;
+import java.awt.Graphics;
 import java.awt.image.BufferedImage;
 import java.io.ByteArrayInputStream;
 import java.io.File;
@@ -16,6 +18,9 @@ import org.opencv.core.Rect2d;
 import org.opencv.imgcodecs.Imgcodecs;
 import org.opencv.tracking.Tracker;
 import org.opencv.tracking.TrackerKCF;
+import org.opencv.tracking.TrackerMIL;
+import org.opencv.tracking.TrackerBoosting;
+import org.opencv.tracking.TrackerMedianFlow;
 import org.opencv.videoio.VideoCapture;
 
 class ObjectTrackerTest {
@@ -23,7 +28,7 @@ class ObjectTrackerTest {
 	@Test
 	void testObjectTracking() throws Exception {
 		System.loadLibrary(Core.NATIVE_LIBRARY_NAME);
-		VideoCapture video = new VideoCapture("/home/dpapp/Videos/gameplay-2018-02-24_10.01.00.mp4");
+		VideoCapture video = new VideoCapture("/home/dpapp/Videos/gameplay-2018-02-24_14.24.53.mp4");
 		System.out.println("loaded video...");
 		
 		ObjectDetector objectDetector = new ObjectDetector();
@@ -32,24 +37,27 @@ class ObjectTrackerTest {
 		boolean frameReadSuccess = video.read(frame);
 		assertTrue(frameReadSuccess);
 		
-		ArrayList<DetectedObject> detectedObjects = objectDetector.getObjectsInImage(Mat2BufferedImage(frame));
+		ArrayList<DetectedObject> detectedObjects = objectDetector.getObjectsInImage(Mat2BufferedImage(frame), 0.60);
 		System.out.println("Tracking " + detectedObjects.size() + " objects.");
+		
 		ArrayList<Tracker> objectTrackers = new ArrayList<Tracker>();
 		ArrayList<Rect2d> boundingBoxes = new ArrayList<Rect2d>();
-		for (int i = 0; i < 3; i++) {
+		for (int i = 0; i < detectedObjects.size(); i++) {
 			boundingBoxes.add(detectedObjects.get(i).getBoundingRect2d());
-			objectTrackers.add(TrackerKCF.create());
+			objectTrackers.add(TrackerBoosting.create());
 			objectTrackers.get(i).init(frame, boundingBoxes.get(i));
 		}
 		//System.out.println("bounding box: " + (int) boundingBoxes.get(0).x + ", " + (int) boundingBoxes.get(0).y + ", " + (int) boundingBoxes.get(0).width + ", " + (int) boundingBoxes.get(0).height);
 		
 		int counter = 0;
 		while (video.read(frame)) {
-			for (int i = 0; i < 3; i++) {
+			BufferedImage screencapture = Mat2BufferedImage(frame);
+			detectedObjects = objectDetector.getObjectsInImage(screencapture, 0.3);
+			
+			for (int i = 0; i < objectTrackers.size(); i++) {
 				objectTrackers.get(i).update(frame, boundingBoxes.get(i));
 				boolean trackingSuccess = objectTrackers.get(i).update(frame, boundingBoxes.get(i));
 				
-				detectedObjects = objectDetector.getObjectsInImage(Mat2BufferedImage(frame));
 				//System.out.println(detectedObjects.size());
 				//System.out.println((int) boundingBoxes.get(i).x + ", " + (int) boundingBoxes.get(i).y + ", " + (int) boundingBoxes.get(i).width + ", " + (int) boundingBoxes.get(i).height);
 				//BufferedImage subImage = screencapture.getSubimage((int) boundingBoxes.get(i).x- 10, (int) boundingBoxes.get(i).y - 10, (int) boundingBoxes.get(i).width + 20, (int) boundingBoxes.get(i).height + 20);
@@ -57,13 +65,27 @@ class ObjectTrackerTest {
 				boolean ironOreDetected = objectDetector.isObjectPresentInBoundingBoxInImage(detectedObjects, boundingBoxes.get(i), "ironOre");
 				boolean oreDetected = objectDetector.isObjectPresentInBoundingBoxInImage(detectedObjects, boundingBoxes.get(i), "ore");
 				
-				//ImageIO.write(screencapture, "jpg", new File("/home/dpapp/Videos/frames/frame_" + counter + ".jpg"));
+				Graphics g = screencapture.getGraphics();
+				if (ironOreDetected) {
+					g.setColor(Color.GREEN);
+				}
+				else if (oreDetected) {
+					g.setColor(Color.RED);
+				}
+				else {
+					g.setColor(Color.WHITE);
+				}
+				g.drawRect((int) boundingBoxes.get(i).x, (int) boundingBoxes.get(i).y, (int) boundingBoxes.get(i).width, (int) boundingBoxes.get(i).height);
+				
 				//ImageIO.write(subImage, "jpg", new File("/home/dpapp/Videos/sub_frames/frame_" + counter + "_sub.jpg"));
 				//System.out.println("wrote file...");
-				if (i == 2) {
+				/*if (i == ) {
 					System.out.println(trackingSuccess + ", ironOre: " + ironOreDetected + ", ore:" + oreDetected);
-				}
+				}*/
+				
 			}
+			System.out.println("Wrote image...");
+			ImageIO.write(screencapture, "jpg", new File("/home/dpapp/Videos/frames/frame_" + counter + ".jpg"));
 			counter++;
 		}