Object tracking beginnings. Still getting a bunch of errors with OpenCV imports.

src/DetectedObject.java

@@ -1,6 +1,8 @@
 import java.awt.Point;
 import java.awt.Rectangle;
 
+import org.opencv.core.Rect2d;
+
 
 
 public class DetectedObject {
@@ -35,10 +37,14 @@ public class DetectedObject {
 		return detectionClass;
 	}
 	
-	public Rectangle getBoundingBox() {
+	public Rectangle getBoundingRectangle() {
 		return boundingBox;
 	}
 	
+	private Rect2d getBoundingRect2d() {
+		return new Rect2d(boundingBox.x, boundingBox.y, boundingBox.x + boundingBox.width, boundingBox.y + boundingBox.height);
+	}
+	
 	public Point getCenterForClicking() {
 		return new Point(boundingBox.x + boundingBox.width / 2 + Constants.GAME_WINDOW_OFFSET_X, boundingBox.y + boundingBox.height / 2 + Constants.GAME_WINDOW_OFFSET_Y);
 	}

src/ImageCollector.java

@@ -78,7 +78,6 @@ public class ImageCollector {
 		return screenshotOutputDirectory + itemName + "_" + counter + ".jpg";
 	}
 	
-		
 	private void generateInventoryImages() throws AWTException, IOException {
 		Inventory inventory = new Inventory();
 		inventory.updateAndWriteAllInventoryImages();

src/IronMiner.java

@@ -10,10 +10,12 @@ import java.util.ArrayList;
 import javax.imageio.ImageIO;
 
 import org.opencv.core.Rect2d;
+import org.opencv.tracking.Tracker;
+import org.opencv.tracking.TrackerKCF;
 
 public class IronMiner {
 	
-	public static final int IRON_ORE_MINING_TIME_MILLISECONDS = 2738;
+	public static final int IRON_ORE_MINING_TIME_MILLISECONDS = 1320;
 	public static final int MAXIMUM_DISTANCE_TO_WALK_TO_IRON_ORE = 400;
 	public static final Point GAME_WINDOW_CENTER = new Point(Constants.GAME_WINDOW_WIDTH / 2, Constants.GAME_WINDOW_HEIGHT / 2);
 	
@@ -37,15 +39,31 @@ public class IronMiner {
 	public void run() throws Exception {
 		
 		while (true) {
-			objectDetector.update();
+			BufferedImage screenCapture = objectDetector.captureScreenshotGameWindow();
-			ArrayList<DetectedObject> ironOres = objectDetector.getRecognizedObjectsOfClassFromImage("ironOre");
+			ArrayList<DetectedObject> detectedObjects = objectDetector.getObjectsInImage(screenCapture);
-			ArrayList<DetectedObject> ores = objectDetector.getRecognizedObjectsOfClassFromImage("ore");
+			ArrayList<DetectedObject> ironOres = objectDetector.getObjectsOfClassInList(detectedObjects, "ironOre");
-			System.out.println(ironOres.size() + " ironOres, " + ores.size() + " ores.");
+			
+			DetectedObject closestIronOre = getClosestObjectToCharacter(ironOres);
+			if (closestIronOre != null) {
+				Tracker objectTracker = TrackerKCF.create();
+				objectTracker.init(screenCapture, closestIronOre.getBoundingRect2d());
+				cursor.moveAndLeftClickAtCoordinatesWithRandomness(closestIronOre.getCenterForClicking(), 10, 10);
+				
+				long mineStartTime = System.currentTimeMillis();
+				int maxTimeToMine = randomizer.nextGaussianWithinRange(3500, 5000);
+				
+				
+				while ((System.currentTimeMillis() - mineStartTime) < maxTimeToMine) {
+					screenCapture = objectDetector.captureScreenshotGameWindow();
+					objectTracker.update(screenCapture, boundingBox);
+					Rectangle newBounds = new Rectangle();
+					if (!objectDetector.isObjectPresentInBoundingBoxInImage(screenCapture, newBounds, "ironOre")) {
+						System.out.println("Lost track! Finding new ore.");
+						break;
+					}
+				}
+			}
 			
-			/*for (DetectedObject ironOre : ironOres) {
-				ironOre.display();
-			}*/
-			mineClosestIronOre(ironOres, ores);
 			dropInventoryIfFull();
 		}
 	}
@@ -58,36 +76,13 @@ public class IronMiner {
 	}
 	
 	
-	private void mineClosestIronOre(ArrayList<DetectedObject> ironOres, ArrayList<DetectedObject> ores) throws Exception {
+	private void mineClosestIronOre(ArrayList<DetectedObject> ironOres) throws Exception {
 		DetectedObject closestIronOre = getClosestObjectToCharacter(ironOres);
 		if (closestIronOre != null) {		
 			cursor.moveAndLeftClickAtCoordinatesWithRandomness(closestIronOre.getCenterForClicking(), 10, 10);
-			Thread.sleep(84, 219);
+			Thread.sleep(randomizer.nextGaussianWithinRange(IRON_ORE_MINING_TIME_MILLISECONDS - 350, IRON_ORE_MINING_TIME_MILLISECONDS + 1850));
-			DetectedObject closestOre = getClosestObjectToCharacter(ores);
-			if (closestOre != null) {
-				cursor.moveCursorToCoordinatesWithRandomness(closestOre.getCenterForClicking(), 10, 10);
-			}
-			Thread.sleep(randomizer.nextGaussianWithinRange(IRON_ORE_MINING_TIME_MILLISECONDS - 250, IRON_ORE_MINING_TIME_MILLISECONDS + -50));
 		}
-			//Thread.sleep(randomizer.nextGaussianWithinRange(150, 350));
-
-		//cursor.moveCursorToCoordinates(goalPoint);
 	}
-	/*private void mineClosestIronOre(String filename) throws Exception {
-		Point ironOreLocation = getClosestIronOre(filename);
-		if (ironOreLocation != null) {
-			System.out.println("Mineable iron at (" + (ironOreLocation.x + 103) + "," + (ironOreLocation.y + 85) + ")");
-			Point actualIronOreLocation = new Point(ironOreLocation.x + 103, ironOreLocation.y + 85);
-			Rect2d trackerBoundingBox = new Rec2d();
-			//Rectangle trackerBoundingBox = new Rectangle(ironOreLocation.x - 10, ironOreLocation.x + 10, ironOreLocation.y - 10, ironOreLocation.y + 10);
-			//tracker.init(image, trackerBoundingBox);
-			cursor.moveAndLeftClickAtCoordinatesWithRandomness(actualIronOreLocation, 12, 12);
-			Thread.sleep(randomizer.nextGaussianWithinRange(IRON_ORE_MINING_TIME_MILLISECONDS - 350, IRON_ORE_MINING_TIME_MILLISECONDS + 150));
-		}
-		
-	}
-	
-	}*/
 	
 	private DetectedObject getClosestObjectToCharacter(ArrayList<DetectedObject> detectedObjects) {
 		int closestDistanceToCharacter = Integer.MAX_VALUE;

src/ObjectDetector.java

@@ -31,6 +31,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import javax.imageio.ImageIO;
+
 import org.tensorflow.SavedModelBundle;
 import org.tensorflow.Tensor;
 import org.tensorflow.types.UInt8;
@@ -42,28 +43,26 @@ import org.tensorflow.types.UInt8;
 public class ObjectDetector {
   
 	SavedModelBundle model;
-	ArrayList<DetectedObject> detectedObjects;
 	Robot robot;
 	
 	public ObjectDetector() throws AWTException {
 		this.model = SavedModelBundle.load("/home/dpapp/tensorflow-1.5.0/models/raccoon_dataset/results/checkpoint_22948/saved_model/", "serve");
-		this.detectedObjects = new ArrayList<DetectedObject>();
 		this.robot = new Robot();
 	}
 	
 	public void update() throws Exception {
 		// TODO: eliminate IO and pass BufferedImage directly.
-		String fileName = "/home/dpapp/Desktop/RunescapeAI/temp/screenshot.jpg";
+		/*String fileName = "/home/dpapp/Desktop/RunescapeAI/temp/screenshot.jpg";
 		BufferedImage image = captureScreenshotGameWindow();
 		ImageIO.write(image, "jpg", new File(fileName));
-		this.detectedObjects = getRecognizedObjectsFromImage(fileName);
+		this.detectedObjects = getRecognizedObjectsFromImage(fileName);*/
 	}
 	
-	private ArrayList<DetectedObject> getRecognizedObjectsFromImage(String fileName) throws Exception {
+	public ArrayList<DetectedObject> getObjectsInImage(BufferedImage image) throws Exception {
 		List<Tensor<?>> outputs = null;
 		ArrayList<DetectedObject> detectedObjectsInImage = new ArrayList<DetectedObject>();
 		
-        try (Tensor<UInt8> input = makeImageTensor(fileName)) {
+        try (Tensor<UInt8> input = makeImageTensor(image)) {
           outputs =
               model
                   .session()
@@ -93,9 +92,15 @@ public class ObjectDetector {
         return detectedObjectsInImage;
 	}
 	
-	public ArrayList<DetectedObject> getRecognizedObjectsOfClassFromImage(String objectClass) {
+	public boolean isObjectPresentInBoundingBoxInImage(BufferedImage image, Rectangle boundingBox, String objectClass) throws Exception {
+		BufferedImage subImage = image.getSubimage(boundingBox.x, boundingBox.y, boundingBox.width, boundingBox.height);
+		ArrayList<DetectedObject> detectedObjectsInSubImage = getObjectsInImage(subImage);
+		return (getObjectsOfClassInList(detectedObjectsInSubImage, objectClass).size() != 0);
+	}
+    
+	public ArrayList<DetectedObject> getObjectsOfClassInList(ArrayList<DetectedObject> detectedObjects, String objectClass) {
 		ArrayList<DetectedObject> detectedObjectsOfType = new ArrayList<DetectedObject>();
-		for (DetectedObject detectedObject : this.detectedObjects) {
+		for (DetectedObject detectedObject : detectedObjects) {
 			if (detectedObject.getDetectionClass().equals(objectClass)) {
 				detectedObjectsOfType.add(detectedObject);
 			}
@@ -103,8 +108,8 @@ public class ObjectDetector {
 		return detectedObjectsOfType;
 	}
 
-  private static Tensor<UInt8> makeImageTensor(String filename) throws IOException {
+  private static Tensor<UInt8> makeImageTensor(BufferedImage image) throws IOException {
-    BufferedImage img = ImageIO.read(new File(filename));
+    BufferedImage img = ImageIO.read(image);
     if (img.getType() != BufferedImage.TYPE_3BYTE_BGR) {
       throw new IOException(
           String.format(
@@ -128,8 +133,8 @@ public class ObjectDetector {
     }
   }
   
-  private BufferedImage captureScreenshotGameWindow() throws IOException {
+  public BufferedImage captureScreenshotGameWindow() throws IOException, AWTException {
-	Rectangle area = new Rectangle(103, 85, 510, 330);
+		Rectangle area = new Rectangle(Constants.GAME_WINDOW_OFFSET_X, Constants.GAME_WINDOW_OFFSET_Y, Constants.GAME_WINDOW_WIDTH, Constants.GAME_WINDOW_HEIGHT);
-	return robot.createScreenCapture(area);
+		return robot.createScreenCapture(area);
-  }
+	  }
 }

src/ObjectTracker.java

@@ -0,0 +1,17 @@
+import java.awt.Rectangle;
+
+import org.opencv.tracking.Tracker;
+import org.opencv.tracking.TrackerKCF;
+
+public class ObjectTracker {
+
+	Rectangle boundingBox;
+	public ObjectTracker() {
+		/*this.boundingBox = boundingBox;
+		Tracker tracker = TrackerKCF.create();
+		tracker.init(image, boundingBox);*/
+	}
+	
+	
+
+}