Added garbage collection and camera rotation code

src/CameraCalibrator.java

@@ -0,0 +1,40 @@
+import java.awt.AWTException;
+import java.awt.Robot;
+import java.awt.event.KeyEvent;
+import java.awt.image.BufferedImage;
+import java.io.IOException;
+import java.util.ArrayList;
+
+public class CameraCalibrator {
+	
+	Robot robot;
+	Randomizer randomizer;
+	
+	public CameraCalibrator() throws AWTException {
+		robot = new Robot();
+		randomizer = new Randomizer();
+	}
+	
+	public void rotateUntilObjectFound(String objectNameToLookFor) throws Exception {
+		ObjectDetector objectDetector = new ObjectDetector();
+		BufferedImage screenCapture = objectDetector.captureScreenshotGameWindow();
+		
+		ArrayList<DetectedObject> detectedObjects = objectDetector.getObjectsInImage(screenCapture, 0.30);
+		ArrayList<DetectedObject> detectedObjectsToLookFor = objectDetector.getObjectsOfClassInList(detectedObjects, objectNameToLookFor);
+		while (detectedObjectsToLookFor.size() == 0) {
+			randomlyRotateKeyboard();
+			screenCapture = objectDetector.captureScreenshotGameWindow();
+			detectedObjects = objectDetector.getObjectsInImage(screenCapture, 0.30);
+			objectDetector.getObjectsOfClassInList(detectedObjects, objectNameToLookFor);
+		}
+	}
+	
+	private void randomlyRotateKeyboard() throws InterruptedException {
+		Randomizer randomizer = new Randomizer();
+		int keyPressLength = randomizer.nextGaussianWithinRange(350, 1105);
+		robot.keyPress(KeyEvent.VK_LEFT);
+		Thread.sleep(keyPressLength);
+		robot.keyRelease(KeyEvent.VK_LEFT);
+		Thread.sleep(randomizer.nextGaussianWithinRange(120, 250));
+	}
+}

src/Inventory.java

@@ -44,6 +44,11 @@ public class Inventory {
 		updateAllInventorySlots(image);
 	}
 	
+	public void updateLastSlot() throws IOException {
+		BufferedImage image = robot.createScreenCapture(this.inventoryRectangleToCapture);
+		updateLastInventorySlot(image);
+	}
+	
 	private void updateAllInventorySlots(BufferedImage image) throws IOException {
 		for (int row = 0; row < Constants.INVENTORY_NUM_ROWS; row++) {
 			for (int column = 0; column < Constants.INVENTORY_NUM_COLUMNS; column++) {
@@ -52,6 +57,10 @@ public class Inventory {
 		}
 	}
 	
+	private void updateLastInventorySlot(BufferedImage image) throws IOException {
+		inventorySlots[Constants.INVENTORY_NUM_ROWS - 1][Constants.INVENTORY_NUM_COLUMNS - 1].updateInventorySlot(image);
+	}
+	
 	public void updateAndWriteAllInventoryImages() throws IOException {
 		BufferedImage image = robot.createScreenCapture(this.inventoryRectangleToCapture);
 		ImageIO.write(image, "png", new File("/home/dpapp/Desktop/RunescapeAI/Tests/Inventory/inventory_TO_RENAME.png"));
@@ -70,6 +79,10 @@ public class Inventory {
 		return inventorySlots[row][column].getItemNameInInventorySlot(items);
 	}
 	
+	public boolean isLastSlotInInventoryFull() {
+		return !inventorySlots[Constants.INVENTORY_NUM_ROWS - 1][Constants.INVENTORY_NUM_COLUMNS - 1].isInventorySlotEmpty(items);
+	}
+	
 	public boolean isInventoryFull() {
 		// TODO: this will fail if some unexpected item shows up
 		for (int row = 0; row < Constants.INVENTORY_NUM_ROWS; row++) {

src/IronMiner.java

@@ -45,16 +45,32 @@ public class IronMiner {
 	}
 	
 	public void run() throws Exception {
-		while (true) {
+		long startTime = System.currentTimeMillis();
+		long garbageCollectionTime = System.currentTimeMillis();
+		int framesWithoutObjects = 0;
+		
+		while (((System.currentTimeMillis() - startTime) / 1000.0 / 60) < 85) {
 			long frameStartTime = System.currentTimeMillis();
 			BufferedImage screenCapture = objectDetector.captureScreenshotGameWindow();
 			System.out.println("looking for iron ores");
-			//int count = objectDetector.getObjectsInImage(screenCapture, 0.6);
-			ArrayList<DetectedObject> detectedObjects = objectDetector.getObjectsInImage(screenCapture, 0.60);
-			//ArrayList<DetectedObject> ironOres = objectDetector.getObjectsOfClassInList(detectedObjects, "ironOre");
+			
+			ArrayList<DetectedObject> detectedObjects = objectDetector.getObjectsInImage(screenCapture, 0.30);
+			ArrayList<DetectedObject> ironOres = objectDetector.getObjectsOfClassInList(detectedObjects, "ironOre");
 			System.out.println("Found " + detectedObjects.size() + " objects.");
 			
-			/*DetectedObject closestIronOre = getClosestObjectToCharacter(ironOres);
+			if (ironOres.size() == 0) {
+				framesWithoutObjects++;
+				System.out.println("no objects found!");
+			}
+			else {
+				framesWithoutObjects = 0;
+			}
+			if (framesWithoutObjects > 50) {
+				CameraCalibrator cameraCalibrator = new CameraCalibrator();
+				cameraCalibrator.rotateUntilObjectFound("ironOre");
+			}
+			
+			DetectedObject closestIronOre = getClosestObjectToCharacter(ironOres);
 			if (closestIronOre != null) {
 				Rect2d boundingBox = closestIronOre.getBoundingRect2d();
 				ObjectTracker ironOreTracker = new ObjectTracker(screenCapture, boundingBox);
@@ -62,25 +78,43 @@ public class IronMiner {
 				cursor.moveAndLeftClickAtCoordinatesWithRandomness(closestIronOre.getCenterForClicking(), 10, 10);
 				
 				long miningStartTime = System.currentTimeMillis();
-				int maxTimeToMine = randomizer.nextGaussianWithinRange(3500, 5000);
+				int maxTimeToMine = randomizer.nextGaussianWithinRange(3400, 4519);
 				
 				boolean objectTrackingFailure = false;
-				while (!objectTrackingFailure && !isTimeElapsedOverLimit(miningStartTime, maxTimeToMine)) {
+				boolean oreAvailable = true;
+				int oreLostCount = 0;
+				while (!objectTrackingFailure && oreLostCount < 3 && !isTimeElapsedOverLimit(miningStartTime, maxTimeToMine)) {
+					long trackingFrameStartTime = System.currentTimeMillis();
+					
 					screenCapture = objectDetector.captureScreenshotGameWindow();
+					detectedObjects = objectDetector.getObjectsInImage(screenCapture, 0.15);
+					ironOres = objectDetector.getObjectsOfClassInList(detectedObjects, "ironOre");
 					objectTrackingFailure = ironOreTracker.update(screenCapture, boundingBox);
+					oreAvailable = objectDetector.isObjectPresentInBoundingBoxInImage(ironOres, boundingBox, "ironOre");
+					if (!oreAvailable) {
+						oreLostCount++;
+					}
+					else {
+						oreLostCount = 0;
+					}
+					System.out.println("Tracking timespan: " + (System.currentTimeMillis() - trackingFrameStartTime));
 				}
-			}*/
+			}
 			
-			// TODO: change this so that we only check the last slot for an item.
-			// 
-			//dropInventoryIfFull();
-			System.out.println("Timespan: " + (System.currentTimeMillis() - frameStartTime));
+			
+			// Garbage Collection
+			if (((System.currentTimeMillis() - garbageCollectionTime) / 1000.0 / 60) > 10) {
+				System.out.println("Running garbage collection.");
+				System.gc();
+				garbageCollectionTime = System.currentTimeMillis() + randomizer.nextGaussianWithinRange(8500, 19340);
+			}
+			dropInventoryIfFull();
 		}
 	}
 	
 	private void dropInventoryIfFull() throws Exception {
-		inventory.update();
-		if (inventory.isInventoryFull()) {
+		inventory.updateLastSlot();
+		if (inventory.isLastSlotInInventoryFull()) {
 			cursorTask.optimizedDropAllItemsInInventory(cursor, inventory);
 		}
 	}

src/ObjectTracker.java

@@ -1,7 +1,14 @@
 import java.awt.image.BufferedImage;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
 
+import javax.imageio.ImageIO;
+
+import org.opencv.core.CvType;
 import org.opencv.core.Mat;
+import org.opencv.core.MatOfByte;
 import org.opencv.core.Rect2d;
+import org.opencv.imgcodecs.Imgcodecs;
 import org.opencv.tracking.Tracker;
 import org.opencv.tracking.TrackerBoosting;
 
@@ -10,9 +17,9 @@ public class ObjectTracker {
 	Tracker objectTracker;
 	private int numberOfFramesLostFor;
 	
-	public ObjectTracker(BufferedImage image, Rect2d boundingBox) {
+	public ObjectTracker(BufferedImage image, Rect2d boundingBox) throws IOException {
 		this.objectTracker = TrackerBoosting.create();
-		this.objectTracker.init(bufferedImageToMat(image), boundingBox);
+		this.objectTracker.init(BufferedImage2Mat(image), boundingBox);
 		this.numberOfFramesLostFor = 0;
 	}
 	
@@ -23,8 +30,9 @@ public class ObjectTracker {
 	}
 	
 	
-	public boolean update(BufferedImage image, Rect2d boundingBox) {
-		boolean trackingSuccessful = objectTracker.update(bufferedImageToMat(image), boundingBox);
+	public boolean update(BufferedImage image, Rect2d boundingBox) throws IOException {
+		boolean trackingSuccessful = objectTracker.update(BufferedImage2Mat(image), boundingBox);
+		System.out.print("Tracking: " + trackingSuccessful + ". Counter at ");
 		updateNumberOfFramesLostFor(trackingSuccessful);
 		return isObjectTrackingFailure();
 	}
@@ -45,6 +53,21 @@ public class ObjectTracker {
 	}
 	
 	private boolean isObjectTrackingFailure() {
+		System.out.println(numberOfFramesLostFor);
 		return numberOfFramesLostFor > 3;
 	}
+	
+	/*private Mat bufferedImageToMat(BufferedImage sourceImage) {
+		Mat mat = new Mat(sourceImage.getHeight(), sourceImage.getWidth(), CvType.CV_8UC3);
+		byte[] data = ((DataBufferByte) sourceImage.getRaster().getDataBuffer()).getData();
+		mat.put(0, 0, data);
+		return mat;
+	}*/
+	
+	private Mat BufferedImage2Mat(BufferedImage image) throws IOException {
+	    ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
+	    ImageIO.write(image, "jpg", byteArrayOutputStream);
+	    byteArrayOutputStream.flush();
+	    return Imgcodecs.imdecode(new MatOfByte(byteArrayOutputStream.toByteArray()), Imgcodecs.CV_LOAD_IMAGE_UNCHANGED);
+	}
 }