Cleaning up, adding random XP left check

.classpath

@@ -13,5 +13,6 @@
 	</classpathentry>
 	<classpathentry kind="con" path="org.eclipse.jdt.USER_LIBRARY/Tensorflow"/>
 	<classpathentry kind="con" path="org.eclipse.jdt.USER_LIBRARY/OpenCV"/>
+	<classpathentry kind="con" path="org.eclipse.jdt.junit.JUNIT_CONTAINER/5"/>
 	<classpathentry kind="output" path="target/classes"/>
 </classpath>

src/CameraCalibrator.java

@@ -8,15 +8,12 @@ import java.util.ArrayList;
 public class CameraCalibrator {
 	
 	Robot robot;
-	Randomizer randomizer;
 	
 	public CameraCalibrator() throws AWTException {
 		robot = new Robot();
-		randomizer = new Randomizer();
 	}
 	
-	public void rotateUntilObjectFound(String objectNameToLookFor) throws Exception {
+	public void rotateUntilObjectFound(ObjectDetector objectDetector, String objectNameToLookFor) throws Exception {
-		ObjectDetector objectDetector = new ObjectDetector();
 		BufferedImage screenCapture = objectDetector.captureScreenshotGameWindow();
 		
 		ArrayList<DetectedObject> detectedObjects = objectDetector.getObjectsInImage(screenCapture, 0.30);
@@ -30,11 +27,10 @@ public class CameraCalibrator {
 	}
 	
 	private void randomlyRotateKeyboard() throws InterruptedException {
-		Randomizer randomizer = new Randomizer();
+		int keyPressLength = Randomizer.nextGaussianWithinRange(150, 505);
-		int keyPressLength = randomizer.nextGaussianWithinRange(350, 1105);
 		robot.keyPress(KeyEvent.VK_LEFT);
 		Thread.sleep(keyPressLength);
 		robot.keyRelease(KeyEvent.VK_LEFT);
-		Thread.sleep(randomizer.nextGaussianWithinRange(120, 250));
+		Thread.sleep(Randomizer.nextGaussianWithinRange(120, 250));
 	}
 }

src/Constants.java

@@ -1,12 +1,19 @@
+import java.awt.Point;
+import java.awt.Rectangle;
 
 public class Constants {
+	public static final int FULL_WINDOW_WIDTH = 960;
+	public static final int FULL_WINDOW_HEIGHT = 1080;
+	
 	public static final int GAME_WINDOW_OFFSET_X = 103;
 	public static final int GAME_WINDOW_OFFSET_Y = 85;
 	public static final int GAME_WINDOW_WIDTH = 510;
 	public static final int GAME_WINDOW_HEIGHT = 330;
 	
+	public static final int CHARACTER_CENTER_X = 356;
+	public static final int CHARACTER_CENTER_Y = 247;
 	
-	public static final int INVENTORY_WINDOW_OFFSET_X = 655; //top left corner of inventory, fromm top left corner of screen
+	public static final int INVENTORY_WINDOW_OFFSET_X = 655;
 	public static final int INVENTORY_WINDOW_OFFSET_Y = 290;
 	public static final int INVENTORY_WINDOW_WIDTH = 171;
 	public static final int INVENTORY_WINDOW_HEIGHT = 350;
@@ -14,4 +21,49 @@ public class Constants {
 	public static final int INVENTORY_SLOT_HEIGHT = 254 / 7;
 	public static final int INVENTORY_NUM_ROWS = 4;
 	public static final int INVENTORY_NUM_COLUMNS = 7;
+	
+	
+	// ---------------------------------- ICONS ----------------------------------------------
+	
+	public static final int STATS_ICON_OFFSET_X = 660;
+	public static final int STATS_ICON_OFFSET_Y = 250;
+	public static final int STATS_ICON_WIDTH = 25;
+	public static final int STATS_ICON_HEIGHT = 25;
+	
+	public static final int INVENTORY_ICON_OFFSET_X = 728;
+	public static final int INVENTORY_ICON_OFFSET_Y = 250;
+	public static final int INVENTORY_ICON_WIDTH = 25;
+	public static final int INVENTORY_ICON_HEIGHT = 25;
+	
+	public static final int LOGOUT_ICON_OFFSET_X = 728;
+	public static final int LOGOUT_ICON_OFFSET_Y = 528;
+	public static final int LOGOUT_ICON_WIDTH = 25;
+	public static final int LOGOUT_ICON_HEIGHT = 25;
+	
+	// ---------------------------------- STATS ----------------------------------------------
+	
+	public static final int MINING_XP_STAT_OFFSET_X = 777;
+	public static final int MINING_XP_STAT_OFFSET_Y = 288;
+	public static final int MINING_XP_STAT_WIDTH = 50;
+	public static final int MINING_XP_STAT_HEIGHT = 23;
+	
+	public static Point getCharacterLocation() {
+		return new Point(CHARACTER_CENTER_X, CHARACTER_CENTER_Y);
+	}
+	
+	public static Rectangle getStatsIconRectangle() {
+		return new Rectangle(STATS_ICON_OFFSET_X, STATS_ICON_OFFSET_Y, STATS_ICON_WIDTH, STATS_ICON_HEIGHT);
+	}
+	
+	public static Rectangle getInventoryIconRectangle() {
+		return new Rectangle(INVENTORY_ICON_OFFSET_X, INVENTORY_ICON_OFFSET_Y, INVENTORY_ICON_WIDTH, INVENTORY_ICON_HEIGHT);
+	}
+	
+	public static Rectangle getLogoutIconRectangle() {
+		return new Rectangle(LOGOUT_ICON_OFFSET_X, LOGOUT_ICON_OFFSET_Y, LOGOUT_ICON_WIDTH, LOGOUT_ICON_HEIGHT);
+	}
+	
+	public static Rectangle getMiningXPRectangle() {
+		return new Rectangle(MINING_XP_STAT_OFFSET_X, MINING_XP_STAT_OFFSET_Y, MINING_XP_STAT_WIDTH, MINING_XP_STAT_HEIGHT);
+	}
 }

src/Cursor.java

@@ -4,6 +4,7 @@ import java.awt.AWTException;
 import java.awt.MouseInfo;
 import java.awt.Point;
 import java.awt.PointerInfo;
+import java.awt.Rectangle;
 import java.awt.Robot;
 import java.awt.event.InputEvent;
 import java.io.BufferedReader;
@@ -16,6 +17,8 @@ import java.util.Random;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
+import org.opencv.core.Rect2d;
+
 public class Cursor {
 	
 	public static final int NUMBER_OF_DISTANCES = 2203; // For 1080p screen
@@ -107,6 +110,24 @@ public class Cursor {
 		return randomizedGoalPoint; // Return the point we moved to in case we need precise movement afterwards
 	}
 	
+	public Point moveInsideBoundingRectangle(Rectangle boundingRectangle) throws Exception {
+		Point randomizedGoalPoint = getRandomPointInBoundingRectangle(boundingRectangle);
+		moveCursorToCoordinates(randomizedGoalPoint);
+		return randomizedGoalPoint;
+	}
+	
+	public Point moveAndLeftClickInBoundingRectangle(Rectangle boundingRectangle) throws Exception {
+		Point randomizedGoalPoint = getRandomPointInBoundingRectangle(boundingRectangle);
+		moveAndLeftClickAtCoordinates(randomizedGoalPoint);
+		return randomizedGoalPoint;
+	}
+	
+	private Point getRandomPointInBoundingRectangle(Rectangle boundingRectangle) {
+		int x = randomizer.nextGaussianWithinRange(boundingRectangle.x, boundingRectangle.x + boundingRectangle.width);
+		int y = randomizer.nextGaussianWithinRange(boundingRectangle.y, boundingRectangle.y + boundingRectangle.height);
+		return new Point(x, y);
+	}
+	
 	public Point moveAndLeftClickAtCoordinatesWithRandomness(Point goalPoint, int xToleranceLeft, int xToleranceRight, int yTolerance) throws Exception {
 		Point randomizedGoalPoint = randomizePoint(goalPoint, xToleranceLeft, xToleranceRight, yTolerance);
 		moveAndLeftClickAtCoordinates(randomizedGoalPoint);

src/HumanBehavior.java

@@ -0,0 +1,30 @@
+
+public class HumanBehavior {
+
+	private int minimumTimeTillXPCheck = 35;
+	private int maximumTimeTillXPCheck = 75;
+	
+	long nextTimeToCheckMiningXP;
+	
+	public HumanBehavior() {
+		nextTimeToCheckMiningXP = System.currentTimeMillis() + getNextTimeTillXPCheck();
+	}
+	
+	public void checkMiningXP(Cursor cursor) throws Exception {
+		cursor.moveAndLeftClickInBoundingRectangle(Constants.getStatsIconRectangle());
+		cursor.moveInsideBoundingRectangle(Constants.getMiningXPRectangle());
+		Thread.sleep(Randomizer.nextGaussianWithinRange(1750, 3420));
+		cursor.moveAndLeftClickInBoundingRectangle(Constants.getInventoryIconRectangle());
+	}
+	
+	public void randomlyCheckMiningXP(Cursor cursor) throws Exception {
+		if (System.currentTimeMillis() > nextTimeToCheckMiningXP) {
+			checkMiningXP(cursor);
+			nextTimeToCheckMiningXP = System.currentTimeMillis() + getNextTimeTillXPCheck();
+		}
+	}
+	
+	private int getNextTimeTillXPCheck() {
+		return Randomizer.nextGaussianWithinRange(1000 * 60 * minimumTimeTillXPCheck, 1000 * 60 * maximumTimeTillXPCheck);
+	}
+}

src/ImageCollector.java

@@ -12,6 +12,7 @@ public class ImageCollector {
 
 	public String screenshotOutputDirectory;
 	public Rectangle gameWindowRectangle;
+	public Rectangle fullWindowRectangle;
 	public Robot robot;
 	
 	/*
@@ -26,6 +27,7 @@ public class ImageCollector {
 	
 	public ImageCollector(String screenshotOutputDirectory) throws AWTException {
 		initializeGameWindowRectangle();
+		initializeFullWindowRectangle();
 		this.screenshotOutputDirectory = screenshotOutputDirectory;
 		this.robot = new Robot();
 	}
@@ -34,6 +36,10 @@ public class ImageCollector {
 		this.gameWindowRectangle = new Rectangle(Constants.GAME_WINDOW_OFFSET_X, Constants.GAME_WINDOW_OFFSET_Y, Constants.GAME_WINDOW_WIDTH, Constants.GAME_WINDOW_HEIGHT);
 	}
 	
+	private void initializeFullWindowRectangle() {
+		this.fullWindowRectangle = new Rectangle(0, 0, Constants.FULL_WINDOW_WIDTH, Constants.FULL_WINDOW_HEIGHT);
+	}
+	
 	public void collectImages(String itemName) throws IOException, InterruptedException, AWTException {
 		int itemCounter = getItemCounter(itemName);
 		int numImagesToCapture = 50;
@@ -74,6 +80,12 @@ public class ImageCollector {
 		System.out.println("Wrote file: " + fileName);
 	}
 	
+	public void captureAndSaveFullWindow() throws IOException {
+		BufferedImage imageCaptured = robot.createScreenCapture(fullWindowRectangle);
+		ImageIO.write(imageCaptured, "jpg", new File(getFileName("fullWindow", 0)));
+		System.out.println("Wrote file.");
+	}
+	
 	private String getFileName(String itemName, int counter) {
 		return screenshotOutputDirectory + itemName + "_" + counter + ".jpg";
 	}
@@ -85,8 +97,9 @@ public class ImageCollector {
 	
 	public static void main(String[] args) throws Exception
     {
-        ImageCollector imageCollector = new ImageCollector("/home/dpapp/Desktop/RunescapeAI/TensorFlow/Ores/Images/");
+        ImageCollector imageCollector = new ImageCollector("/home/dpapp/Desktop/RunescapeAI/Images/");
-        imageCollector.collectImages("ore");
+        //imageCollector.collectImages("ore");
         //imageCollector.generateInventoryImages();
+        imageCollector.captureAndSaveFullWindow();
     }
 }

src/IronMiner.java

@@ -25,14 +25,14 @@ public class IronMiner {
 	
 	public static final int IRON_ORE_MINING_TIME_MILLISECONDS = 1320;
 	public static final int MAXIMUM_DISTANCE_TO_WALK_TO_IRON_ORE = 400;
-	public static final Point GAME_WINDOW_CENTER = new Point(Constants.GAME_WINDOW_WIDTH / 2, Constants.GAME_WINDOW_HEIGHT / 2);
 	
 	Cursor cursor; 
 	CursorTask cursorTask;
 	Inventory inventory;
 	ObjectDetector objectDetector;
 	Robot robot;
-	Randomizer randomizer;
+	HumanBehavior humanBehavior;
+	CameraCalibrator cameraCalibrator;
 	
 	public IronMiner() throws AWTException, IOException 
 	{
@@ -41,34 +41,19 @@ public class IronMiner {
 		inventory = new Inventory();
 		objectDetector = new ObjectDetector();
 		robot = new Robot();
-		randomizer = new Randomizer();
+		humanBehavior = new HumanBehavior();
+		cameraCalibrator = new CameraCalibrator();
 	}
 	
 	public void run() throws Exception {
 		long startTime = System.currentTimeMillis();
-		long garbageCollectionTime = System.currentTimeMillis() + 60 * 5 * 1000;
-		int framesWithoutObjects = 0;
 		
 		while (((System.currentTimeMillis() - startTime) / 1000.0 / 60) < 85) {
-			long frameStartTime = System.currentTimeMillis();
 			BufferedImage screenCapture = objectDetector.captureScreenshotGameWindow();
-			System.out.println("looking for iron ores");
 			
 			ArrayList<DetectedObject> detectedObjects = objectDetector.getObjectsInImage(screenCapture, 0.30);
 			ArrayList<DetectedObject> ironOres = objectDetector.getObjectsOfClassInList(detectedObjects, "ironOre");
-			System.out.println("Found " + detectedObjects.size() + " objects.");
 			
-			if (ironOres.size() == 0) {
-				framesWithoutObjects++;
-				System.out.println("no objects found!");
-			}
-			else {
-				framesWithoutObjects = 0;
-			}
-			if (framesWithoutObjects > 50) {
-				CameraCalibrator cameraCalibrator = new CameraCalibrator();
-				cameraCalibrator.rotateUntilObjectFound("ironOre");
-			}
 			
 			DetectedObject closestIronOre = getClosestObjectToCharacter(ironOres);
 			if (closestIronOre != null) {
@@ -78,7 +63,7 @@ public class IronMiner {
 				cursor.moveAndLeftClickAtCoordinatesWithRandomness(closestIronOre.getCenterForClicking(), 10, 10);
 				
 				long miningStartTime = System.currentTimeMillis();
-				int maxTimeToMine = randomizer.nextGaussianWithinRange(3400, 4519);
+				int maxTimeToMine = Randomizer.nextGaussianWithinRange(3400, 4519);
 				
 				boolean objectTrackingFailure = false;
 				boolean oreAvailable = true;
@@ -101,13 +86,7 @@ public class IronMiner {
 				}
 			}
 			
-			
+			humanBehavior.randomlyCheckMiningXP(cursor);
-			// Garbage Collection
-			if (((System.currentTimeMillis() - garbageCollectionTime) / 1000.0 / 60) > 0) {
-				System.out.println("Running garbage collection.");
-				System.gc();
-				garbageCollectionTime = System.currentTimeMillis() + randomizer.nextGaussianWithinRange(8500, 19340) * 60;
-			}
 			dropInventoryIfFull();
 		}
 	}
@@ -116,6 +95,7 @@ public class IronMiner {
 		inventory.updateLastSlot();
 		if (inventory.isLastSlotInInventoryFull()) {
 			cursorTask.optimizedDropAllItemsInInventory(cursor, inventory);
+			Thread.sleep(Randomizer.nextGaussianWithinRange(1104, 1651));
 		}
 	}
 	
@@ -128,7 +108,7 @@ public class IronMiner {
 		DetectedObject closestObjectToCharacter = null;
 		
 		for (DetectedObject detectedObject : detectedObjects) {
-			int objectDistanceToCharacter = getDistanceBetweenPoints(GAME_WINDOW_CENTER, detectedObject.getCenterForClicking());
+			int objectDistanceToCharacter = getDistanceBetweenPoints(Constants.getCharacterLocation(), detectedObject.getCenterForClicking());
 			if (objectDistanceToCharacter < closestDistanceToCharacter) {
 				closestDistanceToCharacter = objectDistanceToCharacter;
 				closestObjectToCharacter = detectedObject;

src/ObjectDetector.java

@@ -53,11 +53,9 @@ public class ObjectDetector {
 		this.robot = new Robot();
 	}	
 	
-	// Goal: reduce this to < 50 ms
 	public ArrayList<DetectedObject> getObjectsInImage(BufferedImage image, double scoreThreshold) throws Exception {
 		List<Tensor<?>> outputs = null;
 		ArrayList<DetectedObject> detectedObjectsInImage = new ArrayList<DetectedObject>();
-		//int count = 0;
 		makeImageTensor(image);
         try (Tensor<UInt8> input = makeImageTensor(image)) {
           outputs =
@@ -69,6 +67,7 @@ public class ObjectDetector {
                   .fetch("detection_classes")
                   .fetch("detection_boxes")
                   .run();
+          input.close();
         }
         
         try (Tensor<Float> scoresT = outputs.get(0).expect(Float.class);
@@ -82,17 +81,14 @@ public class ObjectDetector {
 
             for (int i = 0; i < scores.length; ++i) {
               if (scores[i] > scoreThreshold) {
-            	  //count++;
             	  detectedObjectsInImage.add(new DetectedObject(scores[i], classes[i], boxes[i]));
               }
             }
-          }
+        }
-       // return count;
+        outputs = null;
         return detectedObjectsInImage;
 	}
 	
-	
-	// This is too slow -> running object detection each time reduces the framerate to 3fps, which messses up object tracking
 	public boolean isObjectPresentInBoundingBoxInImage(ArrayList<DetectedObject> detectedObjects, Rect2d boundingBox, String objectClass) throws Exception {
 		for (DetectedObject detectedObject : detectedObjects) {
 			if (detectedObject.getDetectionClass().equals(objectClass)) {
@@ -117,7 +113,7 @@ public class ObjectDetector {
 		return detectedObjectsOfType;
 	}
 
-  private static Tensor<UInt8> makeImageTensor(BufferedImage image) throws IOException {
+  private Tensor<UInt8> makeImageTensor(BufferedImage image) throws IOException {
 	BufferedImage formattedImage = convertBufferedImage(image, BufferedImage.TYPE_3BYTE_BGR);
 	byte[] data = ((DataBufferByte) formattedImage.getData().getDataBuffer()).getData();
 	bgr2rgb(data);  
@@ -127,7 +123,7 @@ public class ObjectDetector {
     return Tensor.create(UInt8.class, shape, ByteBuffer.wrap(data));
   }
   
-  private static BufferedImage convertBufferedImage(BufferedImage sourceImage, int bufferedImageType) {
+  private BufferedImage convertBufferedImage(BufferedImage sourceImage, int bufferedImageType) {
     BufferedImage image = new BufferedImage(sourceImage.getWidth(), sourceImage.getHeight(), bufferedImageType);
     Graphics2D g2d = image.createGraphics();
     g2d.drawImage(sourceImage, 0, 0, null);
@@ -135,16 +131,16 @@ public class ObjectDetector {
     return image;
   }
   
-  private static void bgr2rgb(byte[] data) {
+  private void bgr2rgb(byte[] data) {
-	    for (int i = 0; i < data.length; i += 3) {
+	for (int i = 0; i < data.length; i += 3) {
-	      byte tmp = data[i];
+	  byte tmp = data[i];
-	      data[i] = data[i + 2];
+	  data[i] = data[i + 2];
-	      data[i + 2] = tmp;
+	  data[i + 2] = tmp;
-	    }
+    }
-	  }
+  }
   
   public BufferedImage captureScreenshotGameWindow() throws IOException, AWTException {
-		Rectangle area = new Rectangle(Constants.GAME_WINDOW_OFFSET_X, Constants.GAME_WINDOW_OFFSET_Y, Constants.GAME_WINDOW_WIDTH, Constants.GAME_WINDOW_HEIGHT);
+	Rectangle area = new Rectangle(Constants.GAME_WINDOW_OFFSET_X, Constants.GAME_WINDOW_OFFSET_Y, Constants.GAME_WINDOW_WIDTH, Constants.GAME_WINDOW_HEIGHT);
-		return robot.createScreenCapture(area);
+	return robot.createScreenCapture(area);
-	  }
+  }
 }

src/ObjectTrackerTest.java

@@ -31,7 +31,7 @@ class ObjectTrackerTest {
 
 	
 	@Test
-	void testObjectTrackingWithAllVideosInDirectory() {
+	void testObjectTrackingWithAllVideosInDirectory() throws Exception {
 		String videoDirectory = "/home/dpapp/Videos/";
 		for (File video : getListOfFilesFromDirectory(videoDirectory)) {
 			if (video.isFile()) {
@@ -75,14 +75,13 @@ class ObjectTrackerTest {
 				boolean trackingSuccess = objectTrackers.get(i).update(frame, boundingBoxes.get(i));
 				assertTrue(trackingSuccess);
 							
-				g.drawRect((int) boundingBoxes.get(i).x, (int) boundingBoxes.get(i).y, (int) boundingBoxes.get(i).width, (int) boundingBoxes.get(i).height);			
+				//g.drawRect((int) boundingBoxes.get(i).x, (int) boundingBoxes.get(i).y, (int) boundingBoxes.get(i).width, (int) boundingBoxes.get(i).height);			
 			}
 			ImageIO.write(screencapture, "jpg", new File(videoDirectory + "/frame_" + videoFileName  + counter + ".jpg"));
 			counter++;
 		}
 	}
 
-	
 	private BufferedImage Mat2BufferedImage(Mat matrix)throws Exception {        
 	    MatOfByte mob=new MatOfByte();
 	    Imgcodecs.imencode(".jpg", matrix, mob);

src/Randomizer.java

@@ -3,43 +3,22 @@ import java.util.Random;
 //import Jama.Matrix;
 
 public class Randomizer {
-
-	Random random;
 	
-	public Randomizer() {
+	public static int nextGaussianWithinRange(double rangeBegin, double rangeEnd) {
-		random = new Random();
+		Random random = new Random();
-	}
-	
-	public int nextGaussianWithinRange(double rangeBegin, double rangeEnd) {
 		double rangeMean = (rangeEnd + rangeBegin) / 2.0;
 		double rangeSTD = (rangeEnd - rangeMean) / 3.0;
+		
 		double result = random.nextGaussian() * rangeSTD + rangeMean;
 		while (result > rangeEnd || result < rangeBegin) {
-			System.out.println("Gaussian result out of range...");
-			System.out.println(rangeMean + ", std: " + rangeSTD);
 			result = random.nextGaussian() * rangeSTD + rangeMean;
 		}
 		return (int) result;
 	}
-	public Point generatePeakForTransformationParabola(int pathDistance) {
+	public static Point generatePeakForTransformationParabola(int pathDistance) {
 		double maxTransformationScale = 0.2;
 		int peakX = nextGaussianWithinRange(0, pathDistance);
 		int peakY = nextGaussianWithinRange(0, pathDistance * maxTransformationScale);
 		return new Point(peakX, peakY);
 	}
-	
-	/*public double[] generateParabolaEquation(int pathDistance) {
-		Point peakPoint = generatePeakForTransformationParabola(pathDistance);
-		double[][] lhsMatrix = {{0, 0, 1}, {peakPoint.x * peakPoint.x, peakPoint.x, 1}, {pathDistance * pathDistance, pathDistance, 1}};
-		double[][] rhsMatrix = {{0, peakPoint.y, 0}};
-		
-		Matrix lhs = new Matrix(lhsMatrix);
-		Matrix rhs = new Matrix(rhsMatrix);
-		Matrix ans = lhs.solve(rhs);
-		
-		double[] result = {ans.get(0, 0), ans.get(1, 0), ans.get(2, 0)};
-		return result;
-	}*/
-	
-	//public Point transformPoint
 }