added random detection, not enough testing

src/CameraCalibrator.java

@@ -16,21 +16,21 @@ public class CameraCalibrator {
 	public void rotateUntilObjectFound(ObjectDetector objectDetector, String objectNameToLookFor) throws Exception {
 		BufferedImage screenCapture = objectDetector.captureScreenshotGameWindow();
 		
-		ArrayList<DetectedObject> detectedObjects = objectDetector.getObjectsInImage(screenCapture, 0.30);
+		ArrayList<DetectedObject> detectedObjects = objectDetector.getObjectsInImage(screenCapture, 0.40);
 		ArrayList<DetectedObject> detectedObjectsToLookFor = objectDetector.getObjectsOfClassInList(detectedObjects, objectNameToLookFor);
 		while (detectedObjectsToLookFor.size() == 0) {
 			randomlyRotateKeyboard();
 			screenCapture = objectDetector.captureScreenshotGameWindow();
-			detectedObjects = objectDetector.getObjectsInImage(screenCapture, 0.30);
+			detectedObjects = objectDetector.getObjectsInImage(screenCapture, 0.40);
 			detectedObjectsToLookFor = objectDetector.getObjectsOfClassInList(detectedObjects, objectNameToLookFor);
 		}
 	}
 	
 	private void randomlyRotateKeyboard() throws InterruptedException {
-		int keyPressLength = Randomizer.nextGaussianWithinRange(150, 505);
+		int keyPressLength = Randomizer.nextGaussianWithinRange(50, 160);
 		robot.keyPress(KeyEvent.VK_LEFT);
 		Thread.sleep(keyPressLength);
 		robot.keyRelease(KeyEvent.VK_LEFT);
-		Thread.sleep(Randomizer.nextGaussianWithinRange(120, 250));
+		Thread.sleep(Randomizer.nextGaussianWithinRange(80, 118));
 	}
 }

src/Cursor.java

@@ -88,6 +88,7 @@ public class Cursor {
 	}
 	
 	public void rightClickCursor() throws InterruptedException {
+		Thread.sleep(50, 80);
 		robot.mousePress(InputEvent.BUTTON3_DOWN_MASK);
 		Thread.sleep(getRandomClickLength() + 20);
 		robot.mouseRelease(InputEvent.BUTTON3_DOWN_MASK);
@@ -255,4 +256,8 @@ public class Cursor {
 		}
 		System.out.println("--------------");
 	}
+
+	public Point getOffsetPoint(Point point) {
+		return new Point(point.x + Constants.GAME_WINDOW_OFFSET_X, point.y + Constants.GAME_WINDOW_OFFSET_Y);
+	}
 }

src/HumanBehavior.java

@@ -1,3 +1,4 @@
+import java.util.ArrayList;
 
 public class HumanBehavior {
 

src/ImageCollector.java

@@ -25,6 +25,10 @@ public class ImageCollector {
 	 * detect last file name
 	 */
 	
+	 public BufferedImage captureScreenshotGameWindow() throws IOException, AWTException {
+			return robot.createScreenCapture(gameWindowRectangle);
+    }
+	
 	public ImageCollector(String screenshotOutputDirectory) throws AWTException {
 		initializeGameWindowRectangle();
 		initializeFullWindowRectangle();
@@ -98,8 +102,8 @@ public class ImageCollector {
 	public static void main(String[] args) throws Exception
     {
         ImageCollector imageCollector = new ImageCollector("/home/dpapp/Desktop/RunescapeAI/Images/");
-        //imageCollector.collectImages("ore");
+        imageCollector.collectImages("chatDialogue");
         //imageCollector.generateInventoryImages();
-        imageCollector.captureAndSaveFullWindow();
+        //imageCollector.captureAndSaveFullWindow();
     }
 }

src/IronMiner.java

@@ -33,6 +33,8 @@ public class IronMiner {
 	Robot robot;
 	HumanBehavior humanBehavior;
 	CameraCalibrator cameraCalibrator;
+	RandomDetector randomDetector;
+	WorldHopper worldHopper;
 	
 	public IronMiner() throws AWTException, IOException 
 	{
@@ -43,17 +45,31 @@ public class IronMiner {
 		robot = new Robot();
 		humanBehavior = new HumanBehavior();
 		cameraCalibrator = new CameraCalibrator();
+		randomDetector = new RandomDetector();
+		worldHopper = new WorldHopper();
 	}
 	
 	public void run() throws Exception {
 		long startTime = System.currentTimeMillis();
+		int noIronOresDetected = 0;
 		
-		while (((System.currentTimeMillis() - startTime) / 1000.0 / 60) < 85) {
+		int count = 0;
+		while (((System.currentTimeMillis() - startTime) / 1000.0 / 75) < 75) {
+			count++;
 			BufferedImage screenCapture = objectDetector.captureScreenshotGameWindow();
 			
-			ArrayList<DetectedObject> detectedObjects = objectDetector.getObjectsInImage(screenCapture, 0.30);
+			ArrayList<DetectedObject> detectedObjects = objectDetector.getObjectsInImage(screenCapture, 0.20);
-			ArrayList<DetectedObject> ironOres = objectDetector.getObjectsOfClassInList(detectedObjects, "ironOre");
+			ArrayList<DetectedObject> ironOres = objectDetector.getIronOres(detectedObjects);
 			
+			if (ironOres.size() == 0) {
+				noIronOresDetected++;
+			}
+			else {
+				noIronOresDetected = 0;
+			}
+			if (noIronOresDetected > 80) {
+				cameraCalibrator.rotateUntilObjectFound(objectDetector, "ironOre");
+			}
 			
 			DetectedObject closestIronOre = getClosestObjectToCharacter(ironOres);
 			if (closestIronOre != null) {
@@ -87,10 +103,15 @@ public class IronMiner {
 			}
 			
 			humanBehavior.randomlyCheckMiningXP(cursor);
+			randomDetector.dealWithRandoms(screenCapture, cursor);
 			dropInventoryIfFull();
+			if (count % 100 == 0) {
+				System.out.println((System.currentTimeMillis() - startTime) / 1000);
+			}
 		}
 	}
 	
+	
 	private void dropInventoryIfFull() throws Exception {
 		inventory.updateLastSlot();
 		if (inventory.isLastSlotInInventoryFull()) {

src/ObjectDetector.java

@@ -85,7 +85,6 @@ public class ObjectDetector {
               }
             }
         }
-        outputs = null;
         return detectedObjectsInImage;
 	}
 	
@@ -112,6 +111,10 @@ public class ObjectDetector {
 		}
 		return detectedObjectsOfType;
 	}
+	
+	public ArrayList<DetectedObject> getIronOres(ArrayList<DetectedObject> detectedObjects) {
+		return getObjectsOfClassInList(detectedObjects, "ironOre");
+	}
 
   private Tensor<UInt8> makeImageTensor(BufferedImage image) throws IOException {
 	BufferedImage formattedImage = convertBufferedImage(image, BufferedImage.TYPE_3BYTE_BGR);
@@ -140,7 +143,7 @@ public class ObjectDetector {
   }
   
   public BufferedImage captureScreenshotGameWindow() throws IOException, AWTException {
-	Rectangle area = new Rectangle(Constants.GAME_WINDOW_OFFSET_X, Constants.GAME_WINDOW_OFFSET_Y, Constants.GAME_WINDOW_WIDTH, Constants.GAME_WINDOW_HEIGHT);
+		Rectangle area = new Rectangle(Constants.GAME_WINDOW_OFFSET_X, Constants.GAME_WINDOW_OFFSET_Y, Constants.GAME_WINDOW_WIDTH, Constants.GAME_WINDOW_HEIGHT);
-	return robot.createScreenCapture(area);
+		return robot.createScreenCapture(area);
-  }
+	  }
 }

src/RandomDetector.java

@@ -1,7 +1,14 @@
 import java.awt.AWTException;
+import java.awt.Color;
+import java.awt.Point;
 import java.awt.Rectangle;
 import java.awt.Robot;
 import java.awt.image.BufferedImage;
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+
+import javax.imageio.ImageIO;
 
 public class RandomDetector {
 	Robot robot;
@@ -10,41 +17,98 @@ public class RandomDetector {
 		robot = new Robot();
 	}
 	
-	public void dealWithRandoms(BufferedImage screenCapture) throws AWTException, InterruptedException {
+	public void dealWithRandoms(BufferedImage screenCapture, Cursor cursor) throws Exception {
-		if (isRandomEventPresent(screenCapture)) {
+		Point chatDialogueCornerPoint = findChatDialogueCornerPoint(screenCapture);
-			System.out.println("Deal with random here");
+		Point speakerPoint = findSpeakerPointFromCornerPoint(screenCapture, chatDialogueCornerPoint);
+
+		if (speakerPoint != null && isSpeakerPointCloseToCharacter(speakerPoint)) {
+			cursor.moveAndRightlickAtCoordinatesWithRandomness(cursor.getOffsetPoint(speakerPoint), 3, 3);
+			Thread.sleep(Randomizer.nextGaussianWithinRange(1332, 1921));
+			
+			if (dialogueHasDismissOption(speakerPoint)) {
+				cursor.moveAndLeftClickAtCoordinatesWithRandomness(cursor.getOffsetPoint(getDismissOptionClickLocation(speakerPoint)), 40, 10, 6);
+				Thread.sleep(1743, 2313);
+			}
 		}
 	}
 
-	public boolean isRandomEventPresent(BufferedImage screenCapture) throws AWTException, InterruptedException
+	private boolean dialogueHasDismissOption(Point speakerPoint) throws IOException, AWTException {
-	{
+		Rectangle dialogueRectangle = new Rectangle(Constants.GAME_WINDOW_OFFSET_X + speakerPoint.x - 25, Constants.GAME_WINDOW_OFFSET_Y + speakerPoint.y, 95, 55);
-		// only do this if it's close to center
+		BufferedImage dialogueCapture = robot.createScreenCapture(dialogueRectangle);
-		boolean foundDialogue = false;
+		for (int x = 0; x < 95; x++) {
-		for (int x = 0; x < Constants.GAME_WINDOW_WIDTH; x += 5) {
+			for (int y = 0; y < 55; y++) {
-			for (int y = 0; y < Constants.GAME_WINDOW_HEIGHT; y += 1) {
+				int pixelColor = dialogueCapture.getRGB(x, y);
-				int color = screenCapture.getRGB(x, y);
+				if (isPixelChatColor(pixelColor)) {
-				if (pixelsAreWithinRGBTolerance(color, 10)) {
 					return true;
 				}
-				
 			}
 		}
 		return false;
 	}
-	
+
-	private boolean pixelsAreWithinRGBTolerance(int rgb1, int rgb2) {
+	public Point findChatDialogueCornerPoint(BufferedImage screenCapture) throws AWTException, InterruptedException {		
-		int[] colors1 = getRGBValuesFromPixel(rgb1);
+		for (int x = 30; x < Constants.GAME_WINDOW_WIDTH - 30; x++) {
-		int[] colors2 = getRGBValuesFromPixel(rgb2);
+			for (int y = 20; y < Constants.GAME_WINDOW_HEIGHT - 20; y++) {
-		for (int i = 0; i < 3; i++) {
+				int pixelColor = screenCapture.getRGB(x, y);
-			if (Math.abs(colors1[i] - colors2[i]) > 3) {
+				if (isPixelChatColor(pixelColor)) {
-				return false;
+					return new Point(x, y);
+				}
 			}
 		}
-		return true;
+		return null;
 	}
 	
-	private int[] getRGBValuesFromPixel(int pixel) {
+	private boolean isPixelChatColor(int color) {
+		int[] colorChannels = getRGBChannelsFromPixel(color);
+		return (isColorWithinTolerance(colorChannels[0], 0, 3) && 
+			    isColorWithinTolerance(colorChannels[1], 255, 3) &&
+			    isColorWithinTolerance(colorChannels[2], 255, 3));
+	}
+	
+	public Point findSpeakerPointFromCornerPoint(BufferedImage screenCapture, Point chatDialogueStart) {	
+		if (chatDialogueStart == null) {
+			return null;
+		}
+		
+		int rightMostChatColorPixel = chatDialogueStart.x;
+		int countSinceLastChatColorPixel = 0;
+		for (int x = chatDialogueStart.x; x < Constants.GAME_WINDOW_WIDTH && countSinceLastChatColorPixel < 30; x++) {
+			for (int y = chatDialogueStart.y; y < chatDialogueStart.y + 20; y++) {
+				int pixelColor = screenCapture.getRGB(x, y);
+				if (isPixelChatColor(pixelColor)) {
+					rightMostChatColorPixel = x;
+					countSinceLastChatColorPixel = 0;
+				}
+			}
+			countSinceLastChatColorPixel++;
+		}
+		
+		int chatDialogueBoxWidth = rightMostChatColorPixel - chatDialogueStart.x;
+		if (chatDialogueBoxWidth > 60 && chatDialogueBoxWidth < 400) {
+			return new Point(chatDialogueStart.x + chatDialogueBoxWidth / 2, chatDialogueStart.y + 25);
+		}
+		return null;
+	}
+	
+	private boolean isSpeakerPointCloseToCharacter(Point speakerPoint) {
+		return (Math.abs(speakerPoint.x + Constants.GAME_WINDOW_OFFSET_X - Constants.CHARACTER_CENTER_X) < 90 && Math.abs(speakerPoint.y + Constants.GAME_WINDOW_OFFSET_Y - Constants.CHARACTER_CENTER_Y) < 80);
+	}
+	
+	private Point getDismissOptionClickLocation(Point speakerLocation) {
+		return new Point(speakerLocation.x, speakerLocation.y + 46);
+	}
+	
+	private boolean isColorWithinTolerance(int color1, int color2, int tolerance) {
+		return (Math.abs(color1 - color2) < tolerance);
+	}
+	
+	private int[] getRGBChannelsFromPixel(int pixel) {
 		int[] colors = {(pixel)&0xFF, (pixel>>8)&0xFF, (pixel>>16)&0xFF, (pixel>>24)&0xFF};
 		return colors;
 	}	
+	
+	public BufferedImage captureScreenshotGameWindow() throws IOException, AWTException {
+		Rectangle area = new Rectangle(Constants.GAME_WINDOW_OFFSET_X, Constants.GAME_WINDOW_OFFSET_Y, Constants.GAME_WINDOW_WIDTH, Constants.GAME_WINDOW_HEIGHT);
+		return robot.createScreenCapture(area);
+	  }
 }

src/RandomDetectorTest.java

@@ -0,0 +1,46 @@
+import static org.junit.jupiter.api.Assertions.*;
+
+import java.awt.AWTException;
+import java.awt.Point;
+import java.awt.image.BufferedImage;
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+
+import javax.imageio.ImageIO;
+
+import org.junit.jupiter.api.Test;
+
+class RandomDetectorTest {
+
+	RandomDetector randomDetector;
+	
+	void initialize() throws AWTException {
+		randomDetector = new RandomDetector();
+	}
+	
+	@Test
+	void testChatDialogueFound() throws IOException, AWTException, InterruptedException {
+		initialize();
+		for (File file : getListOfFilesFromItemDirectory("/home/dpapp/Desktop/RunescapeAI/Images/")) {
+			if (file.isFile()) {
+				BufferedImage screenCapture = ImageIO.read(file);
+				Point chatDialogueCornerPoint = randomDetector.findChatDialogueCornerPoint(screenCapture);
+				Point speakerPoint = randomDetector.findSpeakerPointFromCornerPoint(screenCapture, chatDialogueCornerPoint);
+				
+				assertNotNull(speakerPoint);
+				System.out.println(file.getName());
+				if (speakerPoint != null) {
+					System.out.println("----- Random at " + speakerPoint.x + "," + speakerPoint.y + " -----");
+				}
+			}
+		}
+	}
+
+
+	public File[] getListOfFilesFromItemDirectory(String directoryPath) {
+		File itemDirectory = new File(directoryPath);
+		return itemDirectory.listFiles();
+	}
+
+}

src/WorldHopper.java

@@ -0,0 +1,62 @@
+import java.util.ArrayList;
+import java.util.LinkedList;
+import java.util.Queue;
+
+public class WorldHopper {
+
+	int numberOfOresMined;
+	int numberOfFramesWithOtherPlayers;
+	Queue<Boolean> frames;
+	Queue<Boolean> oresMined;
+	
+	public WorldHopper() {
+		numberOfOresMined = 0;
+		numberOfFramesWithOtherPlayers = 0;
+		frames = new LinkedList<Boolean>();
+		for (int i = 0; i < 600; i++) {
+			frames.add(false);
+		}
+		oresMined = new LinkedList<Boolean>();
+		for (int i = 0; i < 30; i++) {
+			oresMined.add(true);
+		}
+	}
+	
+	public void hopWorldsIfOtherPlayersPresent(ArrayList<DetectedObject> players) {
+		updateOtherPlayerTracking(players);
+		if (areOtherPlayersLikelyPresent()) {
+			hopWorld();
+		}
+	}
+	
+	private void updateOtherPlayerTracking(ArrayList<DetectedObject> players) {
+		if (players.size() > 1) {
+			numberOfFramesWithOtherPlayers++;
+			frames.add(true);
+		}
+		else {
+			frames.add(false);
+		}
+		if (frames.poll()) {
+			numberOfFramesWithOtherPlayers--;
+		}
+	}
+	
+	public void updateOresMinedSuccessTracking(boolean success) {
+		oresMined.add(success);
+		if (success) { 
+			numberOfOresMined++;
+		}
+		if (oresMined.poll()) {
+			numberOfOresMined--;
+		}
+	}
+	
+	private boolean areOtherPlayersLikelyPresent() {
+		return numberOfFramesWithOtherPlayers > 360 || numberOfOresMined < 18;
+	}
+	
+	private void hopWorld() {
+		System.out.println("Hopping worlds!");
+	}
+}