diff --git a/src/DetectedObject.java b/src/DetectedObject.java index a3eb6b3..f1b97aa 100644 --- a/src/DetectedObject.java +++ b/src/DetectedObject.java @@ -1,6 +1,8 @@ import java.awt.Point; import java.awt.Rectangle; +import org.opencv.core.Rect2d; + public class DetectedObject { @@ -35,10 +37,14 @@ public class DetectedObject { return detectionClass; } - public Rectangle getBoundingBox() { + public Rectangle getBoundingRectangle() { return boundingBox; } + private Rect2d getBoundingRect2d() { + return new Rect2d(boundingBox.x, boundingBox.y, boundingBox.x + boundingBox.width, boundingBox.y + boundingBox.height); + } + public Point getCenterForClicking() { return new Point(boundingBox.x + boundingBox.width / 2 + Constants.GAME_WINDOW_OFFSET_X, boundingBox.y + boundingBox.height / 2 + Constants.GAME_WINDOW_OFFSET_Y); } diff --git a/src/ImageCollector.java b/src/ImageCollector.java index 9339eec..a183d55 100644 --- a/src/ImageCollector.java +++ b/src/ImageCollector.java @@ -78,7 +78,6 @@ public class ImageCollector { return screenshotOutputDirectory + itemName + "_" + counter + ".jpg"; } - private void generateInventoryImages() throws AWTException, IOException { Inventory inventory = new Inventory(); inventory.updateAndWriteAllInventoryImages(); diff --git a/src/IronMiner.java b/src/IronMiner.java index 13eaf54..433b838 100644 --- a/src/IronMiner.java +++ b/src/IronMiner.java @@ -10,10 +10,12 @@ import java.util.ArrayList; import javax.imageio.ImageIO; import org.opencv.core.Rect2d; +import org.opencv.tracking.Tracker; +import org.opencv.tracking.TrackerKCF; public class IronMiner { - public static final int IRON_ORE_MINING_TIME_MILLISECONDS = 2738; + public static final int IRON_ORE_MINING_TIME_MILLISECONDS = 1320; public static final int MAXIMUM_DISTANCE_TO_WALK_TO_IRON_ORE = 400; public static final Point GAME_WINDOW_CENTER = new Point(Constants.GAME_WINDOW_WIDTH / 2, Constants.GAME_WINDOW_HEIGHT / 2); @@ -37,15 +39,31 @@ public class IronMiner { public void run() throws Exception { while (true) { - objectDetector.update(); - ArrayList ironOres = objectDetector.getRecognizedObjectsOfClassFromImage("ironOre"); - ArrayList ores = objectDetector.getRecognizedObjectsOfClassFromImage("ore"); - System.out.println(ironOres.size() + " ironOres, " + ores.size() + " ores."); + BufferedImage screenCapture = objectDetector.captureScreenshotGameWindow(); + ArrayList detectedObjects = objectDetector.getObjectsInImage(screenCapture); + ArrayList ironOres = objectDetector.getObjectsOfClassInList(detectedObjects, "ironOre"); + + DetectedObject closestIronOre = getClosestObjectToCharacter(ironOres); + if (closestIronOre != null) { + Tracker objectTracker = TrackerKCF.create(); + objectTracker.init(screenCapture, closestIronOre.getBoundingRect2d()); + cursor.moveAndLeftClickAtCoordinatesWithRandomness(closestIronOre.getCenterForClicking(), 10, 10); + + long mineStartTime = System.currentTimeMillis(); + int maxTimeToMine = randomizer.nextGaussianWithinRange(3500, 5000); + + + while ((System.currentTimeMillis() - mineStartTime) < maxTimeToMine) { + screenCapture = objectDetector.captureScreenshotGameWindow(); + objectTracker.update(screenCapture, boundingBox); + Rectangle newBounds = new Rectangle(); + if (!objectDetector.isObjectPresentInBoundingBoxInImage(screenCapture, newBounds, "ironOre")) { + System.out.println("Lost track! Finding new ore."); + break; + } + } + } - /*for (DetectedObject ironOre : ironOres) { - ironOre.display(); - }*/ - mineClosestIronOre(ironOres, ores); dropInventoryIfFull(); } } @@ -58,36 +76,13 @@ public class IronMiner { } - private void mineClosestIronOre(ArrayList ironOres, ArrayList ores) throws Exception { + private void mineClosestIronOre(ArrayList ironOres) throws Exception { DetectedObject closestIronOre = getClosestObjectToCharacter(ironOres); - if (closestIronOre != null) { + if (closestIronOre != null) { cursor.moveAndLeftClickAtCoordinatesWithRandomness(closestIronOre.getCenterForClicking(), 10, 10); - Thread.sleep(84, 219); - DetectedObject closestOre = getClosestObjectToCharacter(ores); - if (closestOre != null) { - cursor.moveCursorToCoordinatesWithRandomness(closestOre.getCenterForClicking(), 10, 10); - } - Thread.sleep(randomizer.nextGaussianWithinRange(IRON_ORE_MINING_TIME_MILLISECONDS - 250, IRON_ORE_MINING_TIME_MILLISECONDS + -50)); + Thread.sleep(randomizer.nextGaussianWithinRange(IRON_ORE_MINING_TIME_MILLISECONDS - 350, IRON_ORE_MINING_TIME_MILLISECONDS + 1850)); } - //Thread.sleep(randomizer.nextGaussianWithinRange(150, 350)); - - //cursor.moveCursorToCoordinates(goalPoint); } - /*private void mineClosestIronOre(String filename) throws Exception { - Point ironOreLocation = getClosestIronOre(filename); - if (ironOreLocation != null) { - System.out.println("Mineable iron at (" + (ironOreLocation.x + 103) + "," + (ironOreLocation.y + 85) + ")"); - Point actualIronOreLocation = new Point(ironOreLocation.x + 103, ironOreLocation.y + 85); - Rect2d trackerBoundingBox = new Rec2d(); - //Rectangle trackerBoundingBox = new Rectangle(ironOreLocation.x - 10, ironOreLocation.x + 10, ironOreLocation.y - 10, ironOreLocation.y + 10); - //tracker.init(image, trackerBoundingBox); - cursor.moveAndLeftClickAtCoordinatesWithRandomness(actualIronOreLocation, 12, 12); - Thread.sleep(randomizer.nextGaussianWithinRange(IRON_ORE_MINING_TIME_MILLISECONDS - 350, IRON_ORE_MINING_TIME_MILLISECONDS + 150)); - } - - } - - }*/ private DetectedObject getClosestObjectToCharacter(ArrayList detectedObjects) { int closestDistanceToCharacter = Integer.MAX_VALUE; diff --git a/src/ObjectDetector.java b/src/ObjectDetector.java index b2ab4bc..d40e0d0 100644 --- a/src/ObjectDetector.java +++ b/src/ObjectDetector.java @@ -31,6 +31,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import javax.imageio.ImageIO; + import org.tensorflow.SavedModelBundle; import org.tensorflow.Tensor; import org.tensorflow.types.UInt8; @@ -42,28 +43,26 @@ import org.tensorflow.types.UInt8; public class ObjectDetector { SavedModelBundle model; - ArrayList detectedObjects; Robot robot; public ObjectDetector() throws AWTException { this.model = SavedModelBundle.load("/home/dpapp/tensorflow-1.5.0/models/raccoon_dataset/results/checkpoint_22948/saved_model/", "serve"); - this.detectedObjects = new ArrayList(); this.robot = new Robot(); } public void update() throws Exception { // TODO: eliminate IO and pass BufferedImage directly. - String fileName = "/home/dpapp/Desktop/RunescapeAI/temp/screenshot.jpg"; + /*String fileName = "/home/dpapp/Desktop/RunescapeAI/temp/screenshot.jpg"; BufferedImage image = captureScreenshotGameWindow(); ImageIO.write(image, "jpg", new File(fileName)); - this.detectedObjects = getRecognizedObjectsFromImage(fileName); + this.detectedObjects = getRecognizedObjectsFromImage(fileName);*/ } - private ArrayList getRecognizedObjectsFromImage(String fileName) throws Exception { + public ArrayList getObjectsInImage(BufferedImage image) throws Exception { List> outputs = null; ArrayList detectedObjectsInImage = new ArrayList(); - try (Tensor input = makeImageTensor(fileName)) { + try (Tensor input = makeImageTensor(image)) { outputs = model .session() @@ -92,10 +91,16 @@ public class ObjectDetector { } return detectedObjectsInImage; } + + public boolean isObjectPresentInBoundingBoxInImage(BufferedImage image, Rectangle boundingBox, String objectClass) throws Exception { + BufferedImage subImage = image.getSubimage(boundingBox.x, boundingBox.y, boundingBox.width, boundingBox.height); + ArrayList detectedObjectsInSubImage = getObjectsInImage(subImage); + return (getObjectsOfClassInList(detectedObjectsInSubImage, objectClass).size() != 0); + } - public ArrayList getRecognizedObjectsOfClassFromImage(String objectClass) { + public ArrayList getObjectsOfClassInList(ArrayList detectedObjects, String objectClass) { ArrayList detectedObjectsOfType = new ArrayList(); - for (DetectedObject detectedObject : this.detectedObjects) { + for (DetectedObject detectedObject : detectedObjects) { if (detectedObject.getDetectionClass().equals(objectClass)) { detectedObjectsOfType.add(detectedObject); } @@ -103,8 +108,8 @@ public class ObjectDetector { return detectedObjectsOfType; } - private static Tensor makeImageTensor(String filename) throws IOException { - BufferedImage img = ImageIO.read(new File(filename)); + private static Tensor makeImageTensor(BufferedImage image) throws IOException { + BufferedImage img = ImageIO.read(image); if (img.getType() != BufferedImage.TYPE_3BYTE_BGR) { throw new IOException( String.format( @@ -128,8 +133,8 @@ public class ObjectDetector { } } - private BufferedImage captureScreenshotGameWindow() throws IOException { - Rectangle area = new Rectangle(103, 85, 510, 330); - return robot.createScreenCapture(area); - } + public BufferedImage captureScreenshotGameWindow() throws IOException, AWTException { + Rectangle area = new Rectangle(Constants.GAME_WINDOW_OFFSET_X, Constants.GAME_WINDOW_OFFSET_Y, Constants.GAME_WINDOW_WIDTH, Constants.GAME_WINDOW_HEIGHT); + return robot.createScreenCapture(area); + } } \ No newline at end of file diff --git a/src/ObjectTracker.java b/src/ObjectTracker.java new file mode 100644 index 0000000..f70e79a --- /dev/null +++ b/src/ObjectTracker.java @@ -0,0 +1,17 @@ +import java.awt.Rectangle; + +import org.opencv.tracking.Tracker; +import org.opencv.tracking.TrackerKCF; + +public class ObjectTracker { + + Rectangle boundingBox; + public ObjectTracker() { + /*this.boundingBox = boundingBox; + Tracker tracker = TrackerKCF.create(); + tracker.init(image, boundingBox);*/ + } + + + +} diff --git a/target/classes/DetectedObject.class b/target/classes/DetectedObject.class index 3fffeed..0f50857 100644 Binary files a/target/classes/DetectedObject.class and b/target/classes/DetectedObject.class differ diff --git a/target/classes/ImageCollector.class b/target/classes/ImageCollector.class index e4a1e08..b37335a 100644 Binary files a/target/classes/ImageCollector.class and b/target/classes/ImageCollector.class differ diff --git a/target/classes/IronMiner.class b/target/classes/IronMiner.class index 3ddfe46..b5189dd 100644 Binary files a/target/classes/IronMiner.class and b/target/classes/IronMiner.class differ diff --git a/target/classes/ObjectDetector.class b/target/classes/ObjectDetector.class index 9ec4b0e..4075ebb 100644 Binary files a/target/classes/ObjectDetector.class and b/target/classes/ObjectDetector.class differ diff --git a/target/classes/ObjectTracker.class b/target/classes/ObjectTracker.class new file mode 100644 index 0000000..7f35f08 Binary files /dev/null and b/target/classes/ObjectTracker.class differ