diff --git a/src/Cursor.java b/src/Cursor.java index 3a3fd17..f9f9b44 100644 --- a/src/Cursor.java +++ b/src/Cursor.java @@ -125,6 +125,12 @@ public class Cursor { return randomizedGoalPoint; // Return the point we moved to in case we need precise movement afterwards } + public Point moveCursorToCoordinatesWithRandomness(Point goalPoint, int xTolerance, int yTolerance) throws Exception { + Point randomizedGoalPoint = randomizePoint(goalPoint, xTolerance, yTolerance); + moveCursorToCoordinates(randomizedGoalPoint); + return randomizedGoalPoint; // Return the point we moved to in case we need precise movement afterwards + } + public void moveCursorToCoordinates(Point goalPoint) throws Exception { Point startingPoint = getCurrentCursorPoint(); int distanceToMoveCursor = getDistanceBetweenPoints(startingPoint, goalPoint); diff --git a/src/DetectedObject.java b/src/DetectedObject.java new file mode 100644 index 0000000..a3eb6b3 --- /dev/null +++ b/src/DetectedObject.java @@ -0,0 +1,49 @@ +import java.awt.Point; +import java.awt.Rectangle; + + + +public class DetectedObject { + + private Rectangle boundingBox; + private float detectionScore; + private String detectionClass; + + public DetectedObject(float detectionScore, float detectionClass, float[] detectionBox) { + this.boundingBox = initializeBoundingBox(detectionBox); + this.detectionScore = detectionScore; + this.detectionClass = initializeLabel(detectionClass); + } + + private Rectangle initializeBoundingBox(float[] detectionBox) { + int offset_x = (int) (detectionBox[1] * Constants.GAME_WINDOW_WIDTH); + int offset_y = (int) (detectionBox[0] * Constants.GAME_WINDOW_HEIGHT); + int width = (int) (detectionBox[3] * Constants.GAME_WINDOW_WIDTH) - offset_x; + int height = (int) (detectionBox[2] * Constants.GAME_WINDOW_HEIGHT) - offset_y; + + //System.out.println(detectionBox[0] + ", " + detectionBox[1] + ", " + detectionBox[2] + ", " + detectionBox[3]); + return new Rectangle(offset_x, offset_y, width, height); + } + + private String initializeLabel(float detectionClass) { + // TODO: actually load these from a file + String[] labels = {"NA", "ironOre", "ore"}; + return labels[(int) detectionClass]; + } + + public String getDetectionClass() { + return detectionClass; + } + + public Rectangle getBoundingBox() { + return boundingBox; + } + + public Point getCenterForClicking() { + return new Point(boundingBox.x + boundingBox.width / 2 + Constants.GAME_WINDOW_OFFSET_X, boundingBox.y + boundingBox.height / 2 + Constants.GAME_WINDOW_OFFSET_Y); + } + + public void display() { + System.out.println(detectionClass + " with score " + detectionScore + " at (" + getCenterForClicking().x + "," + getCenterForClicking().y + ")"); + } +} diff --git a/src/ImageCollector.java b/src/ImageCollector.java index 037b648..9339eec 100644 --- a/src/ImageCollector.java +++ b/src/ImageCollector.java @@ -12,6 +12,7 @@ public class ImageCollector { public String screenshotOutputDirectory; public Rectangle gameWindowRectangle; + public Robot robot; /* * Methods needed: @@ -23,13 +24,14 @@ public class ImageCollector { * detect last file name */ - public ImageCollector(String screenshotOutputDirectory) { + public ImageCollector(String screenshotOutputDirectory) throws AWTException { initializeGameWindowRectangle(); this.screenshotOutputDirectory = screenshotOutputDirectory; + this.robot = new Robot(); } private void initializeGameWindowRectangle() { - this.gameWindowRectangle = new Rectangle(103, 85, 510, 330); + this.gameWindowRectangle = new Rectangle(Constants.GAME_WINDOW_OFFSET_X, Constants.GAME_WINDOW_OFFSET_Y, Constants.GAME_WINDOW_WIDTH, Constants.GAME_WINDOW_HEIGHT); } public void collectImages(String itemName) throws IOException, InterruptedException, AWTException { @@ -66,7 +68,6 @@ public class ImageCollector { } private void captureAndSaveGameWindow(String itemName, int fileCounter) throws IOException, InterruptedException, AWTException { - Robot robot = new Robot(); BufferedImage imageCaptured = robot.createScreenCapture(gameWindowRectangle); String fileName = getFileName(itemName, fileCounter); ImageIO.write(imageCaptured, "jpg", new File(fileName)); @@ -85,8 +86,8 @@ public class ImageCollector { public static void main(String[] args) throws Exception { - ImageCollector imageCollector = new ImageCollector("/home/dpapp/Desktop/RunescapeAI/TensorFlow/IronOre/"); - //imageCollector.collectImages("ironOre"); - imageCollector.generateInventoryImages(); + ImageCollector imageCollector = new ImageCollector("/home/dpapp/Desktop/RunescapeAI/TensorFlow/Ores/Images/"); + imageCollector.collectImages("ore"); + //imageCollector.generateInventoryImages(); } } diff --git a/src/IronMiner.java b/src/IronMiner.java index 3054cef..13eaf54 100644 --- a/src/IronMiner.java +++ b/src/IronMiner.java @@ -9,11 +9,13 @@ import java.util.ArrayList; import javax.imageio.ImageIO; +import org.opencv.core.Rect2d; + public class IronMiner { public static final int IRON_ORE_MINING_TIME_MILLISECONDS = 2738; public static final int MAXIMUM_DISTANCE_TO_WALK_TO_IRON_ORE = 400; - public static final Point GAME_WINDOW_CENTER = new Point(510 / 2, 330 / 2); + public static final Point GAME_WINDOW_CENTER = new Point(Constants.GAME_WINDOW_WIDTH / 2, Constants.GAME_WINDOW_HEIGHT / 2); Cursor cursor; CursorTask cursorTask; @@ -35,13 +37,15 @@ public class IronMiner { public void run() throws Exception { while (true) { - //Thread.sleep(250); - - String filename = "/home/dpapp/Desktop/RunescapeAI/temp/screenshot.jpg"; - BufferedImage image = captureScreenshotGameWindow(); - ImageIO.write(image, "jpg", new File(filename)); - mineClosestIronOre(filename); + objectDetector.update(); + ArrayList ironOres = objectDetector.getRecognizedObjectsOfClassFromImage("ironOre"); + ArrayList ores = objectDetector.getRecognizedObjectsOfClassFromImage("ore"); + System.out.println(ironOres.size() + " ironOres, " + ores.size() + " ores."); + /*for (DetectedObject ironOre : ironOres) { + ironOre.display(); + }*/ + mineClosestIronOre(ironOres, ores); dropInventoryIfFull(); } } @@ -53,47 +57,56 @@ public class IronMiner { } } - private void mineClosestIronOre(String filename) throws Exception { + + private void mineClosestIronOre(ArrayList ironOres, ArrayList ores) throws Exception { + DetectedObject closestIronOre = getClosestObjectToCharacter(ironOres); + if (closestIronOre != null) { + cursor.moveAndLeftClickAtCoordinatesWithRandomness(closestIronOre.getCenterForClicking(), 10, 10); + Thread.sleep(84, 219); + DetectedObject closestOre = getClosestObjectToCharacter(ores); + if (closestOre != null) { + cursor.moveCursorToCoordinatesWithRandomness(closestOre.getCenterForClicking(), 10, 10); + } + Thread.sleep(randomizer.nextGaussianWithinRange(IRON_ORE_MINING_TIME_MILLISECONDS - 250, IRON_ORE_MINING_TIME_MILLISECONDS + -50)); + } + //Thread.sleep(randomizer.nextGaussianWithinRange(150, 350)); + + //cursor.moveCursorToCoordinates(goalPoint); + } + /*private void mineClosestIronOre(String filename) throws Exception { Point ironOreLocation = getClosestIronOre(filename); - /*if (ironOreLocation == null) { - Thread.sleep(1000); - }*/ if (ironOreLocation != null) { System.out.println("Mineable iron at (" + (ironOreLocation.x + 103) + "," + (ironOreLocation.y + 85) + ")"); Point actualIronOreLocation = new Point(ironOreLocation.x + 103, ironOreLocation.y + 85); + Rect2d trackerBoundingBox = new Rec2d(); + //Rectangle trackerBoundingBox = new Rectangle(ironOreLocation.x - 10, ironOreLocation.x + 10, ironOreLocation.y - 10, ironOreLocation.y + 10); + //tracker.init(image, trackerBoundingBox); cursor.moveAndLeftClickAtCoordinatesWithRandomness(actualIronOreLocation, 12, 12); Thread.sleep(randomizer.nextGaussianWithinRange(IRON_ORE_MINING_TIME_MILLISECONDS - 350, IRON_ORE_MINING_TIME_MILLISECONDS + 150)); } } - private Point getClosestIronOre(String filename) throws IOException { - ArrayList ironOreLocations = objectDetector.getIronOreLocationsFromImage(filename); - System.out.println(ironOreLocations.size()); - int closestDistanceToIronOreFromCharacter = Integer.MAX_VALUE; - Point closestIronOreToCharacter = null; - for (Point ironOreLocation : ironOreLocations) { - int distanceToIronOreFromCharacter = getDistanceBetweenPoints(GAME_WINDOW_CENTER, ironOreLocation); - if (distanceToIronOreFromCharacter < closestDistanceToIronOreFromCharacter) { - closestDistanceToIronOreFromCharacter = distanceToIronOreFromCharacter; - closestIronOreToCharacter = new Point(ironOreLocation.x, ironOreLocation.y); + }*/ + + private DetectedObject getClosestObjectToCharacter(ArrayList detectedObjects) { + int closestDistanceToCharacter = Integer.MAX_VALUE; + DetectedObject closestObjectToCharacter = null; + + for (DetectedObject detectedObject : detectedObjects) { + int objectDistanceToCharacter = getDistanceBetweenPoints(GAME_WINDOW_CENTER, detectedObject.getCenterForClicking()); + if (objectDistanceToCharacter < closestDistanceToCharacter) { + closestDistanceToCharacter = objectDistanceToCharacter; + closestObjectToCharacter = detectedObject; } } - - if (closestIronOreToCharacter != null && closestDistanceToIronOreFromCharacter < MAXIMUM_DISTANCE_TO_WALK_TO_IRON_ORE) { - return closestIronOreToCharacter; + if (closestObjectToCharacter != null && closestDistanceToCharacter < MAXIMUM_DISTANCE_TO_WALK_TO_IRON_ORE) { + return closestObjectToCharacter; } return null; } - public int getDistanceBetweenPoints(Point startingPoint, Point goalPoint) { return (int) (Math.hypot(goalPoint.x - startingPoint.x, goalPoint.y - startingPoint.y)); } - - private BufferedImage captureScreenshotGameWindow() throws IOException { - Rectangle area = new Rectangle(103, 85, 510, 330); - return robot.createScreenCapture(area); - - } } diff --git a/src/ObjectDetector.java b/src/ObjectDetector.java index 95d0240..b2ab4bc 100644 --- a/src/ObjectDetector.java +++ b/src/ObjectDetector.java @@ -13,7 +13,10 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ +import java.awt.AWTException; import java.awt.Point; +import java.awt.Rectangle; +import java.awt.Robot; import java.awt.image.BufferedImage; import java.awt.image.DataBufferByte; import java.io.File; @@ -24,6 +27,7 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Paths; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; import javax.imageio.ImageIO; @@ -38,16 +42,28 @@ import org.tensorflow.types.UInt8; public class ObjectDetector { SavedModelBundle model; + ArrayList detectedObjects; + Robot robot; - public ObjectDetector() { - model = SavedModelBundle.load("/home/dpapp/tensorflow-1.5.0/models/raccoon_dataset/results/checkpoint_23826/saved_model/", "serve"); + public ObjectDetector() throws AWTException { + this.model = SavedModelBundle.load("/home/dpapp/tensorflow-1.5.0/models/raccoon_dataset/results/checkpoint_22948/saved_model/", "serve"); + this.detectedObjects = new ArrayList(); + this.robot = new Robot(); } - public ArrayList getIronOreLocationsFromImage(String filename) throws IOException { + public void update() throws Exception { + // TODO: eliminate IO and pass BufferedImage directly. + String fileName = "/home/dpapp/Desktop/RunescapeAI/temp/screenshot.jpg"; + BufferedImage image = captureScreenshotGameWindow(); + ImageIO.write(image, "jpg", new File(fileName)); + this.detectedObjects = getRecognizedObjectsFromImage(fileName); + } + + private ArrayList getRecognizedObjectsFromImage(String fileName) throws Exception { List> outputs = null; - ArrayList ironOreLocations = new ArrayList(); - - try (Tensor input = makeImageTensor(filename)) { + ArrayList detectedObjectsInImage = new ArrayList(); + + try (Tensor input = makeImageTensor(fileName)) { outputs = model .session() @@ -62,47 +78,30 @@ public class ObjectDetector { try (Tensor scoresT = outputs.get(0).expect(Float.class); Tensor classesT = outputs.get(1).expect(Float.class); Tensor boxesT = outputs.get(2).expect(Float.class)) { - // All these tensors have: - // - 1 as the first dimension - // - maxObjects as the second dimension - // While boxesT will have 4 as the third dimension (2 sets of (x, y) coordinates). - // This can be verified by looking at scoresT.shape() etc. + int maxObjects = (int) scoresT.shape()[1]; float[] scores = scoresT.copyTo(new float[1][maxObjects])[0]; float[] classes = classesT.copyTo(new float[1][maxObjects])[0]; float[][] boxes = boxesT.copyTo(new float[1][maxObjects][4])[0]; - // Print all objects whose score is at least 0.5. - boolean foundSomething = false; + for (int i = 0; i < scores.length; ++i) { - if (scores[i] < 0.75) { - continue; + if (scores[i] > 0.80) { + detectedObjectsInImage.add(new DetectedObject(scores[i], classes[i], boxes[i])); } - foundSomething = true; - //System.out.printf("\tFound %-20s (score: %.4f)\n", "ironOre", scores[i]); - //System.out.println("X:" + 510 * boxes[i][1] + ", Y:" + 330 * boxes[i][0] + ", width:" + 510 * boxes[i][3] + ", height:" + 330 * boxes[i][2]); - ironOreLocations.add(getCenterPointFromBox(boxes[i])); - } - if (!foundSomething) { - System.out.println("No objects detected with a high enough score."); } } - return ironOreLocations; + return detectedObjectsInImage; } - private Point getCenterPointFromBox(float[] box) { - int x = (int) (510 * (box[1] + box[3]) / 2); - int y = (int) (330 * (box[0] + box[2]) / 2); - return new Point(x, y); + public ArrayList getRecognizedObjectsOfClassFromImage(String objectClass) { + ArrayList detectedObjectsOfType = new ArrayList(); + for (DetectedObject detectedObject : this.detectedObjects) { + if (detectedObject.getDetectionClass().equals(objectClass)) { + detectedObjectsOfType.add(detectedObject); + } + } + return detectedObjectsOfType; } - - - private static void bgr2rgb(byte[] data) { - for (int i = 0; i < data.length; i += 3) { - byte tmp = data[i]; - data[i] = data[i + 2]; - data[i + 2] = tmp; - } - } private static Tensor makeImageTensor(String filename) throws IOException { BufferedImage img = ImageIO.read(new File(filename)); @@ -111,8 +110,6 @@ public class ObjectDetector { String.format( "Expected 3-byte BGR encoding in BufferedImage, found %d (file: %s). This code could be made more robust")); } - //System.out.println("Image is of type RGB? " + (img.getType() == BufferedImage.TYPE_INT_RGB)); - //System.out.println("Image is of type RGB? " + (img.getType() == BufferedImage.TYPE_3BYTE_BGR)); byte[] data = ((DataBufferByte) img.getData().getDataBuffer()).getData(); // ImageIO.read seems to produce BGR-encoded images, but the model expects RGB. @@ -122,4 +119,17 @@ public class ObjectDetector { long[] shape = new long[] {BATCH_SIZE, img.getHeight(), img.getWidth(), CHANNELS}; return Tensor.create(UInt8.class, shape, ByteBuffer.wrap(data)); } -} + + private static void bgr2rgb(byte[] data) { + for (int i = 0; i < data.length; i += 3) { + byte tmp = data[i]; + data[i] = data[i + 2]; + data[i + 2] = tmp; + } + } + + private BufferedImage captureScreenshotGameWindow() throws IOException { + Rectangle area = new Rectangle(103, 85, 510, 330); + return robot.createScreenCapture(area); + } +} \ No newline at end of file diff --git a/src/Paths.java b/src/Paths.java index 9eb84fe..9c864d7 100644 --- a/src/Paths.java +++ b/src/Paths.java @@ -7,4 +7,6 @@ public class Paths { public static final String CURSOR_COORDINATES_FILE_PATH = "/home/dpapp/GhostMouse/coordinates.txt"; public static final String TENSORFLOW_MODEL_DIRECTORY = "/home/dpapp/tensorflow-1.5.0/models/raccoon_dataset/results/checkpoint_23826/saved_model/"; + public static final String TENSORFLOW_TRAINING_IMAGE_OUTPUT_DIRECTORY = "/home/dpapp/tensorflow-1.5.0/models/raccoon_dataset/results/checkpoint_23826/saved_model/"; + public static final String TENSORFLOW_MODEL_LABELS_FILE_PATH = "/home/dpapp/tensorflow-1.5.0/models/raccoon_dataset/training/labels.pbtxt"; } diff --git a/src/cascadeTrainingImageCollector.java b/src/cascadeTrainingImageCollector.java index 11507cb..24f7bad 100644 --- a/src/cascadeTrainingImageCollector.java +++ b/src/cascadeTrainingImageCollector.java @@ -33,9 +33,8 @@ public class cascadeTrainingImageCollector { public void captureWindowEveryNMilliseconds(int n) throws InterruptedException, IOException { for (int i = 0; i < 1000; i++) { captureScreenshotGameWindow(i); - System.out.println(i); - //System.out.println("Created image: " + getImageName(i)); - Thread.sleep(n * 1000); + System.out.println("Created image: " + getImageName(i)); + Thread.sleep(n); } } @@ -63,7 +62,7 @@ public class cascadeTrainingImageCollector { return imageOutputDirectory + "screenshot" + counter + ".jpg"; } - private void resizeImagesInDirectory() throws IOException { + /*private void resizeImagesInDirectory() throws IOException { File folder = new File("/home/dpapp/Desktop/RunescapeAIPics/CascadeTraining/Testing/"); File[] listOfFiles = folder.listFiles(); @@ -73,21 +72,21 @@ public class cascadeTrainingImageCollector { System.out.println("Cropped " + listOfFiles[i].getName()); } } - } + }*/ - private void resizeImage(File imageFile, int counter) throws IOException { + /*private void resizeImage(File imageFile, int counter) throws IOException { BufferedImage screenshot = ImageIO.read(imageFile); //Rectangle resizeRectangle = new Rectangle(103, 85, 510, 330); BufferedImage resizedImage = screenshot.getSubimage(103, 85, 510, 330); ImageIO.write(resizedImage, "jpg", new File(getImageName(counter))); - } + }*/ public static void main(String[] args) throws Exception { System.out.println("Starting image collection..."); - cascadeTrainingImageCollector imageCollector = new cascadeTrainingImageCollector("/home/dpapp/Desktop/RunescapeAIPics/CascadeTraining/CoalNegative/"); + cascadeTrainingImageCollector imageCollector = new cascadeTrainingImageCollector(Paths.TENSORFLOW_TRAINING_IMAGE_OUTPUT_DIRECTORY); //imageCollector.resizeImagesInDirectory(); - imageCollector.captureWindowEveryNMilliseconds(5);; + imageCollector.captureWindowEveryNMilliseconds(2000); //cascadeTrainingImageCollector imageCollector = new cascadeTrainingImageCollector("/home/dpapp/Desktop/RunescapeAIPics/CascadeTraining/Testing/"); //imageCollector.captureWindowEveryNMilliseconds(1); } diff --git a/target/classes/Cursor.class b/target/classes/Cursor.class index 5323db5..b7c03b7 100644 Binary files a/target/classes/Cursor.class and b/target/classes/Cursor.class differ diff --git a/target/classes/DetectedObject.class b/target/classes/DetectedObject.class new file mode 100644 index 0000000..3fffeed Binary files /dev/null and b/target/classes/DetectedObject.class differ diff --git a/target/classes/ImageCollector$1.class b/target/classes/ImageCollector$1.class index e3fbe9f..3217e39 100644 Binary files a/target/classes/ImageCollector$1.class and b/target/classes/ImageCollector$1.class differ diff --git a/target/classes/ImageCollector.class b/target/classes/ImageCollector.class index d8851a8..e4a1e08 100644 Binary files a/target/classes/ImageCollector.class and b/target/classes/ImageCollector.class differ diff --git a/target/classes/IronMiner.class b/target/classes/IronMiner.class index 9a6918f..3ddfe46 100644 Binary files a/target/classes/IronMiner.class and b/target/classes/IronMiner.class differ diff --git a/target/classes/ObjectDetector.class b/target/classes/ObjectDetector.class index 09d73a9..9ec4b0e 100644 Binary files a/target/classes/ObjectDetector.class and b/target/classes/ObjectDetector.class differ diff --git a/target/classes/Paths.class b/target/classes/Paths.class index e9412ec..be1bab7 100644 Binary files a/target/classes/Paths.class and b/target/classes/Paths.class differ diff --git a/target/classes/cascadeTrainingImageCollector.class b/target/classes/cascadeTrainingImageCollector.class index 1e937c4..30e6ea6 100644 Binary files a/target/classes/cascadeTrainingImageCollector.class and b/target/classes/cascadeTrainingImageCollector.class differ