Object detection is now fully functional.

src/ColorAnalyzer.java

@@ -124,7 +124,7 @@ public class ColorAnalyzer {
     {
         ColorAnalyzer colorAnalyzer = new ColorAnalyzer();
         //colorAnalyzer.showColorDistribution("screenshot21.jpg");
-        //colorAnalyzer.printCursorColor();
+        colorAnalyzer.printCursorColor();
-        colorAnalyzer.colorImage();
+        //colorAnalyzer.colorImage();
     }
 }

src/IronMiner.java

@@ -1,75 +1,99 @@
 import java.awt.AWTException;
 import java.awt.Point;
+import java.awt.Rectangle;
+import java.awt.Robot;
+import java.awt.image.BufferedImage;
+import java.io.File;
 import java.io.IOException;
 import java.util.ArrayList;
 
+import javax.imageio.ImageIO;
+
 public class IronMiner {
 	
-	public static final int IRON_ORE_MINING_TIME_MILLISECONDS = 650;
+	public static final int IRON_ORE_MINING_TIME_MILLISECONDS = 2738;
-	public static final int MAXIMUM_DISTANCE_TO_WALK_TO_IRON_ORE = 650;
+	public static final int MAXIMUM_DISTANCE_TO_WALK_TO_IRON_ORE = 400;
-	public static final Point GAME_WINDOW_CENTER = new Point(200, 300);
+	public static final Point GAME_WINDOW_CENTER = new Point(510 / 2, 330 / 2);
 	
 	Cursor cursor; 
 	CursorTask cursorTask;
 	Inventory inventory;
+	ObjectDetector objectDetector;
+	Robot robot;
+	Randomizer randomizer;
 	
 	public IronMiner() throws AWTException, IOException 
 	{
 		cursor = new Cursor();
 		cursorTask = new CursorTask();
 		inventory = new Inventory();
+		objectDetector = new ObjectDetector();
+		robot = new Robot();
+		randomizer = new Randomizer();
 	}
 	
 	public void run() throws Exception {
 		
 		while (true) {
-			Thread.sleep(250);
+			//Thread.sleep(250);
 			
-			mineClosestIronOre();
+			String filename = "/home/dpapp/Desktop/RunescapeAI/temp/screenshot.jpg";
+			BufferedImage image = captureScreenshotGameWindow();
+			ImageIO.write(image, "jpg", new File(filename));
+			mineClosestIronOre(filename);
 			
+			dropInventoryIfFull();
+		}
+	}
+	
+	private void dropInventoryIfFull() throws Exception {
 		inventory.update(); // TODO: add iron ore to inventory items
 		if (inventory.isInventoryFull()) {
-				System.out.println("Inventory is full! Dropping...");
 			cursorTask.optimizedDropAllItemsInInventory(cursor, inventory);
 		}
 	}
-	}
 	
-	private void mineClosestIronOre() throws Exception {
+	private void mineClosestIronOre(String filename) throws Exception {
-		Point ironOreLocation = getClosestIronOre();
+		Point ironOreLocation = getClosestIronOre(filename);
-		if (ironOreLocation == null) {
+		/*if (ironOreLocation == null) {
 			Thread.sleep(1000);
-		}
+		}*/
-		cursor.moveAndLeftClickAtCoordinatesWithRandomness(ironOreLocation, 20, 20);
+		if (ironOreLocation != null) {
-		Thread.sleep(IRON_ORE_MINING_TIME_MILLISECONDS);
+			System.out.println("Mineable iron at (" + (ironOreLocation.x + 103) + "," + (ironOreLocation.y + 85) + ")");
+			Point actualIronOreLocation = new Point(ironOreLocation.x + 103, ironOreLocation.y + 85);
+			cursor.moveAndLeftClickAtCoordinatesWithRandomness(actualIronOreLocation, 12, 12);
+			Thread.sleep(randomizer.nextGaussianWithinRange(IRON_ORE_MINING_TIME_MILLISECONDS - 350, IRON_ORE_MINING_TIME_MILLISECONDS + 150));
 		}
 		
-	private Point getClosestIronOre() {
+	}
-		ArrayList<Point> ironOreLocations = getIronOreLocations();
+	
+	private Point getClosestIronOre(String filename) throws IOException {
+		ArrayList<Point> ironOreLocations = objectDetector.getIronOreLocationsFromImage(filename);
+		System.out.println(ironOreLocations.size());
 		int closestDistanceToIronOreFromCharacter = Integer.MAX_VALUE;
 		Point closestIronOreToCharacter = null;
 		for (Point ironOreLocation : ironOreLocations) {
 			int distanceToIronOreFromCharacter = getDistanceBetweenPoints(GAME_WINDOW_CENTER, ironOreLocation);
 			if (distanceToIronOreFromCharacter < closestDistanceToIronOreFromCharacter) {
 				closestDistanceToIronOreFromCharacter = distanceToIronOreFromCharacter;
-				closestIronOreToCharacter = ironOreLocation;
+				closestIronOreToCharacter = new Point(ironOreLocation.x, ironOreLocation.y);
 			}
 		}
 		
-		if (closestDistanceToIronOreFromCharacter < MAXIMUM_DISTANCE_TO_WALK_TO_IRON_ORE) {
+		if (closestIronOreToCharacter != null && closestDistanceToIronOreFromCharacter < MAXIMUM_DISTANCE_TO_WALK_TO_IRON_ORE) {
 			return closestIronOreToCharacter;
 		}
 		return null;
 	}
 	
-	private ArrayList<Point> getIronOreLocations() {
-		// TODO: Use trained DNN here
-		return new ArrayList<Point>();
-	}
 	
 	public int getDistanceBetweenPoints(Point startingPoint, Point goalPoint) {
 		return (int) (Math.hypot(goalPoint.x - startingPoint.x, goalPoint.y - startingPoint.y));
 	}
 	
+	private BufferedImage captureScreenshotGameWindow() throws IOException {
+		Rectangle area = new Rectangle(103, 85, 510, 330);
+		return robot.createScreenCapture(area);
 		
+	}
 }

src/ObjectDetector.java

@@ -13,6 +13,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
+import java.awt.Point;
 import java.awt.image.BufferedImage;
 import java.awt.image.DataBufferByte;
 import java.io.File;
@@ -22,6 +23,7 @@ import java.nio.ByteBuffer;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Paths;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 import javax.imageio.ImageIO;
@@ -41,60 +43,11 @@ public class ObjectDetector {
 		model = SavedModelBundle.load("/home/dpapp/tensorflow-1.5.0/models/raccoon_dataset/results/checkpoint_23826/saved_model/", "serve");
 	}
 	
-	public void getIronOreLocationsFromImage(BufferedImage image) throws IOException {
+	public ArrayList<Point> getIronOreLocationsFromImage(String filename) throws IOException {
-		List<Tensor<?>> outputs = null;
-        
-        try (Tensor<UInt8> input = makeImageTensor(image)) {
-          outputs =
-              model
-                  .session()
-                  .runner()
-                  .feed("image_tensor", input)
-                  .fetch("detection_scores")
-                  .fetch("detection_classes")
-                  .fetch("detection_boxes")
-                  .run();
-        }
-        
-        try (Tensor<Float> scoresT = outputs.get(0).expect(Float.class);
-        	Tensor<Float> classesT = outputs.get(1).expect(Float.class);
-            Tensor<Float> boxesT = outputs.get(2).expect(Float.class)) {
-              // All these tensors have:
-              // - 1 as the first dimension
-              // - maxObjects as the second dimension
-              // While boxesT will have 4 as the third dimension (2 sets of (x, y) coordinates).
-              // This can be verified by looking at scoresT.shape() etc.
-            int maxObjects = (int) scoresT.shape()[1];
-            float[] scores = scoresT.copyTo(new float[1][maxObjects])[0];
-            float[] classes = classesT.copyTo(new float[1][maxObjects])[0];
-            float[][] boxes = boxesT.copyTo(new float[1][maxObjects][4])[0];
-              // Print all objects whose score is at least 0.5.
-            boolean foundSomething = false;
-            for (int i = 0; i < scores.length; ++i) {
-              if (scores[i] < 0.5) {
-                continue;
-              }
-              foundSomething = true;
-              System.out.printf("\tFound %-20s (score: %.4f)\n", "ironOre", 1.0000); //labels[(int) classes[i]], scores[i]);
-              System.out.println("Location:");
-              System.out.println("X:" + 510 * boxes[i][1] + ", Y:" + 330 * boxes[i][0] + ", width:" + 510 * boxes[i][3] + ", height:" + 330 * boxes[i][2]);
-            }
-            if (!foundSomething) {
-              System.out.println("No objects detected with a high enough score.");
-            }
-          }
-	}
-    
-    /*public void test() {
-    	try (SavedModelBundle model = SavedModelBundle.load("/home/dpapp/tensorflow-1.5.0/models/raccoon_dataset/results/checkpoint_23826/saved_model/", "serve")) {
-     // printSignature(model);
-    
-        final String filename = "/home/dpapp/tensorflow-1.5.0/models/raccoon_dataset/test_images/ironOre_test_9.jpg";
 		List<Tensor<?>> outputs = null;
+        ArrayList<Point> ironOreLocations = new ArrayList<Point>();
         
         try (Tensor<UInt8> input = makeImageTensor(filename)) {
-        	System.out.println("Image was converted to tensor.");
-        	long startTime = System.currentTimeMillis();
           outputs =
               model
                   .session()
@@ -104,7 +57,6 @@ public class ObjectDetector {
                   .fetch("detection_classes")
                   .fetch("detection_boxes")
                   .run();
-          System.out.println("Object detection took " + (System.currentTimeMillis() - startTime));
         }
         
         try (Tensor<Float> scoresT = outputs.get(0).expect(Float.class);
@@ -120,68 +72,29 @@ public class ObjectDetector {
             float[] classes = classesT.copyTo(new float[1][maxObjects])[0];
             float[][] boxes = boxesT.copyTo(new float[1][maxObjects][4])[0];
               // Print all objects whose score is at least 0.5.
-          System.out.printf("* %s\n", filename);
             boolean foundSomething = false;
             for (int i = 0; i < scores.length; ++i) {
-            if (scores[i] < 0.5) {
+              if (scores[i] < 0.75) {
                 continue;
               }
               foundSomething = true;
-            System.out.printf("\tFound %-20s (score: %.4f)\n", "ironOre", 1.0000); //labels[(int) classes[i]], scores[i]);
+              //System.out.printf("\tFound %-20s (score: %.4f)\n", "ironOre", scores[i]);
-            System.out.println("Location:");
+              //System.out.println("X:" + 510 * boxes[i][1] + ", Y:" + 330 * boxes[i][0] + ", width:" + 510 * boxes[i][3] + ", height:" + 330 * boxes[i][2]);
-            System.out.println("X:" + 510 * boxes[i][1] + ", Y:" + 330 * boxes[i][0] + ", width:" + 510 * boxes[i][3] + ", height:" + 330 * boxes[i][2]);
+              ironOreLocations.add(getCenterPointFromBox(boxes[i]));
             }
             if (!foundSomething) {
               System.out.println("No objects detected with a high enough score.");
             }
           }
-     
+        return ironOreLocations;
-    }
-  }*/
-
-  private static void printSignature(SavedModelBundle model) throws Exception {
-    /*MetaGraphDef m = MetaGraphDef.parseFrom(model.metaGraphDef());
-    SignatureDef sig = m.getSignatureDefOrThrow("serving_default");
-    int numInputs = sig.getInputsCount();
-    int i = 1;
-    System.out.println("MODEL SIGNATURE");
-    System.out.println("Inputs:");
-    for (Map.Entry<String, TensorInfo> entry : sig.getInputsMap().entrySet()) {
-      TensorInfo t = entry.getValue();
-      System.out.printf(
-          "%d of %d: %-20s (Node name in graph: %-20s, type: %s)\n",
-          i++, numInputs, entry.getKey(), t.getName(), t.getDtype());
-    }
-    int numOutputs = sig.getOutputsCount();
-    i = 1;
-    System.out.println("Outputs:");
-    for (Map.Entry<String, TensorInfo> entry : sig.getOutputsMap().entrySet()) {
-      TensorInfo t = entry.getValue();
-      System.out.printf(
-          "%d of %d: %-20s (Node name in graph: %-20s, type: %s)\n",
-          i++, numOutputs, entry.getKey(), t.getName(), t.getDtype());
-    }*/
-    System.out.println("-----------------------------------------------");
 	}
     
-  /*private static String[] loadLabels(String filename) throws Exception {
+	private Point getCenterPointFromBox(float[] box) {
-    String text = new String(Files.readAllBytes(Paths.get(filename)), StandardCharsets.UTF_8);
+		int x = (int) (510 * (box[1] + box[3]) / 2);
-    StringIntLabelMap.Builder builder = StringIntLabelMap.newBuilder();
+		int y = (int) (330 * (box[0] + box[2]) / 2);
-    TextFormat.merge(text, builder);
+		return new Point(x, y);
-    StringIntLabelMap proto = builder.build();
-    int maxId = 0;
-    for (StringIntLabelMapItem item : proto.getItemList()) {
-      if (item.getId() > maxId) {
-        maxId = item.getId();
 	}
-    }
+    
-    String[] ret = new String[maxId + 1];
-    for (StringIntLabelMapItem item : proto.getItemList()) {
-      ret[item.getId()] = item.getDisplayName();
-    }
-    String[] label = {"ironOre"};
-    return label;
-  }*/
 
   private static void bgr2rgb(byte[] data) {
     for (int i = 0; i < data.length; i += 3) {
@@ -191,14 +104,17 @@ public class ObjectDetector {
     }
   }
 
-  private static Tensor<UInt8> makeImageTensor(BufferedImage img) throws IOException {
+  private static Tensor<UInt8> makeImageTensor(String filename) throws IOException {
-    //BufferedImage img = ImageIO.read(new File(filename));
+    BufferedImage img = ImageIO.read(new File(filename));
     if (img.getType() != BufferedImage.TYPE_3BYTE_BGR) {
       throw new IOException(
           String.format(
               "Expected 3-byte BGR encoding in BufferedImage, found %d (file: %s). This code could be made more robust"));
     }
+	//System.out.println("Image is of type RGB? " + (img.getType() == BufferedImage.TYPE_INT_RGB));
+	//System.out.println("Image is of type RGB? " + (img.getType() == BufferedImage.TYPE_3BYTE_BGR));
     byte[] data = ((DataBufferByte) img.getData().getDataBuffer()).getData();
+
     // ImageIO.read seems to produce BGR-encoded images, but the model expects RGB.
     bgr2rgb(data);
     final long BATCH_SIZE = 1;

src/WillowChopper.java

@@ -17,8 +17,8 @@ public class WillowChopper {
 	
 	public WillowChopper() throws AWTException, IOException 
 	{
-		cursor = new Cursor();
+		//cursor = new Cursor();
-		cursorTask = new CursorTask();
+		//cursorTask = new CursorTask();
 		inventory = new Inventory();
 		objectDetector = new ObjectDetector();
 		robot = new Robot();
@@ -27,8 +27,10 @@ public class WillowChopper {
 	public void run() throws Exception {
 		System.out.println("Starting ironMiner...");
 		while (true) {
+			String filename = "/home/dpapp/Desktop/RunescapeAI/temp/screenshot.jpg";
 			BufferedImage image = captureScreenshotGameWindow();
-			objectDetector.getIronOreLocationsFromImage(image);
+			ImageIO.write(image, "jpg", new File(filename));
+			ArrayList<Point> ironOreLocations = objectDetector.getIronOreLocationsFromImage(filename);
 			System.out.println("--------------------------------\n\n");
 			/*
 			if (character.isCharacterEngaged()) {

src/main.java

@@ -7,9 +7,9 @@ import java.net.URL;
 public class main {
 
 	public static void main(String[] args) throws Exception {
-		System.out.println("Starting Willow Chopper.");
+		System.out.println("Starting Iron Miner.");
-		WillowChopper willowChopper = new WillowChopper();
+		IronMiner ironMiner = new IronMiner();
-		willowChopper.run();
+		ironMiner.run();
 		/*Cursor cursor = new Cursor();
 		CursorTask cursorTask = new CursorTask();
 		Inventory inventory = new Inventory();