moparisthebest
/
PowerMiner
mirror of https://github.com/davpapp/PowerMiner
1
0
Fork 0
Code Issues Releases Wiki Activity

Object detection is now fully functional.

This commit is contained in:
davpapp 2018-02-22 09:28:21 -05:00
parent f27f17b2c3
commit 343c984d5e
10 changed files with 78 additions and 136 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/ColorAnalyzer.java
View File

@ -124,7 +124,7 @@ public class ColorAnalyzer {
{ {
ColorAnalyzer colorAnalyzer = new ColorAnalyzer(); ColorAnalyzer colorAnalyzer = new ColorAnalyzer();
//colorAnalyzer.showColorDistribution("screenshot21.jpg"); //colorAnalyzer.showColorDistribution("screenshot21.jpg");
//colorAnalyzer.printCursorColor(); colorAnalyzer.printCursorColor();
colorAnalyzer.colorImage(); //colorAnalyzer.colorImage();
} }
} }

72
src/IronMiner.java
View File

@ -1,75 +1,99 @@
import java.awt.AWTException; import java.awt.AWTException;
import java.awt.Point; import java.awt.Point;
import java.awt.Rectangle;
import java.awt.Robot;
import java.awt.image.BufferedImage;
import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList; import java.util.ArrayList;
import javax.imageio.ImageIO;
public class IronMiner { public class IronMiner {
public static final int IRON_ORE_MINING_TIME_MILLISECONDS = 650; public static final int IRON_ORE_MINING_TIME_MILLISECONDS = 2738;
public static final int MAXIMUM_DISTANCE_TO_WALK_TO_IRON_ORE = 650; public static final int MAXIMUM_DISTANCE_TO_WALK_TO_IRON_ORE = 400;
public static final Point GAME_WINDOW_CENTER = new Point(200, 300); public static final Point GAME_WINDOW_CENTER = new Point(510 / 2, 330 / 2);
Cursor cursor; Cursor cursor;
CursorTask cursorTask; CursorTask cursorTask;
Inventory inventory; Inventory inventory;
ObjectDetector objectDetector;
Robot robot;
Randomizer randomizer;
public IronMiner() throws AWTException, IOException public IronMiner() throws AWTException, IOException
{ {
cursor = new Cursor(); cursor = new Cursor();
cursorTask = new CursorTask(); cursorTask = new CursorTask();
inventory = new Inventory(); inventory = new Inventory();
objectDetector = new ObjectDetector();
robot = new Robot();
randomizer = new Randomizer();
} }
public void run() throws Exception { public void run() throws Exception {
while (true) { while (true) {
Thread.sleep(250); //Thread.sleep(250);
mineClosestIronOre(); String filename = "/home/dpapp/Desktop/RunescapeAI/temp/screenshot.jpg";
BufferedImage image = captureScreenshotGameWindow();
ImageIO.write(image, "jpg", new File(filename));
mineClosestIronOre(filename);
inventory.update(); // TODO: add iron ore to inventory items dropInventoryIfFull();
if (inventory.isInventoryFull()) {
System.out.println("Inventory is full! Dropping...");
cursorTask.optimizedDropAllItemsInInventory(cursor, inventory);
}
} }
} }
private void mineClosestIronOre() throws Exception { private void dropInventoryIfFull() throws Exception {
Point ironOreLocation = getClosestIronOre(); inventory.update(); // TODO: add iron ore to inventory items
if (ironOreLocation == null) { if (inventory.isInventoryFull()) {
cursorTask.optimizedDropAllItemsInInventory(cursor, inventory);
}
}
private void mineClosestIronOre(String filename) throws Exception {
Point ironOreLocation = getClosestIronOre(filename);
/*if (ironOreLocation == null) {
Thread.sleep(1000); Thread.sleep(1000);
}*/
if (ironOreLocation != null) {
System.out.println("Mineable iron at (" + (ironOreLocation.x + 103) + "," + (ironOreLocation.y + 85) + ")");
Point actualIronOreLocation = new Point(ironOreLocation.x + 103, ironOreLocation.y + 85);
cursor.moveAndLeftClickAtCoordinatesWithRandomness(actualIronOreLocation, 12, 12);
Thread.sleep(randomizer.nextGaussianWithinRange(IRON_ORE_MINING_TIME_MILLISECONDS - 350, IRON_ORE_MINING_TIME_MILLISECONDS + 150));
} }
cursor.moveAndLeftClickAtCoordinatesWithRandomness(ironOreLocation, 20, 20);
Thread.sleep(IRON_ORE_MINING_TIME_MILLISECONDS);
} }
private Point getClosestIronOre() { private Point getClosestIronOre(String filename) throws IOException {
ArrayList<Point> ironOreLocations = getIronOreLocations(); ArrayList<Point> ironOreLocations = objectDetector.getIronOreLocationsFromImage(filename);
System.out.println(ironOreLocations.size());
int closestDistanceToIronOreFromCharacter = Integer.MAX_VALUE; int closestDistanceToIronOreFromCharacter = Integer.MAX_VALUE;
Point closestIronOreToCharacter = null; Point closestIronOreToCharacter = null;
for (Point ironOreLocation : ironOreLocations) { for (Point ironOreLocation : ironOreLocations) {
int distanceToIronOreFromCharacter = getDistanceBetweenPoints(GAME_WINDOW_CENTER, ironOreLocation); int distanceToIronOreFromCharacter = getDistanceBetweenPoints(GAME_WINDOW_CENTER, ironOreLocation);
if (distanceToIronOreFromCharacter < closestDistanceToIronOreFromCharacter) { if (distanceToIronOreFromCharacter < closestDistanceToIronOreFromCharacter) {
closestDistanceToIronOreFromCharacter = distanceToIronOreFromCharacter; closestDistanceToIronOreFromCharacter = distanceToIronOreFromCharacter;
closestIronOreToCharacter = ironOreLocation; closestIronOreToCharacter = new Point(ironOreLocation.x, ironOreLocation.y);
} }
} }
if (closestDistanceToIronOreFromCharacter < MAXIMUM_DISTANCE_TO_WALK_TO_IRON_ORE) { if (closestIronOreToCharacter != null && closestDistanceToIronOreFromCharacter < MAXIMUM_DISTANCE_TO_WALK_TO_IRON_ORE) {
return closestIronOreToCharacter; return closestIronOreToCharacter;
} }
return null; return null;
} }
private ArrayList<Point> getIronOreLocations() {
// TODO: Use trained DNN here
return new ArrayList<Point>();
}
public int getDistanceBetweenPoints(Point startingPoint, Point goalPoint) { public int getDistanceBetweenPoints(Point startingPoint, Point goalPoint) {
return (int) (Math.hypot(goalPoint.x - startingPoint.x, goalPoint.y - startingPoint.y)); return (int) (Math.hypot(goalPoint.x - startingPoint.x, goalPoint.y - startingPoint.y));
} }
private BufferedImage captureScreenshotGameWindow() throws IOException {
Rectangle area = new Rectangle(103, 85, 510, 330);
return robot.createScreenCapture(area);
}
} }

124
src/ObjectDetector.java
View File

@ -13,6 +13,7 @@ See the License for the specific language governing permissions and
limitations under the License. limitations under the License.
==============================================================================*/ ==============================================================================*/
import java.awt.Point;
import java.awt.image.BufferedImage; import java.awt.image.BufferedImage;
import java.awt.image.DataBufferByte; import java.awt.image.DataBufferByte;
import java.io.File; import java.io.File;
@ -22,6 +23,7 @@ import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Paths; import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import javax.imageio.ImageIO; import javax.imageio.ImageIO;
@ -41,10 +43,11 @@ public class ObjectDetector {
model = SavedModelBundle.load("/home/dpapp/tensorflow-1.5.0/models/raccoon_dataset/results/checkpoint_23826/saved_model/", "serve"); model = SavedModelBundle.load("/home/dpapp/tensorflow-1.5.0/models/raccoon_dataset/results/checkpoint_23826/saved_model/", "serve");
} }
public void getIronOreLocationsFromImage(BufferedImage image) throws IOException { public ArrayList<Point> getIronOreLocationsFromImage(String filename) throws IOException {
List<Tensor<?>> outputs = null; List<Tensor<?>> outputs = null;
ArrayList<Point> ironOreLocations = new ArrayList<Point>();
try (Tensor<UInt8> input = makeImageTensor(image)) { try (Tensor<UInt8> input = makeImageTensor(filename)) {
outputs = outputs =
model model
.session() .session()
@ -71,117 +74,27 @@ public class ObjectDetector {
// Print all objects whose score is at least 0.5. // Print all objects whose score is at least 0.5.
boolean foundSomething = false; boolean foundSomething = false;
for (int i = 0; i < scores.length; ++i) { for (int i = 0; i < scores.length; ++i) {
if (scores[i] < 0.5) { if (scores[i] < 0.75) {
continue; continue;
} }
foundSomething = true; foundSomething = true;
System.out.printf("\tFound %-20s (score: %.4f)\n", "ironOre", 1.0000); //labels[(int) classes[i]], scores[i]); //System.out.printf("\tFound %-20s (score: %.4f)\n", "ironOre", scores[i]);
System.out.println("Location:"); //System.out.println("X:" + 510 * boxes[i][1] + ", Y:" + 330 * boxes[i][0] + ", width:" + 510 * boxes[i][3] + ", height:" + 330 * boxes[i][2]);
System.out.println("X:" + 510 * boxes[i][1] + ", Y:" + 330 * boxes[i][0] + ", width:" + 510 * boxes[i][3] + ", height:" + 330 * boxes[i][2]); ironOreLocations.add(getCenterPointFromBox(boxes[i]));
} }
if (!foundSomething) { if (!foundSomething) {
System.out.println("No objects detected with a high enough score."); System.out.println("No objects detected with a high enough score.");
} }
} }
return ironOreLocations;
} }
/*public void test() { private Point getCenterPointFromBox(float[] box) {
try (SavedModelBundle model = SavedModelBundle.load("/home/dpapp/tensorflow-1.5.0/models/raccoon_dataset/results/checkpoint_23826/saved_model/", "serve")) { int x = (int) (510 * (box[1] + box[3]) / 2);
// printSignature(model); int y = (int) (330 * (box[0] + box[2]) / 2);
return new Point(x, y);
}
final String filename = "/home/dpapp/tensorflow-1.5.0/models/raccoon_dataset/test_images/ironOre_test_9.jpg";
List<Tensor<?>> outputs = null;
try (Tensor<UInt8> input = makeImageTensor(filename)) {
System.out.println("Image was converted to tensor.");
long startTime = System.currentTimeMillis();
outputs =
model
.session()
.runner()
.feed("image_tensor", input)
.fetch("detection_scores")
.fetch("detection_classes")
.fetch("detection_boxes")
.run();
System.out.println("Object detection took " + (System.currentTimeMillis() - startTime));
}
try (Tensor<Float> scoresT = outputs.get(0).expect(Float.class);
Tensor<Float> classesT = outputs.get(1).expect(Float.class);
Tensor<Float> boxesT = outputs.get(2).expect(Float.class)) {
// All these tensors have:
// - 1 as the first dimension
// - maxObjects as the second dimension
// While boxesT will have 4 as the third dimension (2 sets of (x, y) coordinates).
// This can be verified by looking at scoresT.shape() etc.
int maxObjects = (int) scoresT.shape()[1];
float[] scores = scoresT.copyTo(new float[1][maxObjects])[0];
float[] classes = classesT.copyTo(new float[1][maxObjects])[0];
float[][] boxes = boxesT.copyTo(new float[1][maxObjects][4])[0];
// Print all objects whose score is at least 0.5.
System.out.printf("* %s\n", filename);
boolean foundSomething = false;
for (int i = 0; i < scores.length; ++i) {
if (scores[i] < 0.5) {
continue;
}
foundSomething = true;
System.out.printf("\tFound %-20s (score: %.4f)\n", "ironOre", 1.0000); //labels[(int) classes[i]], scores[i]);
System.out.println("Location:");
System.out.println("X:" + 510 * boxes[i][1] + ", Y:" + 330 * boxes[i][0] + ", width:" + 510 * boxes[i][3] + ", height:" + 330 * boxes[i][2]);
}
if (!foundSomething) {
System.out.println("No objects detected with a high enough score.");
}
}
}
}*/
private static void printSignature(SavedModelBundle model) throws Exception {
/*MetaGraphDef m = MetaGraphDef.parseFrom(model.metaGraphDef());
SignatureDef sig = m.getSignatureDefOrThrow("serving_default");
int numInputs = sig.getInputsCount();
int i = 1;
System.out.println("MODEL SIGNATURE");
System.out.println("Inputs:");
for (Map.Entry<String, TensorInfo> entry : sig.getInputsMap().entrySet()) {
TensorInfo t = entry.getValue();
System.out.printf(
"%d of %d: %-20s (Node name in graph: %-20s, type: %s)\n",
i++, numInputs, entry.getKey(), t.getName(), t.getDtype());
}
int numOutputs = sig.getOutputsCount();
i = 1;
System.out.println("Outputs:");
for (Map.Entry<String, TensorInfo> entry : sig.getOutputsMap().entrySet()) {
TensorInfo t = entry.getValue();
System.out.printf(
"%d of %d: %-20s (Node name in graph: %-20s, type: %s)\n",
i++, numOutputs, entry.getKey(), t.getName(), t.getDtype());
}*/
System.out.println("-----------------------------------------------");
}
/*private static String[] loadLabels(String filename) throws Exception {
String text = new String(Files.readAllBytes(Paths.get(filename)), StandardCharsets.UTF_8);
StringIntLabelMap.Builder builder = StringIntLabelMap.newBuilder();
TextFormat.merge(text, builder);
StringIntLabelMap proto = builder.build();
int maxId = 0;
for (StringIntLabelMapItem item : proto.getItemList()) {
if (item.getId() > maxId) {
maxId = item.getId();
}
}
String[] ret = new String[maxId + 1];
for (StringIntLabelMapItem item : proto.getItemList()) {
ret[item.getId()] = item.getDisplayName();
}
String[] label = {"ironOre"};
return label;
}*/
private static void bgr2rgb(byte[] data) { private static void bgr2rgb(byte[] data) {
for (int i = 0; i < data.length; i += 3) { for (int i = 0; i < data.length; i += 3) {
@ -191,14 +104,17 @@ public class ObjectDetector {
} }
} }
private static Tensor<UInt8> makeImageTensor(BufferedImage img) throws IOException { private static Tensor<UInt8> makeImageTensor(String filename) throws IOException {
//BufferedImage img = ImageIO.read(new File(filename)); BufferedImage img = ImageIO.read(new File(filename));
if (img.getType() != BufferedImage.TYPE_3BYTE_BGR) { if (img.getType() != BufferedImage.TYPE_3BYTE_BGR) {
throw new IOException( throw new IOException(
String.format( String.format(
"Expected 3-byte BGR encoding in BufferedImage, found %d (file: %s). This code could be made more robust")); "Expected 3-byte BGR encoding in BufferedImage, found %d (file: %s). This code could be made more robust"));
} }
//System.out.println("Image is of type RGB? " + (img.getType() == BufferedImage.TYPE_INT_RGB));
//System.out.println("Image is of type RGB? " + (img.getType() == BufferedImage.TYPE_3BYTE_BGR));
byte[] data = ((DataBufferByte) img.getData().getDataBuffer()).getData(); byte[] data = ((DataBufferByte) img.getData().getDataBuffer()).getData();
// ImageIO.read seems to produce BGR-encoded images, but the model expects RGB. // ImageIO.read seems to produce BGR-encoded images, but the model expects RGB.
bgr2rgb(data); bgr2rgb(data);
final long BATCH_SIZE = 1; final long BATCH_SIZE = 1;

8
src/WillowChopper.java
View File

@ -17,8 +17,8 @@ public class WillowChopper {
public WillowChopper() throws AWTException, IOException public WillowChopper() throws AWTException, IOException
{ {
cursor = new Cursor(); //cursor = new Cursor();
cursorTask = new CursorTask(); //cursorTask = new CursorTask();
inventory = new Inventory(); inventory = new Inventory();
objectDetector = new ObjectDetector(); objectDetector = new ObjectDetector();
robot = new Robot(); robot = new Robot();
@ -27,8 +27,10 @@ public class WillowChopper {
public void run() throws Exception { public void run() throws Exception {
System.out.println("Starting ironMiner..."); System.out.println("Starting ironMiner...");
while (true) { while (true) {
String filename = "/home/dpapp/Desktop/RunescapeAI/temp/screenshot.jpg";
BufferedImage image = captureScreenshotGameWindow(); BufferedImage image = captureScreenshotGameWindow();
objectDetector.getIronOreLocationsFromImage(image); ImageIO.write(image, "jpg", new File(filename));
ArrayList<Point> ironOreLocations = objectDetector.getIronOreLocationsFromImage(filename);
System.out.println("--------------------------------\n\n"); System.out.println("--------------------------------\n\n");
/* /*
if (character.isCharacterEngaged()) { if (character.isCharacterEngaged()) {

6
src/main.java
View File

@ -7,9 +7,9 @@ import java.net.URL;
public class main { public class main {
public static void main(String[] args) throws Exception { public static void main(String[] args) throws Exception {
System.out.println("Starting Willow Chopper."); System.out.println("Starting Iron Miner.");
WillowChopper willowChopper = new WillowChopper(); IronMiner ironMiner = new IronMiner();
willowChopper.run(); ironMiner.run();
/*Cursor cursor = new Cursor(); /*Cursor cursor = new Cursor();
CursorTask cursorTask = new CursorTask(); CursorTask cursorTask = new CursorTask();
Inventory inventory = new Inventory(); Inventory inventory = new Inventory();

BIN
target/classes/ColorAnalyzer.class
View File

Binary file not shown.

BIN
target/classes/IronMiner.class
View File

Binary file not shown.

BIN
target/classes/ObjectDetector.class
View File

Binary file not shown.

BIN
target/classes/WillowChopper.class
View File

Binary file not shown.

BIN
target/classes/main.class
View File

Binary file not shown.