diff --git a/src/IronMiner.java b/src/IronMiner.java index f21b98a..a725619 100644 --- a/src/IronMiner.java +++ b/src/IronMiner.java @@ -16,7 +16,10 @@ import org.opencv.core.CvType; import org.opencv.core.Mat; import org.opencv.core.Rect2d; import org.opencv.tracking.Tracker; +import org.opencv.tracking.TrackerBoosting; +import org.opencv.tracking.TrackerGOTURN; import org.opencv.tracking.TrackerKCF; +import org.opencv.tracking.TrackerMOSSE; public class IronMiner { @@ -33,9 +36,9 @@ public class IronMiner { public IronMiner() throws AWTException, IOException { - cursor = new Cursor(); - cursorTask = new CursorTask(); - inventory = new Inventory(); + //cursor = new Cursor(); + //cursorTask = new CursorTask(); + //inventory = new Inventory(); objectDetector = new ObjectDetector(); robot = new Robot(); randomizer = new Randomizer(); @@ -44,47 +47,56 @@ public class IronMiner { public void run() throws Exception { while (true) { BufferedImage screenCapture = objectDetector.captureScreenshotGameWindow(); - ArrayList detectedObjects = objectDetector.getObjectsInImage(screenCapture); + ArrayList detectedObjects = objectDetector.getObjectsInImage(screenCapture, 0.60); ArrayList ironOres = objectDetector.getObjectsOfClassInList(detectedObjects, "ironOre"); DetectedObject closestIronOre = getClosestObjectToCharacter(ironOres); if (closestIronOre != null) { System.out.println("Found iron ore! Starting tracking!"); - Tracker objectTracker = TrackerKCF.create(); + Tracker objectTracker = TrackerBoosting.create(); Rect2d boundingBox = closestIronOre.getBoundingRect2d(); objectTracker.init(getMatFromBufferedImage(screenCapture), boundingBox); - cursor.moveAndLeftClickAtCoordinatesWithRandomness(closestIronOre.getCenterForClicking(), 10, 10); + //cursor.moveAndLeftClickAtCoordinatesWithRandomness(closestIronOre.getCenterForClicking(), 10, 10); long mineStartTime = System.currentTimeMillis(); int maxTimeToMine = randomizer.nextGaussianWithinRange(3500, 5000); // track until either we lose the object or too much time passes int lostTrackCounter = 0; - while (((System.currentTimeMillis() - mineStartTime) < maxTimeToMine) && lostTrackCounter < 3) { - - screenCapture = objectDetector.captureScreenshotGameWindow(); - detectedObjects = objectDetector.getObjectsInImage(screenCapture); + //while (((System.currentTimeMillis() - mineStartTime) < maxTimeToMine) && lostTrackCounter < 3) { + while (lostTrackCounter < 600) { + //screenCapture = objectDetector.captureScreenshotGameWindow(); + detectedObjects = objectDetector.getObjectsInImage(screenCapture, 0.3); // Call with lower threshold value here + /* + * 600 iterations: 30 seconds boolean ok = objectTracker.update(getMatFromBufferedImage(screenCapture), boundingBox); - if (!ok || !objectDetector.isObjectPresentInBoundingBoxInImage(detectedObjects, boundingBox, "ironOre")) { + if (!ok) { System.out.println("Lost track for + " + lostTrackCounter + "! Finding new ore soon."); lostTrackCounter++; } - else if (ok) { - lostTrackCounter = 0; - System.out.println("Tracking at " + boundingBox.x + ", " + boundingBox.y + ", " + boundingBox.width + ", " + boundingBox.height); + else if (!objectDetector.isObjectPresentInBoundingBoxInImage(detectedObjects, boundingBox, "ironOre")) { + System.out.println("Can't find object in bounding box. Error " + lostTrackCounter + "! Finding new ore soon."); + lostTrackCounter++; } - + else if (ok) { + //lostTrackCounter = 0; + System.out.println("Tracking at " + boundingBox.x + ", " + boundingBox.y + ", " + boundingBox.width + ", " + boundingBox.height); + }*/ + System.out.println(lostTrackCounter); + lostTrackCounter++; } + System.out.println("600 iterations took " + (System.currentTimeMillis() - mineStartTime)); + break; } - dropInventoryIfFull(); + //dropInventoryIfFull(); } } private void dropInventoryIfFull() throws Exception { - inventory.update(); // TODO: add iron ore to inventory items + inventory.update(); // TODO: add iron ore to indonventory items if (inventory.isInventoryFull()) { cursorTask.optimizedDropAllItemsInInventory(cursor, inventory); } diff --git a/src/MirrorDisplay.java b/src/MirrorDisplay.java new file mode 100644 index 0000000..3f5b131 --- /dev/null +++ b/src/MirrorDisplay.java @@ -0,0 +1,46 @@ +import java.awt.AWTException; +import java.awt.Dimension; +import java.awt.Graphics; +import java.awt.Rectangle; +import java.awt.Robot; +import java.awt.image.BufferedImage; +import java.util.logging.Level; +import java.util.logging.Logger; +import javax.swing.JFrame; +import javax.swing.JPanel; + +public class MirrorDisplay extends JPanel{ + + private BufferedImage image; + + public MirrorDisplay() { + try { + Robot robot = new Robot(); + image = robot.createScreenCapture(new Rectangle(0, 0, 500, 500)); + } catch (AWTException ex) { + Logger.getLogger(MirrorDisplay.class.getName()).log(Level.SEVERE, null, ex); + } + + } + + @Override + public void paintComponent(Graphics g) { + g.drawImage(image, 0, 0, null); + + } +public static void main(String[] args) throws InterruptedException { + + JFrame test = new JFrame(); + + while (true) { + test.add(new MirrorDisplay()); + Dimension b = new Dimension(500,500); + test.setMinimumSize(b); + + test.setVisible(true); + Thread.sleep(50); + test.dispose(); + } + + } +} \ No newline at end of file diff --git a/src/ObjectDetector.java b/src/ObjectDetector.java index eb70c94..4232deb 100644 --- a/src/ObjectDetector.java +++ b/src/ObjectDetector.java @@ -61,11 +61,15 @@ public class ObjectDetector { this.detectedObjects = getRecognizedObjectsFromImage(fileName);*/ } - public ArrayList getObjectsInImage(BufferedImage image) throws Exception { + public ArrayList getObjectsInImage(BufferedImage image, double scoreThreshold) throws Exception { List> outputs = null; ArrayList detectedObjectsInImage = new ArrayList(); + long timebefore = System.currentTimeMillis(); + makeImageTensor(image); + System.out.println("Conversion took: " + (System.currentTimeMillis() - timebefore)); try (Tensor input = makeImageTensor(image)) { + long timebefore2 = System.currentTimeMillis(); outputs = model .session() @@ -75,6 +79,7 @@ public class ObjectDetector { .fetch("detection_classes") .fetch("detection_boxes") .run(); + System.out.println("Model took: " + (System.currentTimeMillis() - timebefore)); } try (Tensor scoresT = outputs.get(0).expect(Float.class); @@ -87,7 +92,7 @@ public class ObjectDetector { float[][] boxes = boxesT.copyTo(new float[1][maxObjects][4])[0]; for (int i = 0; i < scores.length; ++i) { - if (scores[i] > 0.80) { + if (scores[i] > scoreThreshold) { detectedObjectsInImage.add(new DetectedObject(scores[i], classes[i], boxes[i])); } } diff --git a/src/ObjectDetectorTest.java b/src/ObjectDetectorTest.java index c03493a..6f219c1 100644 --- a/src/ObjectDetectorTest.java +++ b/src/ObjectDetectorTest.java @@ -17,7 +17,7 @@ class ObjectDetectorTest { void testObjectDetection() throws Exception { ObjectDetector objectDetector = new ObjectDetector(); BufferedImage testImage1 = ImageIO.read(new File("/home/dpapp/tensorflow-1.5.0/models/raccoon_dataset/test_images/ironOre_test_9.jpg")); - ArrayList detectedObjects1 = objectDetector.getObjectsInImage(testImage1); + ArrayList detectedObjects1 = objectDetector.getObjectsInImage(testImage1, 0.8); ArrayList detectedIronOres1 = objectDetector.getObjectsOfClassInList(detectedObjects1, "ironOre"); ArrayList detectedOres1 = objectDetector.getObjectsOfClassInList(detectedObjects1, "ore"); @@ -26,6 +26,13 @@ class ObjectDetectorTest { //assertDetectedObjectsAreEqual(); } + + /*@Test + void testObjectDetectionInSubImage() throws Exception { + ObjectDetector objectDetector = new ObjectDetector(); + BufferedImage testImage = ImageIO.read(new File("/home/dpapp/tensorflow-1.5.0/models/raccoon_dataset/test_images/ironOre_test_9.jpg")); + }*/ + void assertDetectedObjectsAreEqual(DetectedObject obj1, DetectedObject obj2) { } diff --git a/src/ObjectTrackerTest.java b/src/ObjectTrackerTest.java index 4ecd897..2cbae85 100644 --- a/src/ObjectTrackerTest.java +++ b/src/ObjectTrackerTest.java @@ -1,6 +1,8 @@ import static org.junit.jupiter.api.Assertions.*; import java.awt.AWTException; +import java.awt.Color; +import java.awt.Graphics; import java.awt.image.BufferedImage; import java.io.ByteArrayInputStream; import java.io.File; @@ -16,6 +18,9 @@ import org.opencv.core.Rect2d; import org.opencv.imgcodecs.Imgcodecs; import org.opencv.tracking.Tracker; import org.opencv.tracking.TrackerKCF; +import org.opencv.tracking.TrackerMIL; +import org.opencv.tracking.TrackerBoosting; +import org.opencv.tracking.TrackerMedianFlow; import org.opencv.videoio.VideoCapture; class ObjectTrackerTest { @@ -23,7 +28,7 @@ class ObjectTrackerTest { @Test void testObjectTracking() throws Exception { System.loadLibrary(Core.NATIVE_LIBRARY_NAME); - VideoCapture video = new VideoCapture("/home/dpapp/Videos/gameplay-2018-02-24_10.01.00.mp4"); + VideoCapture video = new VideoCapture("/home/dpapp/Videos/gameplay-2018-02-24_14.24.53.mp4"); System.out.println("loaded video..."); ObjectDetector objectDetector = new ObjectDetector(); @@ -32,24 +37,27 @@ class ObjectTrackerTest { boolean frameReadSuccess = video.read(frame); assertTrue(frameReadSuccess); - ArrayList detectedObjects = objectDetector.getObjectsInImage(Mat2BufferedImage(frame)); + ArrayList detectedObjects = objectDetector.getObjectsInImage(Mat2BufferedImage(frame), 0.60); System.out.println("Tracking " + detectedObjects.size() + " objects."); + ArrayList objectTrackers = new ArrayList(); ArrayList boundingBoxes = new ArrayList(); - for (int i = 0; i < 3; i++) { + for (int i = 0; i < detectedObjects.size(); i++) { boundingBoxes.add(detectedObjects.get(i).getBoundingRect2d()); - objectTrackers.add(TrackerKCF.create()); + objectTrackers.add(TrackerBoosting.create()); objectTrackers.get(i).init(frame, boundingBoxes.get(i)); } //System.out.println("bounding box: " + (int) boundingBoxes.get(0).x + ", " + (int) boundingBoxes.get(0).y + ", " + (int) boundingBoxes.get(0).width + ", " + (int) boundingBoxes.get(0).height); int counter = 0; while (video.read(frame)) { - for (int i = 0; i < 3; i++) { + BufferedImage screencapture = Mat2BufferedImage(frame); + detectedObjects = objectDetector.getObjectsInImage(screencapture, 0.3); + + for (int i = 0; i < objectTrackers.size(); i++) { objectTrackers.get(i).update(frame, boundingBoxes.get(i)); boolean trackingSuccess = objectTrackers.get(i).update(frame, boundingBoxes.get(i)); - detectedObjects = objectDetector.getObjectsInImage(Mat2BufferedImage(frame)); //System.out.println(detectedObjects.size()); //System.out.println((int) boundingBoxes.get(i).x + ", " + (int) boundingBoxes.get(i).y + ", " + (int) boundingBoxes.get(i).width + ", " + (int) boundingBoxes.get(i).height); //BufferedImage subImage = screencapture.getSubimage((int) boundingBoxes.get(i).x- 10, (int) boundingBoxes.get(i).y - 10, (int) boundingBoxes.get(i).width + 20, (int) boundingBoxes.get(i).height + 20); @@ -57,13 +65,27 @@ class ObjectTrackerTest { boolean ironOreDetected = objectDetector.isObjectPresentInBoundingBoxInImage(detectedObjects, boundingBoxes.get(i), "ironOre"); boolean oreDetected = objectDetector.isObjectPresentInBoundingBoxInImage(detectedObjects, boundingBoxes.get(i), "ore"); - //ImageIO.write(screencapture, "jpg", new File("/home/dpapp/Videos/frames/frame_" + counter + ".jpg")); + Graphics g = screencapture.getGraphics(); + if (ironOreDetected) { + g.setColor(Color.GREEN); + } + else if (oreDetected) { + g.setColor(Color.RED); + } + else { + g.setColor(Color.WHITE); + } + g.drawRect((int) boundingBoxes.get(i).x, (int) boundingBoxes.get(i).y, (int) boundingBoxes.get(i).width, (int) boundingBoxes.get(i).height); + //ImageIO.write(subImage, "jpg", new File("/home/dpapp/Videos/sub_frames/frame_" + counter + "_sub.jpg")); //System.out.println("wrote file..."); - if (i == 2) { + /*if (i == ) { System.out.println(trackingSuccess + ", ironOre: " + ironOreDetected + ", ore:" + oreDetected); - } + }*/ + } + System.out.println("Wrote image..."); + ImageIO.write(screencapture, "jpg", new File("/home/dpapp/Videos/frames/frame_" + counter + ".jpg")); counter++; } diff --git a/target/classes/IronMiner.class b/target/classes/IronMiner.class index 722072e..71dbcea 100644 Binary files a/target/classes/IronMiner.class and b/target/classes/IronMiner.class differ diff --git a/target/classes/MirrorDisplay.class b/target/classes/MirrorDisplay.class new file mode 100644 index 0000000..2bdff3e Binary files /dev/null and b/target/classes/MirrorDisplay.class differ diff --git a/target/classes/ObjectDetector.class b/target/classes/ObjectDetector.class index 92603e3..8e978eb 100644 Binary files a/target/classes/ObjectDetector.class and b/target/classes/ObjectDetector.class differ diff --git a/target/classes/ObjectDetectorTest.class b/target/classes/ObjectDetectorTest.class index 91e94c2..17be09d 100644 Binary files a/target/classes/ObjectDetectorTest.class and b/target/classes/ObjectDetectorTest.class differ diff --git a/target/classes/ObjectTrackerTest.class b/target/classes/ObjectTrackerTest.class index 759d443..513a8ea 100644 Binary files a/target/classes/ObjectTrackerTest.class and b/target/classes/ObjectTrackerTest.class differ