Refactoring cursor code

This commit is contained in:
davpapp 2018-02-12 19:40:09 -05:00
parent e6ab945729
commit 90c8cbb33b
20 changed files with 204 additions and 110 deletions

View File

@ -1,17 +1,18 @@
import numpy as np
import cv2
willowCascade = cv2.CascadeClassifier('/home/dpapp/open/opencv-haar-classifier-training/classifier/stage9.xml')
img = cv2.imread('/home/dpapp/Desktop/RunescapeAIPics/CascadeTraining/Testing/screenshot0.png')
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
willowCascade = cv2.CascadeClassifier('/home/dpapp/open/opencv-haar-classifier-training/classifier/stage8.xml')
img = cv2.imread('/home/dpapp/Desktop/RunescapeAIPics/CascadeTraining/Testing/screenshot1.jpg')
#gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
willows = willowCascade.detectMultiScale(gray, 1.3, 5)
willows = willowCascade.detectMultiScale(img, 1.3, 5)
for (x,y,w,h) in willows:
cv2.rectangle(img,(x,y),(x+w,y+h),(255,0,0),2)
roi_gray = gray[y:y+h, x:x+w]
roi_color = img[y:y+h, x:x+w]
print("Found willow!")
cv2.imshow('img', img)
cv2.waitKey(0)
cv2.destroyAllWindows()

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

BIN
bin/ImageCollector$1.class Normal file

Binary file not shown.

BIN
bin/ImageCollector.class Normal file

Binary file not shown.

Binary file not shown.

View File

@ -28,6 +28,7 @@ public class Cursor {
private Robot robot;
private Randomizer randomizer;
private Random random;
private ArrayList<ArrayList<CursorPath>> cursorPathsByDistance;
@ -37,6 +38,7 @@ public class Cursor {
robot = new Robot();
randomizer = new Randomizer();
random = new Random();
}
private void initializeCursorPathsByDistanceFromFile(String path) {
@ -115,22 +117,25 @@ public class Cursor {
public void moveCursorToCoordinates(Point goalPoint) throws InterruptedException {
Point startingPoint = getCurrentCursorPoint();
int distanceToMoveCursor = (int) startingPoint.distance(goalPoint);
int distanceToMoveCursor = getDistanceBetweenPoints(startingPoint, goalPoint);
if (distanceToMoveCursor == 0) {
return;
}
double angleToMoveCursor = getThetaBetweenPoints(startingPoint, goalPoint);
// TODO: check if exists
CursorPath cursorPathToFollow = chooseCursorPathToFollowBasedOnDistance(distanceToMoveCursor);
double angleToTranslatePathBy = angleToMoveCursor - cursorPathToFollow.getCursorPathTheta();
followCursorPath(startingCursorPoint, angleToTranslatePathBy, cursorPathToFollow);
}
public int getDistanceBetweenPoints(Point startingPoint, Point goalPoint) {
return (int) (Math.hypot(goalPoint.x - startingPoint.x, goalPoint.y - startingPoint.y));
}
//TODO: fix
private double getThetaBetweenPoints(Point startingPoint, Point goalPoint) {
return Math.atan2(startingPoint.x * goalPoint.y, 1.0 * goalPoint.x);
public double getThetaBetweenPoints(Point startingPoint, Point goalPoint) {
return Math.atan2(goalPoint.x - startingPoint.x, goalPoint.y - startingPoint.y);
}
private void followCursorPath(Point startingCursorPoint, double angleToTranslatePathBy, CursorPath cursorPathToFollow) throws InterruptedException {
@ -146,15 +151,12 @@ public class Cursor {
}
private CursorPath chooseCursorPathToFollowBasedOnDistance(int distanceToMoveCursor) {
if (distanceToMoveCursor == 0) {
return new CursorPath(new ArrayList<CursorPoint>());
}
int newDistanceToMoveCursor = findNearestPathLengthThatExists(distanceToMoveCursor);
double scaleFactor = getScaleFactor(newDistanceToMoveCursor, distanceToMoveCursor);
double scaleToFactorBy = getScaleToFactorBy(newDistanceToMoveCursor, distanceToMoveCursor);
ArrayList<CursorPath> cursorPathsWithSameDistance = cursorPathsByDistance.get(newDistanceToMoveCursor);
CursorPath scaledCursorPath = cursorPathsWithSameDistance.get(new Random().nextInt(cursorPathsWithSameDistance.size())).getScaledCopyOfCursorPath(1.0);
int indexOfRandomPathToFollow = random.nextInt(cursorPathsWithSameDistance.size());
ArrayList<CursorPoint> scaledCursorPath = cursorPathsWithSameDistance.get(indexOfRandomPathToFollow).getScaledCopyOfCursorPath(scaleToFactorBy);
return scaledCursorPath;
}
@ -171,7 +173,7 @@ public class Cursor {
return distanceToMoveCursor + offset;
}
private double getScaleFactor(int newDistanceToMoveCursor, int distanceToMoveCursor) {
private double getScaleToFactorBy(int newDistanceToMoveCursor, int distanceToMoveCursor) {
return (1.0 * newDistanceToMoveCursor / distanceToMoveCursor);
}

View File

@ -26,15 +26,15 @@ public class CursorPath {
return normalizedDelayCursorPoints;
}
private ArrayList<CursorPoint> getTranslatedCopyOfCursorPath(ArrayList<CursorPoint> cursorPoints, CursorPoint cursorPointToTranslateBy) {
private CursorPath getTranslatedCopyOfCursorPath(ArrayList<CursorPoint> cursorPoints, CursorPoint cursorPointToTranslateBy) {
ArrayList<CursorPoint> offsetCursorPath = new ArrayList<CursorPoint>();
for (CursorPoint cursorPoint : cursorPoints) {
offsetCursorPath.add(cursorPoint.getCursorPointTranslatedBy(cursorPointToTranslateBy));
}
return offsetCursorPath;
return new CursorPath(offsetCursorPath);
}
private ArrayList<CursorPoint> getNormalizedDelayCopyOfCursorPath(ArrayList<CursorPoint> cursorPoints) {
private CursorPath getNormalizedDelayCopyOfCursorPath(ArrayList<CursorPoint> cursorPoints) {
ArrayList<CursorPoint> normalizedDelayCursorPoints = new ArrayList<CursorPoint>();
for (int i = 0; i < cursorPoints.size() - 1; i++) {
CursorPoint cursorPoint = cursorPoints.get(i);
@ -42,24 +42,24 @@ public class CursorPath {
normalizedDelayCursorPoints.add(cursorPoint.getCursorPointWithNewDelay(nextCursorPoint.delay - cursorPoint.delay));
}
normalizedDelayCursorPoints.add(cursorPoints.get(cursorPoints.size() - 1).getCursorPointWithNewDelay(0));
return normalizedDelayCursorPoints;
return new CursorPath(normalizedDelayCursorPoints);
}
public ArrayList<CursorPoint> getScaledCopyOfCursorPath(double factorToScaleBy) {
public CursorPath getScaledCopyOfCursorPath(ArrayList<CursorPoint> cursorPoints, double factorToScaleBy) {
ArrayList<CursorPoint> scaledCursorPath = new ArrayList<CursorPoint>();
for (CursorPoint cursorPoint : this.cursorPoints) {
for (CursorPoint cursorPoint : cursorPoints) {
scaledCursorPath.add(cursorPoint.getCursorPointScaledBy(factorToScaleBy));
}
return scaledCursorPath;
return new CursorPath(scaledCursorPath);
}
public ArrayList<CursorPoint> getRotatedCopyOfCursorPath(double angleToRotateBy) {
public CursorPath getRotatedCopyOfCursorPath(ArrayList<CursorPoint> cursorPoints, double angleToRotateBy) {
ArrayList<CursorPoint> rotatedCursorPath = new ArrayList<CursorPoint>();
for (CursorPoint cursorPoint : this.cursorPoints) {
for (CursorPoint cursorPoint : cursorPoints) {
rotatedCursorPath.add(cursorPoint.getCursorPointRotatedBy(angleToRotateBy));
}
return rotatedCursorPath;
return new CursorPath(rotatedCursorPath);
}
private int calculateCursorPathTimespan() {
@ -75,7 +75,8 @@ public class CursorPath {
}
private double calculateCursorPathTheta() {
return getStartingCursorPoint().getThetaFrom(getEndingCursorPoint());
CursorPoint endingCursorPoint = getEndingCursorPoint();
return Math.atan2(endingCursorPoint.y, endingCursorPoint.x);
}
public CursorPoint getStartingCursorPoint() {

View File

@ -16,8 +16,10 @@ class CursorPathTest {
for (CursorPath cursorPath : cursorPaths) {
testCursorPathStartsAtOrigin(cursorPath);
testEndingCursorPointInCursorPathHasZeroDelay(cursorPath);
testCursorPathDistance(cursorPath);
testCursorPathTheta(cursorPath);
//testCursorPathLocations(cursorPath);
testCursorPathRotation(cursorPath, 15);
testCursorPathRotation(cursorPath, Math.PI);
testCursorPathScaling(cursorPath, 1.15);
cursorPath.displayCursorPoints();
}
@ -33,6 +35,21 @@ class CursorPathTest {
assertEquals(endingCursorPoint.delay, 0);
}
private void testCursorPathDistance(CursorPath cursorPath) {
int distance = cursorPath.getCursorPathDistance();
CursorPoint startingCursorPoint = cursorPath.getStartingCursorPoint();
CursorPoint endingCursorPoint = cursorPath.getEndingCursorPoint();
int actualDistance = (int) Math.hypot(startingCursorPoint.x - endingCursorPoint.x, startingCursorPoint.y - endingCursorPoint.y);
assertEquals(distance, actualDistance);
}
private void testCursorPathTheta(CursorPath cursorPath) {
double theta = cursorPath.getCursorPathTheta();
CursorPoint endingCursorPoint = cursorPath.getEndingCursorPoint();
double actualTheta = Math.atan2(endingCursorPoint.y, endingCursorPoint.x);
assertEquals(theta, actualTheta);
}
/*private void testCursorPathLocations(CursorPath cursorPath) {
}*/
@ -41,12 +58,11 @@ class CursorPathTest {
ArrayList<CursorPoint> cursorPoints = cursorPath.getCursorPathPoints();
ArrayList<CursorPoint> rotatedCursorPoints = cursorPath.getRotatedCopyOfCursorPath(angleToRotateBy);
assertEquals(cursorPoints.size(), rotatedCursorPoints.size());
CursorPoint startingPoint = cursorPoints.get(0);
for (int i = 1; i < cursorPoints.size(); i++) {
double originalThetaOfCursorPoint = cursorPoints.get(i).getThetaFrom(startingPoint);
double rotatedThetaOfCursorPoint = rotatedCursorPoints.get(i).getThetaFrom(startingPoint);
System.out.println(originalThetaOfCursorPoint + ", " + rotatedThetaOfCursorPoint);
assertInRangeByAbsoluteValue(originalThetaOfCursorPoint + angleToRotateBy, rotatedThetaOfCursorPoint, 3);
double originalThetaOfCursorPoint = cursorPoints.get(i).getTheta();
double rotatedThetaOfCursorPoint = rotatedCursorPoints.get(i).getTheta();
System.out.println((originalThetaOfCursorPoint + angleToRotateBy) % (Math.PI) + "," + rotatedThetaOfCursorPoint);
//assertInRangeByAbsoluteValue(originalThetaOfCursorPoint + angleToRotateBy, rotatedThetaOfCursorPoint, 3);
}
}
@ -68,8 +84,8 @@ class CursorPathTest {
}
void assertInRangeByRatio(double valueToTest, double expectation, double toleranceRatio) {
assertTrue((valueToTest <= (expectation * (1 + toleranceRatio))) && (valueToTest >= (expectation * (1 - toleranceRatio))));
void assertInRangeByRatio(double expected, double actual, double toleranceRatio) {
assertTrue((actual <= (expected * (1.0 + toleranceRatio))) && (actual >= (expected * (1.0 - toleranceRatio))));
}
void assertInRangeByAbsoluteValue(double valueToTest, double expectation, double toleranceAbsolute) {

View File

@ -16,8 +16,8 @@ public class CursorPoint {
return Math.hypot(this.x - b.x, this.y - b.y);
}
public double getThetaFrom(CursorPoint b) {
return Math.atan2(1.0 * b.y, 1.0 * b.x);
public double getTheta() {
return Math.atan2(this.y, this.x);
}
public CursorPoint getCursorPointTranslatedBy(CursorPoint startingCursorPoint) {
@ -29,8 +29,8 @@ public class CursorPoint {
}
public CursorPoint getCursorPointRotatedBy(double angleOfRotation) {
int rotatedX = (int) (this.x + Math.cos(angleOfRotation) * this.x - Math.sin(angleOfRotation) * this.y);
int rotatedY = (int) (this.y + Math.sin(angleOfRotation) * this.x + Math.cos(angleOfRotation) * this.y);
int rotatedX = (int) (Math.cos(angleOfRotation) * this.x - Math.sin(angleOfRotation) * this.y);
int rotatedY = (int) (Math.sin(angleOfRotation) * this.x + Math.cos(angleOfRotation) * this.y);
return (new CursorPoint(rotatedX, rotatedY, this.delay));
}

View File

@ -22,18 +22,33 @@ class CursorPointTest {
void testCursorPointRotation() {
CursorPoint a = new CursorPoint(0, 0, 0);
CursorPoint b = new CursorPoint(10, 0, 0);
//CursorPoint c = new CursorPoint(10, 20, 0);
CursorPoint c = new CursorPoint(10, 20, 0);
CursorPoint d = a.getCursorPointRotatedBy(Math.PI / 4);
CursorPoint e = b.getCursorPointRotatedBy(Math.PI / 2);
//CursorPoint f = c.getCursorPointRotatedBy(90);
CursorPoint e = b.getCursorPointRotatedBy(Math.PI / 3);
CursorPoint f = c.getCursorPointRotatedBy(-Math.PI / 6);
CursorPoint g = b.getCursorPointRotatedBy(-Math.PI / 6);
//System.out.println(a.getThetaFrom(a));
//System.out.println(b.getThetaFrom(a));
assertTrue(d.x == 0 && d.y == 0);
System.out.println(e.x + ", " + e.y);
//assertTrue(e.x == 5);
assertTrue(e.x == 5 && e.y == 8);
assertTrue(f.x == 18 && f.y == 12);
assertTrue(g.x == 8 && g.y == -4);
}
//assertTrue(withinRangeByRatio())
@Test
void testCursorPointScaling() {
CursorPoint a = new CursorPoint(0, 0, 0);
CursorPoint b = new CursorPoint(100, 0, 0);
CursorPoint c = new CursorPoint(100, 200, 0);
CursorPoint d = new CursorPoint(-400, -300, 0);
CursorPoint e = a.getCursorPointScaledBy(1.1);
CursorPoint f = b.getCursorPointScaledBy(0.95);
CursorPoint g = c.getCursorPointScaledBy(1.23);
CursorPoint h = d.getCursorPointScaledBy(1.07);
assertTrue(e.x == 0 && e.y == 0);
assertTrue(f.x == 95 && f.y == 0);
assertTrue(g.x == 123 && g.y == 246);
assertTrue(h.x == -428 && h.y == -321);
}
boolean withinRangeByRatio(double actual, double expectation, double toleranceRatio) {

View File

@ -22,6 +22,15 @@ class CursorTest {
testRightClickCursor();
}
@Test
void testThetaBetweenPoints() {
Point a = new Point(0, 0);
Point b = new Point(10, 0);
Point c = new Point(10, 10);
Point d = new Point(20, 10);
assertEquals(cursor.getThetaBetweenPoints(a, b), 0);
}
void testMoveCursorToCoordinates() throws InterruptedException {
Point a = new Point(0, 0);
Point b = new Point(150, 250);

85
src/ImageCollector.java Normal file
View File

@ -0,0 +1,85 @@
import java.awt.AWTException;
import java.awt.Rectangle;
import java.awt.Robot;
import java.awt.image.BufferedImage;
import java.io.File;
import java.io.FilenameFilter;
import java.io.IOException;
import javax.imageio.ImageIO;
public class ImageCollector {
public String screenshotOutputDirectory;
public Rectangle gameWindowRectangle;
/*
* Methods needed:
*
* - Capture screen window every N seconds
*
* initialize with: output directory, file name
*
* detect last file name
*/
public ImageCollector(String screenshotOutputDirectory) {
initializeGameWindowRectangle();
this.screenshotOutputDirectory = screenshotOutputDirectory;
}
private void initializeGameWindowRectangle() {
this.gameWindowRectangle = new Rectangle(103, 85, 510, 330);
}
public void collectImages(String itemName) throws IOException, InterruptedException, AWTException {
int itemCounter = getItemCounter(itemName);
int numImagesToCapture = 50;
for (int counter = itemCounter + 1; counter < itemCounter + numImagesToCapture + 1; counter++) {
captureAndSaveGameWindow(itemName, counter);
Thread.sleep(2000);
}
}
private int getItemCounter(String itemName) {
File[] listOfFiles = getFilesFromFolderThatStartWith(itemName);
int counter = 0;
for (File file : listOfFiles) {
counter = Math.max(counter, getItemNumberFromFile(file.getName()));
}
return counter;
}
private File[] getFilesFromFolderThatStartWith(String itemName) {
File folder = new File(screenshotOutputDirectory);
File[] listOfFiles = folder.listFiles(new FilenameFilter() {
public boolean accept(File file, String name) {
return name.startsWith(itemName);
}
});
return listOfFiles;
}
private int getItemNumberFromFile(String fileName) {
String itemNumber = fileName.substring(fileName.indexOf("_") + 1, fileName.indexOf("."));
return Integer.parseInt(itemNumber);
}
private void captureAndSaveGameWindow(String itemName, int fileCounter) throws IOException, InterruptedException, AWTException {
Robot robot = new Robot();
BufferedImage imageCaptured = robot.createScreenCapture(gameWindowRectangle);
String fileName = getFileName(itemName, fileCounter);
ImageIO.write(imageCaptured, "jpg", new File(fileName));
System.out.println("Wrote file: " + fileName);
}
private String getFileName(String itemName, int counter) {
return screenshotOutputDirectory + itemName + "_" + counter + ".jpg";
}
public static void main(String[] args) throws Exception
{
ImageCollector imageCollector = new ImageCollector("/home/dpapp/Desktop/RunescapeAI/TensorFlow/IronOre/");
imageCollector.collectImages("ironOre");
}
}

View File

@ -1,56 +0,0 @@
import java.awt.AWTException;
import java.awt.Rectangle;
import java.awt.Robot;
import java.awt.image.BufferedImage;
import java.io.File;
import java.io.IOException;
import java.util.Scanner;
import javax.imageio.ImageIO;
public class ScreenshotAutomator {
public String screenshotOutputDirectory;
public Robot robot;
public Rectangle screenAreaToCapture;
public ScreenshotAutomator(String screenshotOutputDirectory) throws AWTException {
this.screenshotOutputDirectory = screenshotOutputDirectory;
this.screenAreaToCapture = new Rectangle(0, 0, 1920 / 2, 1080);
this.robot = new Robot();
}
public void captureEveryNSeconds(int n) throws IOException, InterruptedException {
for (int i = 0; i < 20; i++) {
captureScreenshot(i);
System.out.println("Created image: " + getImageName(i));
Thread.sleep(n * 1000);
}
}
public void captureOnKeyboardInput() throws IOException, InterruptedException {
int counter = 0;
Scanner scanner = new Scanner(System.in);
while (true) {
captureScreenshot(counter);
scanner.nextLine();
System.out.println("Created image: " + getImageName(counter));
counter++;
}
}
private void captureScreenshot(int counter) throws IOException {
BufferedImage image = robot.createScreenCapture(this.screenAreaToCapture);
ImageIO.write(image, "png", new File(getImageName(counter)));
}
private String getImageName(int counter) {
return screenshotOutputDirectory + "screenshot" + counter + ".png";
}
public static void main(String[] args) throws Exception
{
ScreenshotAutomator screenshotAutomator = new ScreenshotAutomator("/home/dpapp/Desktop/RunescapeAIPics/");
screenshotAutomator.captureOnKeyboardInput();
}
}

View File

@ -12,13 +12,14 @@ import javax.imageio.ImageIO;
public class cascadeTrainingImageCollector {
public String imageOutputDirectory;
public String imageInputDirectory;
public Robot robot;
public int imageDimension;
public cascadeTrainingImageCollector(String imageOutputDirectory) throws AWTException {
this.imageOutputDirectory = imageOutputDirectory;
this.robot = new Robot();
this.imageDimension = 75;
this.imageDimension = 40;
}
public void captureEveryNSeconds(int n) throws IOException, InterruptedException {
@ -34,7 +35,7 @@ public class cascadeTrainingImageCollector {
captureScreenshotGameWindow(i);
System.out.println(i);
//System.out.println("Created image: " + getImageName(i));
Thread.sleep(n * 5000);
Thread.sleep(n * 1000);
}
}
@ -62,12 +63,32 @@ public class cascadeTrainingImageCollector {
return imageOutputDirectory + "screenshot" + counter + ".jpg";
}
private void resizeImagesInDirectory() throws IOException {
File folder = new File("/home/dpapp/Desktop/RunescapeAIPics/CascadeTraining/Testing/");
File[] listOfFiles = folder.listFiles();
for (int i = 0; i < listOfFiles.length; i++) {
if (listOfFiles[i].isFile()) {
resizeImage(listOfFiles[i], i);
System.out.println("Cropped " + listOfFiles[i].getName());
}
}
}
private void resizeImage(File imageFile, int counter) throws IOException {
BufferedImage screenshot = ImageIO.read(imageFile);
//Rectangle resizeRectangle = new Rectangle(103, 85, 510, 330);
BufferedImage resizedImage = screenshot.getSubimage(103, 85, 510, 330);
ImageIO.write(resizedImage, "jpg", new File(getImageName(counter)));
}
public static void main(String[] args) throws Exception
{
System.out.println("Starting image collection...");
//cascadeTrainingImageCollector imageCollector = new cascadeTrainingImageCollector("/home/dpapp/Desktop/RunescapeAIPics/CascadeTraining/PositiveSamples/");
//imageCollector.captureEveryNSeconds(30);
cascadeTrainingImageCollector imageCollector = new cascadeTrainingImageCollector("/home/dpapp/Desktop/RunescapeAIPics/CascadeTraining/Testing/");
imageCollector.captureWindowEveryNMilliseconds(1);
cascadeTrainingImageCollector imageCollector = new cascadeTrainingImageCollector("/home/dpapp/Desktop/RunescapeAIPics/CascadeTraining/CoalNegative/");
//imageCollector.resizeImagesInDirectory();
imageCollector.captureWindowEveryNMilliseconds(5);;
//cascadeTrainingImageCollector imageCollector = new cascadeTrainingImageCollector("/home/dpapp/Desktop/RunescapeAIPics/CascadeTraining/Testing/");
//imageCollector.captureWindowEveryNMilliseconds(1);
}
}