From d38d4afe542d25465897bccc457609cb3a29b779 Mon Sep 17 00:00:00 2001 From: davpapp Date: Fri, 23 Feb 2018 17:04:28 -0500 Subject: [PATCH] Started writing tests for object detection --- src/DetectedObject.java | 2 +- src/IronMiner.java | 20 +++++++++++++++----- src/ObjectDetector.java | 5 +++-- src/ObjectDetectorTest.java | 15 +++++++++++++++ target/classes/DetectedObject.class | Bin 2349 -> 2349 bytes target/classes/IronMiner.class | Bin 3549 -> 5337 bytes target/classes/ObjectDetector.class | Bin 5822 -> 5889 bytes target/classes/ObjectDetectorTest.class | Bin 0 -> 1089 bytes 8 files changed, 34 insertions(+), 8 deletions(-) create mode 100644 src/ObjectDetectorTest.java create mode 100644 target/classes/ObjectDetectorTest.class diff --git a/src/DetectedObject.java b/src/DetectedObject.java index f1b97aa..fc334c3 100644 --- a/src/DetectedObject.java +++ b/src/DetectedObject.java @@ -41,7 +41,7 @@ public class DetectedObject { return boundingBox; } - private Rect2d getBoundingRect2d() { + public Rect2d getBoundingRect2d() { return new Rect2d(boundingBox.x, boundingBox.y, boundingBox.x + boundingBox.width, boundingBox.y + boundingBox.height); } diff --git a/src/IronMiner.java b/src/IronMiner.java index 433b838..3c79502 100644 --- a/src/IronMiner.java +++ b/src/IronMiner.java @@ -3,12 +3,14 @@ import java.awt.Point; import java.awt.Rectangle; import java.awt.Robot; import java.awt.image.BufferedImage; +import java.awt.image.DataBufferByte; import java.io.File; import java.io.IOException; import java.util.ArrayList; import javax.imageio.ImageIO; +import org.opencv.core.Mat; import org.opencv.core.Rect2d; import org.opencv.tracking.Tracker; import org.opencv.tracking.TrackerKCF; @@ -46,18 +48,19 @@ public class IronMiner { DetectedObject closestIronOre = getClosestObjectToCharacter(ironOres); if (closestIronOre != null) { Tracker objectTracker = TrackerKCF.create(); - objectTracker.init(screenCapture, closestIronOre.getBoundingRect2d()); + Rect2d boundingBox = closestIronOre.getBoundingRect2d(); + objectTracker.init(getMatFromBufferedImage(screenCapture), boundingBox); + cursor.moveAndLeftClickAtCoordinatesWithRandomness(closestIronOre.getCenterForClicking(), 10, 10); long mineStartTime = System.currentTimeMillis(); int maxTimeToMine = randomizer.nextGaussianWithinRange(3500, 5000); - + // track until either we lose the object or too much time passes while ((System.currentTimeMillis() - mineStartTime) < maxTimeToMine) { screenCapture = objectDetector.captureScreenshotGameWindow(); - objectTracker.update(screenCapture, boundingBox); - Rectangle newBounds = new Rectangle(); - if (!objectDetector.isObjectPresentInBoundingBoxInImage(screenCapture, newBounds, "ironOre")) { + boolean ok = objectTracker.update(getMatFromBufferedImage(screenCapture), boundingBox); + if (!ok || !objectDetector.isObjectPresentInBoundingBoxInImage(screenCapture, boundingBox, "ironOre")) { System.out.println("Lost track! Finding new ore."); break; } @@ -101,6 +104,13 @@ public class IronMiner { return null; } + public Mat getMatFromBufferedImage(BufferedImage image) { + Mat matImage = new Mat(); + byte[] pixels = ((DataBufferByte) image.getRaster().getDataBuffer()).getData(); + matImage.put(0, 0, pixels); + return matImage; + } + public int getDistanceBetweenPoints(Point startingPoint, Point goalPoint) { return (int) (Math.hypot(goalPoint.x - startingPoint.x, goalPoint.y - startingPoint.y)); } diff --git a/src/ObjectDetector.java b/src/ObjectDetector.java index d40e0d0..f8248aa 100644 --- a/src/ObjectDetector.java +++ b/src/ObjectDetector.java @@ -32,6 +32,7 @@ import java.util.List; import java.util.Map; import javax.imageio.ImageIO; +import org.opencv.core.Rect2d; import org.tensorflow.SavedModelBundle; import org.tensorflow.Tensor; import org.tensorflow.types.UInt8; @@ -92,8 +93,8 @@ public class ObjectDetector { return detectedObjectsInImage; } - public boolean isObjectPresentInBoundingBoxInImage(BufferedImage image, Rectangle boundingBox, String objectClass) throws Exception { - BufferedImage subImage = image.getSubimage(boundingBox.x, boundingBox.y, boundingBox.width, boundingBox.height); + public boolean isObjectPresentInBoundingBoxInImage(BufferedImage image, Rect2d boundingBox, String objectClass) throws Exception { + BufferedImage subImage = image.getSubimage((int) boundingBox.x, (int) boundingBox.y, (int) boundingBox.width, (int) boundingBox.height); ArrayList detectedObjectsInSubImage = getObjectsInImage(subImage); return (getObjectsOfClassInList(detectedObjectsInSubImage, objectClass).size() != 0); } diff --git a/src/ObjectDetectorTest.java b/src/ObjectDetectorTest.java new file mode 100644 index 0000000..77a3039 --- /dev/null +++ b/src/ObjectDetectorTest.java @@ -0,0 +1,15 @@ +import static org.junit.jupiter.api.Assertions.*; + +import java.awt.AWTException; + +import org.junit.jupiter.api.Test; + +class ObjectDetectorTest { + + @Test + void testObjectDetection() throws AWTException { + ObjectDetector objectDetector = new ObjectDetector(); + ArrayList detectedObjects1 = objectDetector.getObjectsInImage(loadimages here in bufferedimage format)); + + } +} diff --git a/target/classes/DetectedObject.class b/target/classes/DetectedObject.class index 0f50857982ab31a714d2eb698f52fbf1e6717532..b4dce75e061bf1009142d8001f3c88c86bf32377 100644 GIT binary patch delta 14 VcmZ20v{q=tNp?oY&8OIt838E?1nB?( delta 14 VcmZ20v{q=tNp?o2&8OIt838E|1nK|) diff --git a/target/classes/IronMiner.class b/target/classes/IronMiner.class index b5189dd7dc95d903771cde2d87dfbf3f1182a835..f54e8e701ea0e2e764e4e0714d2321fc946e7754 100644 GIT binary patch literal 5337 zcma)A33Oal8UF4hnU_pnLlT#?7D&?2tYzAnq$ES8v`BcFyd;lK=7l#e zZGsD;vK$3O!G)rzC>8~kEon;?L9r?xj|=YNhP&dvf=K-CeKV7p5ICH+^X~dD-~avp zy|+)k_{gIGHj6bHd;$4-Q;gz0F=7bv~L*l)DNjl@LDz~~ib%oV5&4i5B>3=H;; zgoFLT{_P`?U|6<8q2N$&_dtKokbnpZtm?KCN!LiY!$y3@%mWJQ1lET;cL&24hevvX zLy^w@?%t8ez(}++v{Qc5g1J`QB2d1)lY*nc{+@y8NOy04q<4@8g_K3c0k>tBZ6(|` z0e@`9N!pG;VW?ZRI2Q26B1UqbfSwwcJj4QlCHdumTXT`*WbTm*sBbreGvI3dI!_0LR=noh(Rjhn+gvRi@!nTY`M&?Jqi*06#0cmM~` zj3#N!SvppsM8ny1-kXvb2e8ST-lF3KlxjFn@+ysU0OxwMZ92+Ou3;-7$b~t8c5lfx z9VcS7hOTUs6x;!HdP{nBRG?DBcABt+cmRFg^bQ>-VGZGSC#|H4cbeY}N}e;5&@hm8 zUC}(t^>$pOV-V!?$uVQvopH>em}8oWS*}#c&@GsHv-8Vx=`@OqeZQGvni? zV~z!7uTA=Mv5sLx1)QOlk5GB9B#nQQB_SnbJT4O=35TKqM5i-tGn95q98 zN!Ogx@m5^RFtL<7G&N#PnPDp)mubury2HQi-rYR70ROg!Zcq)G^l#U3BW_}Z66P$E zbS9a!jD)nzO3=0mGN+)fr>8zB5r2n{TO^?Vndvb)AXDgFI&Q^ntg=?pW9Tl&Oj3C; zk!DJlJ)6nfomn2Pl=m+eixu*1PZ6%cdvv@P@1uM639Uq9K*tC1A=Zs)DKnm+nv@ec=|3#l zewU7qpqgUF9D6!bzk}m_Gx0dj_oOO@-Ckv60xyMjB`kPijj3}ro%;-9%EM*Vh=*mW zsBuY2%&Rf-TEa3X&)wW)W*J{)t7AJ=ECf!;#(jAbaeeVByvlE_zr+CH= zxrU?2$7{_FIcCgCDn@L1OA_$eykUqv0+qSP@7#3y2tLyTCfCdM#Rk z$=HM^bbJxVdCJ+mM##&}SP$vY8IK2xUKkVOj^R84$xaWyfN%#i7rQw@7}EV zrFkh;y((NEl&1Vz$8RJTv`Hh`ue4UDvM((-=Zs&In)C`~&QwWmstZ(kDY}9j4Sytu zms6F8VgS#gDS*THi;lnIZzKgh95i?`@P1L7hC}+~Wk%GoTzT>JmJpkN;9na4$#SqP zg>o0__&5H;)RXWTuHhZ$-1JffOVe!{o`0#_rTMNod73dSY6`tn7fzP$FUFYos1%S$}Jg-TX6C7;aRS;$v&Wyi4gk}@oo;qJpY^&zZ#fWtnl z=dUyZ6mfbs8^zHqTzDo_J+e~Y&eGIOAe!D9vj)*#O68k*v#g+ zY@R=lRyNz&?3l*|Y<9BQJ&ztX``Em29>FC8*3n|%G+J8Ax6nE&Kb^*`$436P(TqNt zAdS;K_1K9JdjjE%nfdJXOC!ZIM75Z+4>mlE3l^{|B)^Rdh=gUMseJbW_VhP2AI4>^ z`BnMHaorJ&wiZ<7FJQ9OUsce+rHLy40-PwqC+@O%9V<;Hs8D@P!YHt<#MXA$}g3^^B_td#JkIHU%(xQP)b>qnpshF?QM4~ zK2AJ8d}#3^TAi-~GN|I(L{ysj`rE*_?nWYV7V3D?G~;ZX!#|za!e`fch~j+S`rCPJ z-HI#Gfva%=Z)e+Z2Upz5@q3Bhr_reJ8%IDQlfthMhwxF{P2|eB@?-4zsOeVR!&V;k z-H4Cl6SSzA_u6}LA8|bg2k=RJiiC+GLaFzYyW{A^0~{%$=F=$qA?kRTS}G~clRKZ0 zJ5RX_oa>;hMNFbdgV1nT!%_9u(!lrkv3zpu1=Ne@5m;o%3zjY7exJN@@mCTw6M4S4 zJbKm=_0|P^*86Om61+07|0@8Z3-x9GDX)=Z=Nl;zt6H!D`KKco6GU3M}B~Tl^LNwRfSY!oRgD zzhAEUmB7Kpr<#vLUt1<_xeEo&_lx^84sB3rxR**UrE8bbp%D`Ja?;r#l}AaY%SZ$h zK~iWZ$siNFgD%majH6*iOIa{?;J2LfQPPF@9bL|&ylVVj5wwmh`2%PDSun_*X&CB} zx5x_WS428W##!|HY2b`SvNS(yRMr$eUHL03r_0Dldej`lvn*ME3KOC~FW~Qw;kgC; zx4$VbuXzEB(I)A#S4cfN#fkeAB`0tyY~@yiCnw`pjmT4ZSIGZ#$rlCOU5SA3(=y3m z-c(W+M7d8gs;cQ>QP{|TW7o4)%vSN(^+!a>&G%(gdo?_nMVG3Kx+<|cRXVFg1zSD{ KArB0nIO+dglTUsC delta 1521 zcmY*YTT>iG7(LA{Gcb(Y1ayH^1_HPXlt9X06;v=1O$7{ua0w7%;_Pm-4B4GoXJ;w! zwlNxsm!Q(#5>3>2A!-sOQk$q%#s@18zWd;VkNyK6E%BQHxjb~w*ZtkjIbY8op+BpX z-~RpaX8?P!vaA&OGBS=Z6iw?hdP`h4V=a%{wq@sGh%sfvw0X*k&+w=gu@Xrmt~-Wh zYDwD)$9W>vp(**pQ(Q}MXUd9dhG{s#KHX{726@EU9SzZq=8)f+OLA@4)+495tr;oJ zv>Z)OCgVm#563fHo3v~%PPb#}1UH?O7VJ(>PI8+^dlGt#Zxr0tF2$vGrDaM zO0Fmvz=#SBN5!gqzo^NpctpX&7*}x&l?+lk8Py%`UL9BQBuEa9bxCRJ@1x#pQxp>4J)j_)y#}*k31bNyTMX zFnLN(_48SWq>FAFT}o7`ydxjg_(H{(kQmskd%CLPEAKR?;2T_*agAi!WJYgsO2rL) zyI6Rg$>K&~J1Y}^6`tO)1Agp85e>5vHK<1|TCoeAs3WoJ#hIdD2Offt#sDZkVPL#; z@D_&e&?$#PNL9zKT1UYtwSg}D)U+RyWaS`+D*9eTF8P4w#->}C%Jeng zMV;HVIg5C)QmkZQkJ7oP3^Qez&ElD}xAB6D#qVLMpmfj!mv96M21%A7G-1TUqpcNQ z$3rdfGR_hJiJ-iKd4h0`a@TZleXoLXxUloU{y~$B1clIJrhN_ZYc66|N@Zn=>MMPH zFY9m0;*E~nC4>UGKVXsJ7bH6TjW3$g6vuzJvvuA)x-SkysEuMfZ5>N13DZVN1+`GMvehOU9oosXP3=Hw3(5x& zXpD*kbEAQXA0&~wR2EyO(rN)$EGmlohT^`9xS>Mz+*ji#eJ}6KJ^TOs@0roEo6obx z#)n6N3;1h~mF-va^_}*#{JKPEJe*h->uBl;Hzhh_;RUg#t{Kg`-ajpx_uoF7``MvT ztK^&GXFCtrJm^?%G}^;9{YLYwY;EppF_xV+yNt!#5^IgObXja;kIkbD=!5=Q8IReN zP->wVvU!}nI@7wYYQ*LVo>aJ3wB8g`xQdEv6@jIR?vCcxj(AN%dY|!s(EF@G?{hZK zb5KVE)4gN7=;M&SBarWUiI;U>pwRn@&8r;IhXdi@>o#w&Ua-ZMcSqY-1Q_Ejo40vK z=iAq-5A`a0S>DHr^tNbEG~6ETh==EPb|yMg`BabFz4kFaxAF6ZBDY9i<v8o)Wu=6lP;lBhEQYW-e!P2?borxm-pmwUjfDnapPv7Zah5ZmwV> zS8^})>|}vZypTgI;wXzbK?5hbT6t(x1vIDMxECJMM%6)9} z5f-Pcl2j@aM+(pA)j7e?vk^}pg+b3=UQq03b;NDpA1+Pu+E%PGqpWQ(?l<*lPF_4t zDlI~9E5)>NzMLc6;0#bH9ijT)0jhadaG3$BrS1V~m`Ww@iOnVX*}TsOg5RLdbA~hV zTO{6z(p9+yi^gE8+-jx#J6$q}VNb~oO`!isScH&xObcs-V~ts#4Ds1M(NY|iy^d$O z1J1B#@g-mBUAeQ@tP`l~W#u*qdA;PbPIl!MC$2#1A?x)u-#B%qau=JWj$gWc%Xd=Q zkm@7l6dsi$j_>(Fj0T)nevirx7+4D(4W^86yfGu5u{cwWXR3KaoTv*8QeRk-3$g6vuye*7nWobfGO732ki@=(OWh1r5^%wXzk#rNts@B7=<6F-;4@R8SMK z0!9rVNNzL`-~*`OQWb2O0jjOL7F-c`S={%nZnzQ8eI%5_l2BM)iZ4R+ocqNu~ z$2*n>80Kx8cX(G%v=^x(`VM<(`A3REM<7Rx8R>LaI*(6uuwZT8F+Q`&=X1TOpr^u; zIe#?fPYVB}W_fDbEGqfg<`;g|Ex|TD6x?1=kB2iU=PXX;Y@IG_QVo5(3+K5K({OPf z78gkHLQY{iXCbG)@JJIbE!2qa~Rje9aSO4Qu6}n%k2M%5>&7#8b?6 z3oyjf67~qZm}j_C7~LZYwg5&w8^p6oJOR`{MEq>_3tE}=kV+8=xOi3%mWIO5#k@PH z4te+Tl9J*#$2^Aqt94lpZp4Zh|b zKFR*5s0d++m=@Lm#}<}+oa56Wv0RSIJ}0u&A!pRH_<}F>NZI+TdW7n2vTCbEyfuua zNA~1)C$CECA*=NjUpsXwxSRD-CttdK!?#k|i0a4nH(pgLf$#WUoQ9lFzK7)x7+S54 z1>*+Twy-E!ba}BlQLN_V_+f6ipG&K2vyAL?8klvk1c~TU!d(uDQ9Z_;RI)^x|0WBd UBeHD!jik)%icoE&+@5> zCi?D=GR_{f&{&KnKHToi&dfJ6-_88^`Q;k`JcEa2C@`qK>y0pSN9gD@wu6}~Ly5uW zNKRxBOFal?9t_r=E3MoM2F3k@E`!ob(?iBrDd(0hAML+l0K#Ap|}ikMo8hNDe_8ts+~L&=)oTgr8Bfn z>>cb0^6}U){RQy`-O%)3>j`n^OJ5b%yc%|e>LZL|Y4h+l5r7G5$1AaseU-=oW-IHQ z%>Nay4AJ6Bw?DP>?{!^I%f;AS^kQuGcOCyf!g}MA6QW^8+BIZw4t3b|~#Ln_)&(`gUj{*U-SG1P^1^Q^8NH3EV(9QhIqjOmQoSQkF z-VEe$lg{eQf$m$R0o*2Q5jIFFk!2HVBzgZWx;^AmvifNL73!aAchBM8chU;5z^yLf M@;NSp2f0_{7bllF@Bjb+ literal 0 HcmV?d00001