diff --git a/bellatrix.playwright/src/main/java/solutions/bellatrix/playwright/components/ActionImage.java b/bellatrix.playwright/src/main/java/solutions/bellatrix/playwright/components/ActionImage.java index 7beb8c1b..847bc766 100644 --- a/bellatrix.playwright/src/main/java/solutions/bellatrix/playwright/components/ActionImage.java +++ b/bellatrix.playwright/src/main/java/solutions/bellatrix/playwright/components/ActionImage.java @@ -32,7 +32,7 @@ public Point getLocation() { var encodedImage = findStrategy.getEncodedImage(); - var location = OpenCvService.getLocation(encodedImage, true); + var location = OpenCvService.getLocation(encodedImage); return new Point((int)location.x + encodedImage.getXOffset(), (int)location.y + encodedImage.getYOffset()); } diff --git a/bellatrix.playwright/src/main/java/solutions/bellatrix/playwright/findstrategies/ImageBase64FindStrategy.java b/bellatrix.playwright/src/main/java/solutions/bellatrix/playwright/findstrategies/ImageBase64FindStrategy.java index 58d03431..7ccc69de 100644 --- a/bellatrix.playwright/src/main/java/solutions/bellatrix/playwright/findstrategies/ImageBase64FindStrategy.java +++ b/bellatrix.playwright/src/main/java/solutions/bellatrix/playwright/findstrategies/ImageBase64FindStrategy.java @@ -16,14 +16,11 @@ import com.microsoft.playwright.Locator; import com.microsoft.playwright.Page; import lombok.Getter; -import solutions.bellatrix.core.utilities.SingletonFactory; import solutions.bellatrix.core.utilities.parsing.TypeParser; import solutions.bellatrix.playwright.components.common.webelement.WebElement; -import solutions.bellatrix.playwright.services.JavaScriptService; import solutions.bellatrix.plugins.opencv.Base64Encodable; import solutions.bellatrix.plugins.opencv.OpenCvService; -import java.awt.*; import java.util.ArrayList; import java.util.List; @@ -37,7 +34,7 @@ public ImageBase64FindStrategy(Base64Encodable encodedImage) { @Override public WebElement convert(Page page) { - var location = OpenCvService.getLocation(encodedImage, false); + var location = OpenCvService.getLocation(encodedImage); var foundLocators = findElementsOn(location, page); @@ -48,7 +45,7 @@ public WebElement convert(Page page) { @Override public WebElement convert(WebElement webElement) { - var location = OpenCvService.getLocation(encodedImage, false); + var location = OpenCvService.getLocation(encodedImage); var foundLocators = findElementsOn(location, webElement.page()); diff --git a/bellatrix.plugins.opencv/src/main/java/solutions/bellatrix/plugins/opencv/OpenCvService.java b/bellatrix.plugins.opencv/src/main/java/solutions/bellatrix/plugins/opencv/OpenCvService.java index 705ca8e9..ea235542 100644 --- a/bellatrix.plugins.opencv/src/main/java/solutions/bellatrix/plugins/opencv/OpenCvService.java +++ b/bellatrix.plugins.opencv/src/main/java/solutions/bellatrix/plugins/opencv/OpenCvService.java @@ -8,6 +8,8 @@ import org.opencv.imgcodecs.Imgcodecs; import org.opencv.imgproc.Imgproc; import plugins.screenshots.ScreenshotPlugin; +import solutions.bellatrix.core.configuration.ConfigurationService; +import solutions.bellatrix.core.utilities.Log; import solutions.bellatrix.core.utilities.SingletonFactory; import javax.imageio.ImageIO; @@ -39,16 +41,24 @@ public static double getJavaMonitorScaling() { return scaleX; } + public static Point getLocation(Base64Encodable encodedImage) { + OpenCvServiceSettings openCvServiceSettings = ConfigurationService.get(OpenCvServiceSettings.class); + if (openCvServiceSettings == null) { + openCvServiceSettings = new OpenCvServiceSettings(); + } + + var precisionThreshold = openCvServiceSettings.getDefaultMatchThreshold(); + var shouldGrayscale = openCvServiceSettings.isShouldGrayscale(); + return getLocation(encodedImage, shouldGrayscale, precisionThreshold); + } /** * @return the coordinates of the image found on the screen */ - public static Point getLocation(Base64Encodable encodedImage, boolean shouldGrayScale) { + public static Point getLocation(Base64Encodable encodedImage, boolean shouldGrayScale, double precisionThreshold) { var screenshotPlugin = SingletonFactory.getInstance(ScreenshotPlugin.class); - + Log.info("Locating image using precision: %s; Should Grayscale = %s".formatted(precisionThreshold, shouldGrayScale)); if (screenshotPlugin == null) { - throw new IllegalArgumentException("It seems that the screenshot plugin isn't registered by the 'ScreenshotPlugin.class' key inside SingletonFactory's map or isn't registered at all!\n" + - "Check the BaseTest class of your project where the plugins are registered. Register the specific screenshot plugin implementation as the base ScreenshotPlugin.class.\n" + - "for example: addPluginAs(ScreenshotPlugin.class, WebScreenshotPlugin.class);"); + throw new IllegalArgumentException("Screenshot plugin not registered!"); } var screenshot = screenshotPlugin.takeScreenshot(); @@ -57,13 +67,17 @@ public static Point getLocation(Base64Encodable encodedImage, boolean shouldGray Mat result = loadImages(encodedImage, screenshot, shouldGrayScale); - return getMatchLocation(encodedImage, result); + return getMatchLocation(encodedImage, result, precisionThreshold); } - private static Point getMatchLocation(Base64Encodable encodedImage, Mat result) { + private static Point getMatchLocation(Base64Encodable encodedImage, Mat result, double precisionThreshold) { BufferedImage bufferedImage; - Core.MinMaxLocResult mmr = Core.minMaxLoc(result); + if (mmr.maxVal < precisionThreshold) { + throw new RuntimeException("Match not found above threshold. Closest match at: %s".formatted(mmr.maxVal)); + } + Log.info("Image located with precision: %s".formatted(mmr.maxVal)); + Point matchLoc = mmr.maxLoc; if (encodedImage.getXOffset() == 0 && encodedImage.getYOffset() == 0) { @@ -73,13 +87,17 @@ private static Point getMatchLocation(Base64Encodable encodedImage, Mat result) throw new RuntimeException(e); } - double[] imageCenterCoordinates = {matchLoc.x / getJavaMonitorScaling() + (double)(bufferedImage.getWidth() / 2), matchLoc.y / getJavaMonitorScaling() + (double)(bufferedImage.getHeight() / 2)}; + double[] imageCenterCoordinates = { + matchLoc.x / getJavaMonitorScaling() + (double)(bufferedImage.getWidth() / 2), + matchLoc.y / getJavaMonitorScaling() + (double)(bufferedImage.getHeight() / 2) + }; matchLoc.set(imageCenterCoordinates); } return matchLoc; } + private static BufferedImage getImageWidthHeight(Base64Encodable encodedImage) throws IOException { String cleanBase64 = removePrefixFromBase64(encodedImage); diff --git a/bellatrix.plugins.opencv/src/main/java/solutions/bellatrix/plugins/opencv/OpenCvServiceSettings.java b/bellatrix.plugins.opencv/src/main/java/solutions/bellatrix/plugins/opencv/OpenCvServiceSettings.java new file mode 100644 index 00000000..7adaf94e --- /dev/null +++ b/bellatrix.plugins.opencv/src/main/java/solutions/bellatrix/plugins/opencv/OpenCvServiceSettings.java @@ -0,0 +1,12 @@ +package solutions.bellatrix.plugins.opencv; + +import com.google.gson.annotations.SerializedName; +import lombok.Getter; + +@Getter +public class OpenCvServiceSettings { + @SerializedName("defaultMatchThreshold") + private double defaultMatchThreshold = 0.8; + @SerializedName("shouldGrayscale") + private boolean shouldGrayscale = true; +} \ No newline at end of file diff --git a/bellatrix.web/src/main/java/solutions/bellatrix/web/components/ActionImage.java b/bellatrix.web/src/main/java/solutions/bellatrix/web/components/ActionImage.java index fee784ee..dfc5b4b1 100644 --- a/bellatrix.web/src/main/java/solutions/bellatrix/web/components/ActionImage.java +++ b/bellatrix.web/src/main/java/solutions/bellatrix/web/components/ActionImage.java @@ -30,7 +30,7 @@ public Point getLocation() { var encodedImage = findStrategy.getEncodedImage(); - var location = OpenCvService.getLocation(encodedImage, true); + var location = OpenCvService.getLocation(encodedImage); return new Point((int)location.x + encodedImage.getXOffset(), (int)location.y + encodedImage.getYOffset()); } diff --git a/bellatrix.web/src/main/java/solutions/bellatrix/web/findstrategies/ImageBase64FindStrategy.java b/bellatrix.web/src/main/java/solutions/bellatrix/web/findstrategies/ImageBase64FindStrategy.java index 6efa1658..60cea2c2 100644 --- a/bellatrix.web/src/main/java/solutions/bellatrix/web/findstrategies/ImageBase64FindStrategy.java +++ b/bellatrix.web/src/main/java/solutions/bellatrix/web/findstrategies/ImageBase64FindStrategy.java @@ -4,23 +4,39 @@ import org.openqa.selenium.By; import org.openqa.selenium.SearchContext; import org.openqa.selenium.WebElement; +import solutions.bellatrix.core.configuration.ConfigurationService; import solutions.bellatrix.core.utilities.Log; import solutions.bellatrix.core.utilities.SingletonFactory; import solutions.bellatrix.plugins.opencv.Base64Encodable; import solutions.bellatrix.plugins.opencv.OpenCvService; -import solutions.bellatrix.web.services.App; +import solutions.bellatrix.plugins.opencv.OpenCvServiceSettings; import solutions.bellatrix.web.services.JavaScriptService; import java.util.List; -import java.util.Objects; @Getter public class ImageBase64FindStrategy extends FindStrategy { private final Base64Encodable encodedImage; + private final boolean shouldGrayScale; + private final double precisionThreshold; public ImageBase64FindStrategy(Base64Encodable encodedImage) { super(encodedImage.getBase64Image()); this.encodedImage = encodedImage; + var serviceSettings = ConfigurationService.get(OpenCvServiceSettings.class); + if (serviceSettings == null) { + serviceSettings = new OpenCvServiceSettings(); + } + + this.shouldGrayScale = serviceSettings.isShouldGrayscale(); + this.precisionThreshold = serviceSettings.getDefaultMatchThreshold(); + } + + public ImageBase64FindStrategy(Base64Encodable encodedImage, boolean shouldGrayScale, double precisionThreshold) { + super(encodedImage.getBase64Image()); + this.shouldGrayScale = shouldGrayScale; + this.precisionThreshold = precisionThreshold; + this.encodedImage = encodedImage; } @Override @@ -46,7 +62,7 @@ public static By byImageBase64(Base64Encodable encodedImage) { @Override public List findElements(SearchContext context) { - var location = OpenCvService.getLocation(base64EncodedImage, false); + var location = OpenCvService.getLocation(base64EncodedImage); Log.info("Coordinates located: %s", location.toString()); return SingletonFactory.getInstance(JavaScriptService.class).>genericExecute("return document.elementsFromPoint(%s, %s);".formatted(location.x, location.y)); } diff --git a/bellatrix.web/src/main/java/solutions/bellatrix/web/services/ComponentCreateService.java b/bellatrix.web/src/main/java/solutions/bellatrix/web/services/ComponentCreateService.java index f94dd512..5941f00d 100644 --- a/bellatrix.web/src/main/java/solutions/bellatrix/web/services/ComponentCreateService.java +++ b/bellatrix.web/src/main/java/solutions/bellatrix/web/services/ComponentCreateService.java @@ -15,10 +15,8 @@ import solutions.bellatrix.core.utilities.InstanceFactory; import solutions.bellatrix.plugins.opencv.Base64Encodable; -import solutions.bellatrix.web.components.ActionImage; import solutions.bellatrix.web.components.WebComponent; import solutions.bellatrix.web.findstrategies.*; -import solutions.bellatrix.web.infrastructure.DriverService; import java.util.ArrayList; import java.util.List; @@ -42,6 +40,10 @@ public TComponent byImage(Class co return by(componentClass, new ImageBase64FindStrategy(encodedImage)); } + public TComponent byImage(Class componentClass, Base64Encodable encodedImage, boolean shouldGrayscale, double matchPrecision) { + return by(componentClass, new ImageBase64FindStrategy(encodedImage, shouldGrayscale, matchPrecision)); + } + public TComponent byAttributeContaining(Class componentClass, String attributeName, String value) { return by(componentClass, new AttributeContainingWithFindStrategy(attributeName, value)); } @@ -106,6 +108,10 @@ public List allByImage(Class List allByImage(Class componentClass, Base64Encodable encodedImage, boolean shouldGrayscale, double matchPrecision) { + return allBy(componentClass, new ImageBase64FindStrategy(encodedImage, shouldGrayscale, matchPrecision)); + } + public List allByAttributeContaining(Class componentClass, String attributeName, String value) { return allBy(componentClass, new AttributeContainingWithFindStrategy(attributeName, value)); } diff --git a/framework-tests/bellatrix.web.tests/src/main/resources/testFrameworkSettings.dev.json b/framework-tests/bellatrix.web.tests/src/main/resources/testFrameworkSettings.dev.json index 1ab43a8b..8cf9e838 100644 --- a/framework-tests/bellatrix.web.tests/src/main/resources/testFrameworkSettings.dev.json +++ b/framework-tests/bellatrix.web.tests/src/main/resources/testFrameworkSettings.dev.json @@ -110,6 +110,10 @@ "shopUrl": "http://demos.bellatrix.solutions/cart/", "accountUrl": "http://demos.bellatrix.solutions/account/" }, + "openCvServiceSettings": { + "shouldGrayscale": true, + "defaultMatchThreshold": "0.9" + }, "testPagesSettings": { "anchorLocalPage": "testpages/anchor/anchor.html", "buttonLocalPage": "testpages/button/button.html", diff --git a/framework-tests/bellatrix.web.tests/src/main/resources/testFrameworkSettings.qa.json b/framework-tests/bellatrix.web.tests/src/main/resources/testFrameworkSettings.qa.json index 2c46b262..0193aa20 100644 --- a/framework-tests/bellatrix.web.tests/src/main/resources/testFrameworkSettings.qa.json +++ b/framework-tests/bellatrix.web.tests/src/main/resources/testFrameworkSettings.qa.json @@ -110,6 +110,10 @@ "shopUrl": "http://demos.bellatrix.solutions/cart/", "accountUrl": "http://demos.bellatrix.solutions/account/" }, + "openCvServiceSettings": { + "shouldGrayscale": true, + "defaultMatchThreshold": "0.9" + }, "testPagesSettings": { "anchorLocalPage": "testpages/anchor/anchor.html", "buttonLocalPage": "testpages/button/button.html", diff --git a/framework-tests/bellatrix.web.tests/src/test/java/opencv/OpenCvTests.java b/framework-tests/bellatrix.web.tests/src/test/java/opencv/OpenCvTests.java index 04bd7cb0..b57a1275 100644 --- a/framework-tests/bellatrix.web.tests/src/test/java/opencv/OpenCvTests.java +++ b/framework-tests/bellatrix.web.tests/src/test/java/opencv/OpenCvTests.java @@ -22,7 +22,7 @@ public void beforeEach() throws Exception { @Test public void actionPerformed_when_convertBase64ToImage_and_clickImage() { - var falcon9Image = app().create().byImage(Anchor.class, EncodedImageDemo.FALCON_9); + var falcon9Image = app().create().byImage(Anchor.class, EncodedImageDemo.FALCON_9, false, 0.99); app().navigate().to("http://demos.bellatrix.solutions/"); falcon9Image.click();