DRAFT: Enable quack, first round

build.gradle

@@ -166,7 +166,8 @@ dependencies {
     implementation "com.blackduck.integration:blackduck-common:${blackDuckCommonVersion}"
     implementation 'com.blackduck.integration:blackduck-upload-common:4.1.3'
     implementation 'com.blackducksoftware:method-analyzer-core:1.0.1'
-    implementation "${locatorGroup}:${locatorModule}:2.2.0"
+//    implementation "${locatorGroup}:${locatorModule}:2.2.0"
+    implementation files ("/Users/shanty/blackduck/gitlab-folder/component-locator/build/libs/component-locator-2.1.2-quack-all.jar")
 
     implementation 'org.apache.maven.shared:maven-invoker:3.0.0'
 

src/main/java/com/blackduck/integration/detect/configuration/DetectConfigurationFactory.java

@@ -226,9 +226,20 @@ public Boolean isComponentLocationAnalysisEnabled() {
         return detectConfiguration.getValue(DetectProperties.DETECT_COMPONENT_LOCATION_ANALYSIS_ENABLED);
     }
 
+    public Boolean isQuackPatchPossible() {
+        return isComponentLocationAnalysisEnabled()
+                && !detectConfiguration.getValue(DetectProperties.DETECT_LLM_NAME).toString().isEmpty()
+                && !detectConfiguration.getValue(DetectProperties.DETECT_LLM_API_ENDPOINT).toString().isEmpty()
+                && !detectConfiguration.getValue(DetectProperties.DETECT_LLM_API_KEY).toString().isEmpty();
+    }
+
     public Boolean doesComponentLocatorAffectStatus() {
         return detectConfiguration.getValue(DetectProperties.DETECT_COMPONENT_LOCATION_ANALYSIS_STATUS);
-    }    
+    }
+
+    public DetectPropertyConfiguration getDetectPropertyConfiguration() {
+        return detectConfiguration;
+    }
 
     public DetectToolFilter createToolFilter(RunDecision runDecision, BlackDuckDecision blackDuckDecision, Map<DetectTool, Set<String>> scanTypeEvidenceMap) {
         Optional<Boolean> impactEnabled = Optional.of(detectConfiguration.getValue(DetectProperties.DETECT_IMPACT_ANALYSIS_ENABLED));

src/main/java/com/blackduck/integration/detect/configuration/DetectProperties.java

@@ -78,6 +78,18 @@ public class DetectProperties {
     private DetectProperties() {
     }
 
+    public static final StringProperty DETECT_LLM_API_KEY =
+            StringProperty.newBuilder("detect.llm.api.key", "")
+                    .build();
+
+    public static final StringProperty DETECT_LLM_API_ENDPOINT =
+            StringProperty.newBuilder("detect.llm.api.endpoint", "")
+                    .build();
+
+    public static final StringProperty DETECT_LLM_NAME =
+            StringProperty.newBuilder("detect.llm.name", "")
+                    .build();
+
     public static final NullableStringProperty BLACKDUCK_API_TOKEN =
         NullableStringProperty.newBuilder("blackduck.api.token")
             .setInfo("Black Duck SCA API Token", DetectPropertyFromVersion.VERSION_4_2_0)

src/main/java/com/blackduck/integration/detect/lifecycle/run/operation/OperationRunner.java

@@ -375,7 +375,7 @@ public final DetectorToolResult executeDetectors() throws OperationException {
                 directoryEvaluator
             );
             return detectorTool.performDetectors(
-                directoryManager.getSourceDirectory(),
+                directoryManager,
                 detectRuleSet,
                 detectConfigurationFactory.createDetectorFinderOptions(),
                 detectorToolOptions.getProjectBomTool(),
@@ -724,7 +724,42 @@ public List<DeveloperScansScanView> waitForRapidResults(BlackDuckRunData blackDu
             int fibonacciSequenceIndex = getFibonacciSequenceIndex();
 
             try {
-                return new RapidModeWaitOperation(blackDuckServicesFactory.getBlackDuckApiClient()).waitForScans(
+                return new RapidModeWaitOperation(blackDuckServicesFactory.getBlackDuckApiClient()).waitForRegularScans(
+                        rapidScans, // url is not full here
+                        detectConfigurationFactory.findTimeoutInSeconds(),
+                        RapidModeWaitOperation.DEFAULT_WAIT_INTERVAL_IN_SECONDS,
+                        mode,
+                        calculateMaxWaitInSeconds(fibonacciSequenceIndex)
+                );
+            } catch (InterruptedException e) {
+                throw e;
+            } catch (IntegrationRestException e) {
+                throw handleRapidScanException(e);
+            } catch (Exception e) {
+                throw new OperationException(e);
+            }
+        });
+    }
+
+    public List<Response> waitForRapidFullResults(BlackDuckRunData blackDuckRunData, List<HttpUrl> rapidScans, BlackduckScanMode mode) throws OperationException {
+//        // First, append /full-result to all these URLs (TODO has only been tested w/ pkg mngr scans)
+//        List<HttpUrl> fullResultUrls = new ArrayList<>();
+//        for (HttpUrl url : rapidScans) {
+//            try {
+//                HttpUrl fullVersion = url.appendRelativeUrl("full-result");
+//                fullResultUrls.add(fullVersion);
+//            } catch (Exception e) {
+//                logger.debug("uh oh something went wrong");
+//                logger.error(e.getMessage(), e);
+//            }
+//        }
+
+        return auditLog.namedInternal("Rapid Full Wait", () -> {
+            BlackDuckServicesFactory blackDuckServicesFactory = blackDuckRunData.getBlackDuckServicesFactory();
+            int fibonacciSequenceIndex = getFibonacciSequenceIndex();
+
+            try {
+                return new RapidModeWaitOperation(blackDuckServicesFactory.getBlackDuckApiClient()).waitForFullScans(
                         rapidScans,
                         detectConfigurationFactory.findTimeoutInSeconds(),
                         RapidModeWaitOperation.DEFAULT_WAIT_INTERVAL_IN_SECONDS,
@@ -741,6 +776,7 @@ public List<DeveloperScansScanView> waitForRapidResults(BlackDuckRunData blackDu
         });
     }
 
+
     private OperationException handleRapidScanException(IntegrationRestException e) {
         RapidCompareMode rapidCompareMode = detectConfigurationFactory.createRapidScanOptions().getCompareMode();
 
@@ -778,10 +814,19 @@ public final File generateRapidJsonFile(NameVersion projectNameVersion, List<Dev
         return auditLog.namedPublic(
             "Generate Rapid Json File",
             "RapidScan",
-            () -> new RapidModeGenerateJsonOperation(htmlEscapeDisabledGson, directoryManager).generateJsonFile(projectNameVersion, scanResults)
+            () -> new RapidModeGenerateJsonOperation(htmlEscapeDisabledGson, directoryManager).generateJsonFile(projectNameVersion, scanResults, "")
+        );
+    }
+
+    public final File generateFULLRapidJsonFile(List<Response> scanResults) throws OperationException {
+        return auditLog.namedPublic(
+                "Generate Rapid FULL Json File",
+                "RapidScan",
+                () -> new RapidModeGenerateJsonOperation(htmlEscapeDisabledGson, directoryManager).generateJsonFileFromString(scanResults.get(0).getContentString())
         );
     }
 
+
     public final void publishRapidResults(File jsonFile, RapidScanResultSummary summary, BlackduckScanMode mode) throws OperationException {
         auditLog.namedInternal("Publish Rapid Results", () -> statusEventPublisher.publishDetectResult(new RapidScanDetectResult(jsonFile.getCanonicalPath(), summary, mode, detectConfigurationFactory.getPoliciesToFailOn())));
     }
@@ -806,7 +851,7 @@ private void failComponentLocationAnalysisOperationTask(String reason) throws Op
      * @param bdio
      * @throws OperationException
      */
-    public void generateComponentLocationAnalysisIfEnabled(BdioResult bdio) throws OperationException {
+    public void generateComponentLocationAnalysisIfEnabled(BdioResult bdio, File rapidFullResultsFile) throws OperationException {
         if (detectConfigurationFactory.isComponentLocationAnalysisEnabled()) {
             if (bdio.getCodeLocationNamesResult().getCodeLocationNames().isEmpty()) {
                 failComponentLocationAnalysisOperationTask("Component Location Analysis requires non-empty BDIO results. Skipping location analysis.");
@@ -822,7 +867,7 @@ public void generateComponentLocationAnalysisIfEnabled(BdioResult bdio) throws O
                             () -> {
                                 publishResult(
                                     new GenerateComponentLocationAnalysisOperation(detectConfigurationFactory, statusEventPublisher, exitCodePublisher)
-                                        .locateComponents(componentsSet, directoryManager.getScanOutputDirectory(), directoryManager.getSourceDirectory())
+                                        .locateComponents(componentsSet, directoryManager.getScanOutputDirectory(), directoryManager.getSourceDirectory(), rapidFullResultsFile, detectConfigurationFactory)
                                 );
                             }
                     );
@@ -838,7 +883,7 @@ public void generateComponentLocationAnalysisIfEnabled(BdioResult bdio) throws O
      * @param bdio
      * @throws OperationException
      */
-    public void generateComponentLocationAnalysisIfEnabled(List<DeveloperScansScanView> rapidResults, BdioResult bdio) throws OperationException {
+    public void generateComponentLocationAnalysisIfEnabled(List<DeveloperScansScanView> rapidResults, BdioResult bdio, File rapidFullResultsFile) throws OperationException {
         if (detectConfigurationFactory.isComponentLocationAnalysisEnabled()) {
             if (rapidResults.isEmpty()) {
                 failComponentLocationAnalysisOperationTask("Component Location Analysis requires non-empty Rapid/Stateless Scan results. Skipping location analysis.");
@@ -854,7 +899,7 @@ public void generateComponentLocationAnalysisIfEnabled(List<DeveloperScansScanVi
                             () -> {
                                 publishResult(
                                     new GenerateComponentLocationAnalysisOperation(detectConfigurationFactory, statusEventPublisher, exitCodePublisher)
-                                        .locateComponents(componentsSet, directoryManager.getScanOutputDirectory(), directoryManager.getSourceDirectory())
+                                        .locateComponents(componentsSet, directoryManager.getScanOutputDirectory(), directoryManager.getSourceDirectory(), rapidFullResultsFile, detectConfigurationFactory)
                                 );
                             }
                     );

src/main/java/com/blackduck/integration/detect/lifecycle/run/step/IntelligentModeStepRunner.java

@@ -90,7 +90,7 @@ public void runOffline(NameVersion projectNameVersion, DockerTargetData dockerTa
             iacScanStepRunner.runIacScanOffline();
         });
 
-        operationRunner.generateComponentLocationAnalysisIfEnabled(bdio);
+        operationRunner.generateComponentLocationAnalysisIfEnabled(bdio, null);
     }
 
     //TODO: Change black duck post options to a decision and stick it in Run Data somewhere.

src/main/java/com/blackduck/integration/detect/lifecycle/run/step/RapidModeStepRunner.java

@@ -12,6 +12,7 @@
 import java.util.UUID;
 import java.util.Set;
 
+import com.blackduck.integration.rest.response.Response;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -45,6 +46,9 @@ public class RapidModeStepRunner {
     private final Gson gson;
     private final String detectRunUuid;
     private final DirectoryManager directoryManager;
+    public static final String RAPID_SCAN_ENDPOINT = "/api/developer-scans";
+//    public static final String RAPID_SCAN_FULL_RESULT_ENDPOINT = "/api/developer-scans/%s/full-result";
+
 
     public RapidModeStepRunner(OperationRunner operationRunner, StepHelper stepHelper, Gson gson, String detectRunUuid, DirectoryManager directoryManager) {
         this.operationRunner = operationRunner;
@@ -65,10 +69,10 @@ public void runOnline(BlackDuckRunData blackDuckRunData, NameVersion projectVers
         List<HttpUrl> parsedUrls = new ArrayList<>();
         Set<FormattedCodeLocation> formattedCodeLocations = new HashSet<>();
 
-        List<HttpUrl> uploadResultsUrls = operationRunner.performRapidUpload(blackDuckRunData, bdioResult, rapidScanConfig.orElse(null));
-
+        // pkg mgr rapid scan
+        List<HttpUrl> uploadResultsUrls = operationRunner.performRapidUpload(blackDuckRunData, bdioResult, rapidScanConfig.orElse(null)); // pkg mngr rapid bdio upload, returns the upload url with scan-id
         if (uploadResultsUrls != null && uploadResultsUrls.size() > 0) {
-            processScanResults(uploadResultsUrls, parsedUrls, formattedCodeLocations, DetectTool.DETECTOR.name());
+            processScanResults(uploadResultsUrls, parsedUrls, formattedCodeLocations, DetectTool.DETECTOR.name()); // adds URLs from the BDIO upload that will be polled LATER
         }
 
         stepHelper.runToolIfIncluded(DetectTool.SIGNATURE_SCAN, "Signature Scanner", () -> {
@@ -95,9 +99,7 @@ public void runOnline(BlackDuckRunData blackDuckRunData, NameVersion projectVers
             }
         });
 
-        stepHelper.runToolIfIncluded(
-            DetectTool.CONTAINER_SCAN, "Container Scanner",
-            () -> {
+        stepHelper.runToolIfIncluded(DetectTool.CONTAINER_SCAN, "Container Scanner", () -> {
                 logger.debug("Stateless container scan detected.");
                 // Check if this is an SCA environment.
                 if (scaaasFilePath.isPresent()) {
@@ -116,20 +118,26 @@ public void runOnline(BlackDuckRunData blackDuckRunData, NameVersion projectVers
                         formattedCodeLocations.add(new FormattedCodeLocation(containerScanStepRunner.getCodeLocationName(), scanId.get(), DetectTool.CONTAINER_SCAN.name()));
                     }
                 }
-            }
-        );
+            });
 
         // Get info about any scans that were done
         BlackduckScanMode mode = blackDuckRunData.getScanMode();
-        List<DeveloperScansScanView> rapidResults = operationRunner.waitForRapidResults(blackDuckRunData, parsedUrls, mode);
+        List<DeveloperScansScanView> rapidResults = operationRunner.waitForRapidResults(blackDuckRunData, parsedUrls, mode); // parsedurls have all the urls we need to poll for results
+        // Get FULL rapid results for quackpatch separately for now
+        // TODO only bother with below if quackpatch is possible (add explicit flag to enable quackpatch?)
+        List<Response> rapidFullResults = operationRunner.waitForRapidFullResults(blackDuckRunData, parsedUrls, mode); // TODO write to file as is
 
-        operationRunner.generateComponentLocationAnalysisIfEnabled(rapidResults, bdioResult);
 
         // Generate a report, even an empty one if no scans were done as that is what previous detect versions did.
         File jsonFile = operationRunner.generateRapidJsonFile(projectVersion, rapidResults);
+        File jsonFileFULL = operationRunner.generateFULLRapidJsonFile(rapidFullResults);
+
+        operationRunner.generateComponentLocationAnalysisIfEnabled(rapidResults, bdioResult, jsonFileFULL);
+
         RapidScanResultSummary summary = operationRunner.logRapidReport(rapidResults, mode);
 
         operationRunner.publishRapidResults(jsonFile, summary, mode);
+
         operationRunner.publishCodeLocationData(formattedCodeLocations);
     }
 
@@ -165,10 +173,10 @@ private void invokeBdbaRapidScan(BlackDuckRunData blackDuckRunData, String black
         rapidBdbaStepRunner.downloadAndExtractBdio(directoryManager);
 
         UUID bdScanId = operationRunner.initiateStatelessBdbaScan(blackDuckRunData);
-        operationRunner.uploadBdioEntries(blackDuckRunData, bdScanId);
+        operationRunner.uploadBdioEntries(blackDuckRunData, bdScanId); // uploads to rapid scan endpoint
 
         // add this scan to the URLs to wait for
-        parsedUrls.add(new HttpUrl(blackDuckUrl + "/api/developer-scans/" + bdScanId.toString()));
+        parsedUrls.add(new HttpUrl(blackDuckUrl + String.format(RAPID_SCAN_ENDPOINT + "/" + bdScanId.toString())));
     }
 
     /**
@@ -200,7 +208,7 @@ private List<HttpUrl> parseScanUrls(String scanMode, SignatureScanOuputResult si
                     Set<String> parsedIds = result.parseScanIds();
 
                     for (String id : parsedIds) {
-                        HttpUrl url = new HttpUrl(blackDuckUrl + "/api/developer-scans/" + id);
+                        HttpUrl url = new HttpUrl(blackDuckUrl + String.format(RAPID_SCAN_ENDPOINT + "/" + id));
 
                         logger.info(scanMode + " mode signature scan URL: {}", url);
                         parsedUrls.add(url);