feat(e2e-tests): model deployment and inference of test models from python tests to bdd (#7007)

* refactor feature to use scenario outline

* refactor inference with do methods

* preparation for default models in test cases

* trace for watcher events

* default model inference

* fix errors in godog repeating twice

* include body response when http errors occur

* tidy up model responses

* more models for basic inferencing

* suppress some of the comments

Author: MiguelAAe

tests/integration/godog/features/model/deployment.feature

@@ -4,16 +4,28 @@ Feature: Model deployment
   As a model user
   I need to create a Model resource and verify it is deployed
 
-  Scenario: Success - Load a model
-    Given I have an "iris" model
+  Scenario Outline: Success - Load a <model> model
+    Given I have an "<model>" model
     When the model is applied
     Then the model should eventually become Ready
 
+    Examples:
+      | model         |
+      | iris          |
+      | income-xgb    |
+      | mnist-onnx    |
+      | income-lgb    |
+      | wine          |
+      | mnist-pytorch |
+      | tfsimple1     |
+
 
-  Scenario: Success - Load a model again
+  Scenario: Success - Load a model and expect status model available
     Given I have an "iris" model
     When the model is applied
-    Then the model should eventually become Ready
+    And the model eventually becomes Ready
+    Then the model status message should eventually be "ModelAvailable"
+
 
   Scenario: Load a specific model
     Given I deploy model spec with timeout "10s":
@@ -31,22 +43,4 @@ Feature: Model deployment
     """
     Then the model should eventually become Ready
 
-  Scenario: Success - Load a model and expect status model available
-    Given I have an "iris" model
-    When the model is applied
-    And the model eventually becomes Ready
-    Then the model status message should eventually be "ModelAvailable"
-
-  Scenario: Success - Load a model with min replicas
-    Given I have an "iris" model
-    And the model has "1" min replicas
-    When the model is applied
-    Then the model should eventually become Ready
-
-# todo: change model type
-  Scenario: Success - Load a big model
-    Given I have an "iris" model
-    When the model is applied
-    Then the model should eventually become Ready
-
 

tests/integration/godog/features/model/inference.feature

@@ -1,16 +1,20 @@
-#@ModelInference @Models @Inference
-#Feature Basic model inferencing
-#
-#  Background:
-#    Given a clean test namespace
-#
-#  Scenario: Model can serve prediction
-#    Given I have an "iris" model
-#    And the model is applied
-#    And the model eventually becomes Ready
-#    When I send a prediction request with payload:
-#      """
-#      { "inputs": [1.0, 2.0, 3.0] }
-#      """
-#    Then the response status should be 200
-#    And the response body should contain "predictions"
+@ModelInference @Models @Inference @Functional
+Feature: Basic model inferencing
+
+  Scenario Outline: Success - Inference for <model> model
+    Given I have an "<model>" model
+    When the model is applied
+    Then the model should eventually become Ready
+    When I send a valid HTTP inference request with timeout "20s"
+    Then expect http response status code "200"
+    When I send a valid gRPC inference request with timeout "20s"
+
+    Examples:
+      | model         |
+      | iris          |
+#      | income-xgb | having errors with GRPC
+#      | mnist-onnx |
+#      | income-lgb | having errors with response
+      | tfsimple1     |
+      | wine          |
+#      | mnist-pytorch | having errors with response

tests/integration/godog/k8sclient/watcher_store.go

@@ -90,7 +90,7 @@ func (s *WatcherStore) Start() {
 				if err != nil {
 					s.logger.WithError(err).Error("failed to access model watcher")
 				} else {
-					s.logger.Debugf("new model watch event with name: %s on namespace: %s", accessor.GetName(), accessor.GetNamespace())
+					s.logger.WithField("event", event).Tracef("new model watch event with name: %s on namespace: %s", accessor.GetName(), accessor.GetNamespace())
 				}
 
 				if event.Object == nil {

tests/integration/godog/steps/infer.go

@@ -25,24 +25,48 @@ import (
 	"google.golang.org/grpc/metadata"
 )
 
-func (i *inference) sendHTTPModelInferenceRequest(ctx context.Context, model string, payload *godog.DocString) error {
-	req, err := http.NewRequestWithContext(ctx, http.MethodPost,
-		fmt.Sprintf("%s://%s:%d/v2/models/%s/infer", httpScheme(i.ssl), i.host, i.httpPort, model), strings.NewReader(payload.Content))
+func (i *inference) doHTTPModelInferenceRequest(ctx context.Context, modelName, body string) error {
+	url := fmt.Sprintf(
+		"%s://%s:%d/v2/models/%s/infer",
+		httpScheme(i.ssl),
+		i.host,
+		i.httpPort,
+		modelName,
+	)
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, strings.NewReader(body))
 	if err != nil {
 		return fmt.Errorf("could not create http request: %w", err)
 	}
+
 	req.Header.Add("Content-Type", "application/json")
 	req.Header.Add("Host", "seldon-mesh.inference.seldon")
-	req.Header.Add("Seldon-model", model)
+	req.Header.Add("Seldon-model", modelName)
 
 	resp, err := i.http.Do(req)
 	if err != nil {
 		return fmt.Errorf("could not send http request: %w", err)
 	}
+
 	i.lastHTTPResponse = resp
 	return nil
 }
 
+// Used from steps that pass an explicit payload (DocString)
+func (i *inference) sendHTTPModelInferenceRequest(ctx context.Context, model string, payload *godog.DocString) error {
+	return i.doHTTPModelInferenceRequest(ctx, model, payload.Content)
+}
+
+// Used from steps that work from a *Model and testModels table
+func (i *inference) sendHTTPModelInferenceRequestFromModel(ctx context.Context, m *Model) error {
+	testModel, ok := testModels[m.modelType]
+	if !ok {
+		return fmt.Errorf("could not find test model %s", m.model.Name)
+	}
+
+	return i.doHTTPModelInferenceRequest(ctx, m.modelName, testModel.ValidInferenceRequest)
+}
+
 func httpScheme(useSSL bool) string {
 	if useSSL {
 		return "https"
@@ -51,20 +75,35 @@ func httpScheme(useSSL bool) string {
 }
 
 func (i *inference) sendGRPCModelInferenceRequest(ctx context.Context, model string, payload *godog.DocString) error {
-	var msg *v2_dataplane.ModelInferRequest
-	if err := json.Unmarshal([]byte(payload.Content), &msg); err != nil {
+	return i.doGRPCModelInferenceRequest(ctx, model, payload.Content)
+}
+
+func (i *inference) sendGRPCModelInferenceRequestFromModel(ctx context.Context, m *Model) error {
+	testModel, ok := testModels[m.modelType]
+	if !ok {
+		return fmt.Errorf("could not find test model %s", m.model.Name)
+	}
+	return i.doGRPCModelInferenceRequest(ctx, m.modelName, testModel.ValidInferenceRequest)
+}
+
+func (i *inference) doGRPCModelInferenceRequest(
+	ctx context.Context,
+	model string,
+	payload string,
+) error {
+	var req v2_dataplane.ModelInferRequest
+	if err := json.Unmarshal([]byte(payload), &req); err != nil {
 		return fmt.Errorf("could not unmarshal gRPC json payload: %w", err)
 	}
-	msg.ModelName = model
+	req.ModelName = model
 
 	md := metadata.Pairs("seldon-model", model)
-	ctx = metadata.NewOutgoingContext(context.Background(), md)
-	resp, err := i.grpc.ModelInfer(ctx, msg)
-	if err != nil {
-		i.lastGRPCResponse.err = err
-	}
+	ctx = metadata.NewOutgoingContext(ctx, md)
+
+	resp, err := i.grpc.ModelInfer(ctx, &req)
 
 	i.lastGRPCResponse.response = resp
+	i.lastGRPCResponse.err = err
 	return nil
 }
 
@@ -196,7 +235,12 @@ func (i *inference) httpRespCheckStatus(status int) error {
 		return errors.New("no http response found")
 	}
 	if status != i.lastHTTPResponse.StatusCode {
-		return fmt.Errorf("expected http response status code %d, got %d", status, i.lastHTTPResponse.StatusCode)
+		body, err := io.ReadAll(i.lastHTTPResponse.Body)
+		if err != nil {
+			return fmt.Errorf("expected http response status code %d, got %d", status, i.lastHTTPResponse.StatusCode)
+		}
+		return fmt.Errorf("expected http response status code %d, got %d with body: %s", status, i.lastHTTPResponse.StatusCode, body)
+
 	}
 	return nil
 }