diff --git a/tests/integration/godog/features/high avalability/etcd_discovery.feature b/tests/integration/godog/features/high avalability/etcd_discovery.feature new file mode 100644 index 0000000000..a9c88b787f --- /dev/null +++ b/tests/integration/godog/features/high avalability/etcd_discovery.feature @@ -0,0 +1,37 @@ +@etcdDiscoveryHighAvailability @HighAvailability @Functional +Feature: + In order to ensure scheduler and control plane high availability + As a Core 2 platform operator + I want to be able to change the cluster membership of scheduler when needed + + @etcdDiscoveryRuleFromOneNodeScaleUp + Scenario:The cluster size of scheduler is set to 1 but the Core 2 platform operator wants to increase the scheduler + cluster size by two more nodes + Given the control plane is deployed with 1 scheduler replicas + And the scheduler uses a highly available data store + When I increase the scheduler cluster size by "2" + Then a new leader should be elected within "5s" + And exactly 1 scheduler pod should be Ready + And the Ready scheduler pod should be the new leader + And the scheduler Service should route traffic to the new leader + + @etcdDiscoveryRuleFromThreeNodeScaleDownToOne + Scenario:The cluster size of scheduler is set to 3 but the Core 2 platform operator wants to decrease the scheduler + cluster size by two more nodes + Given the control plane is deployed with 3 scheduler replicas + And the scheduler uses a highly available data store + When I decrease the scheduler cluster size by "2" + Then exactly 1 scheduler pod should be Ready + And the Ready scheduler pod should be the new leader + And the scheduler Service should route traffic to the new leader + + @etcdDiscoveryRuleFromThreeNodeScaleUpToFive + Scenario:The cluster size of scheduler is set to 3 but the Core 2 platform operator wants to increase the scheduler + cluster size by two more nodes + Given the control plane is deployed with 3 scheduler replicas + And the scheduler uses a highly available data store + When I increase the scheduler cluster size by "2" + Then a new leader should be elected within "5s" + And exactly 1 scheduler pod should be Ready + And the Ready scheduler pod should be the new leader + And the scheduler Service should route traffic to the new leader \ No newline at end of file diff --git a/tests/integration/godog/features/high avalability/scheduler.feature b/tests/integration/godog/features/high avalability/scheduler.feature new file mode 100644 index 0000000000..b6ef0f2db5 --- /dev/null +++ b/tests/integration/godog/features/high avalability/scheduler.feature @@ -0,0 +1,53 @@ +@SchedulerHighAvailability @HighAvailability @Functional +Feature: Scheduler High Availability + In order to ensure reliable model scheduling and orchestration + As a Core 2 platform operator + I want the control plane to continue functioning even if one or more scheduler replicas fail + + Background: + Given the control plane is deployed with at least 3 scheduler replicas + + Scenario: Scheduler elects a new leader and exposes it via the service when the current leader fails + Given exactly 1 scheduler pod is Ready + And the Ready scheduler pod is the leader + When I terminate the scheduler leader pod + Then a new leader should be elected within "5" seconds + And exactly 1 scheduler pod should be Ready + And the Ready scheduler pod should be the new leader + And the scheduler Service should route traffic to the new leader + + Scenario: Followers remain unroutable and do not become leaders on restart + Given exactly 1 scheduler pod is Ready + And the Ready scheduler pod is the leader + When I terminate a scheduler follower pod + Then the scheduler cluster should remain Ready + And a new follower pod should be running within "10" seconds + And exactly 1 scheduler pod should still be Ready + And the Ready scheduler pod should still be the leader + And no follower pod should be Ready or receive traffic + + Scenario: Only the leader scheduler pod is Ready and routable + When I inspect the scheduler pods + Then exactly 1 scheduler pod should be Ready + And the Ready scheduler pod should be the leader + And all follower scheduler pods should be NotReady + And the scheduler Service should route traffic to the leader pod + + Scenario: Followers do not accept scheduling requests directly + Given exactly 1 scheduler pod is Ready + And the Ready scheduler pod is the leader + When I send a scheduling request directly to a follower pod + Then the request should be rejected or not routable + And the follower should not make scheduling decisions + + + Scenario: There is only a leader when the scheduler cluster is restarted + When the scheduler cluster is restarted + Then a new leader should be elected withing "5" seconds + And exactly 1 scheduler pod should be Ready + And the Ready scheduler pod should be the new leader + And the scheduler Service should route traffic to the new leader + +# Scenario: Scheduler cluster data is the same when there is a leadership change +# THis test case might be difficult to do since data in the scheduler can easily change without intervention +# e.g server restart while we do the operation etc \ No newline at end of file diff --git a/tests/integration/godog/features/autoscaling/model_autoscaling.feature b/tests/integration/godog/features/model/model_autoscaling.feature similarity index 100% rename from tests/integration/godog/features/autoscaling/model_autoscaling.feature rename to tests/integration/godog/features/model/model_autoscaling.feature diff --git a/tests/integration/godog/go.mod b/tests/integration/godog/go.mod index b6af6adbf3..e7b58dc41c 100644 --- a/tests/integration/godog/go.mod +++ b/tests/integration/godog/go.mod @@ -16,12 +16,20 @@ require ( sigs.k8s.io/controller-runtime v0.22.4 ) +replace ( + github.com/seldonio/seldon-core/apis/go/v2 => ../../../apis/go + github.com/seldonio/seldon-core/operator/v2 => ../../../operator +) + require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cucumber/gherkin/go/v26 v26.2.0 // indirect github.com/cucumber/messages/go/v21 v21.0.1 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/emicklei/go-restful/v3 v3.12.2 // indirect github.com/evanphx/json-patch/v5 v5.9.11 // indirect + github.com/fsnotify/fsnotify v1.9.0 // indirect github.com/fxamacker/cbor/v2 v2.9.0 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-openapi/jsonpointer v0.21.1 // indirect @@ -29,6 +37,7 @@ require ( github.com/go-openapi/swag v0.23.1 // indirect github.com/gofrs/uuid v4.3.1+incompatible // indirect github.com/gogo/protobuf v1.3.2 // indirect + github.com/google/btree v1.1.3 // indirect github.com/google/gnostic-models v0.7.0 // indirect github.com/google/go-cmp v0.7.0 // indirect github.com/google/uuid v1.6.0 // indirect @@ -41,12 +50,18 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/prometheus/client_golang v1.22.0 // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/common v0.65.0 // indirect + github.com/prometheus/procfs v0.17.0 // indirect github.com/x448/float16 v0.8.4 // indirect go.yaml.in/yaml/v2 v2.4.2 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/net v0.41.0 // indirect golang.org/x/oauth2 v0.30.0 // indirect + golang.org/x/sync v0.15.0 // indirect golang.org/x/sys v0.33.0 // indirect golang.org/x/term v0.32.0 // indirect golang.org/x/text v0.26.0 // indirect @@ -54,7 +69,9 @@ require ( gomodules.xyz/jsonpatch/v2 v2.5.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822 // indirect google.golang.org/protobuf v1.36.6 // indirect + gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect + k8s.io/apiextensions-apiserver v0.34.1 // indirect k8s.io/klog/v2 v2.130.1 // indirect k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b // indirect k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 // indirect diff --git a/tests/integration/godog/go.sum b/tests/integration/godog/go.sum index 6f84ff438c..726fc059ac 100644 --- a/tests/integration/godog/go.sum +++ b/tests/integration/godog/go.sum @@ -20,6 +20,8 @@ github.com/evanphx/json-patch v5.9.0+incompatible h1:fBXyNpNMuTTDdquAq/uisOr2lSh github.com/evanphx/json-patch v5.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjTM0wiaDU= github.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/XEtnUf6OZxqIQTM= +github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= +github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= @@ -43,11 +45,15 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= +github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo= github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo= github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= @@ -73,6 +79,8 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -80,6 +88,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4= github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -111,10 +121,6 @@ github.com/prometheus/procfs v0.17.0/go.mod h1:oPQLaDAMRbA+u8H5Pbfq+dl3VDAvHxMUO github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/seldonio/seldon-core/apis/go/v2 v2.9.1 h1:9UcxnTFRuCDApZqy7cy3Rm6B/aaW2+3bjNXUKYVvlRY= -github.com/seldonio/seldon-core/apis/go/v2 v2.9.1/go.mod h1:ptbV8xxTT6DI5hWGcOx74bizYhms/LhXBJ/04RD41jk= -github.com/seldonio/seldon-core/operator/v2 v2.10.1 h1:Btn8xcFt5rPd4+xCMFAKwcuXGHAq4/nzE5EuYuNg0uI= -github.com/seldonio/seldon-core/operator/v2 v2.10.1/go.mod h1:WMy17S3Q6QZTR2IP1OaIgRdh36RiNboT8jqCajJ6X9A= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= diff --git a/tests/integration/godog/k8sclient/client.go b/tests/integration/godog/k8sclient/client.go index 9139b51e97..4e90d1faee 100644 --- a/tests/integration/godog/k8sclient/client.go +++ b/tests/integration/godog/k8sclient/client.go @@ -33,10 +33,14 @@ type K8sClient struct { KubeClient client.WithWatch } -var CRDLabels = map[string]string{ +var DefaultCRDLabelMap = map[string]string{ "test-suite": "godog", } +const ( + DefaultCRDLabel = "test-suite=godog" +) + // New todo: separate k8s client init and pass to new func New(namespace string) (*K8sClient, error) { k8sScheme := runtime.NewScheme() @@ -78,8 +82,10 @@ func (k8s *K8sClient) ApplyModel(model *mlopsv1alpha1.Model) error { model.Labels = map[string]string{} } - // Add label - model.Labels = CRDLabels + // add labels + for k, v := range DefaultCRDLabelMap { + model.Labels[k] = v + } existing := &mlopsv1alpha1.Model{} key := client.ObjectKey{ @@ -102,12 +108,12 @@ func (k8s *K8sClient) ApplyModel(model *mlopsv1alpha1.Model) error { return k8s.KubeClient.Update(ctx, model) } -func (k8s *K8sClient) DeleteGodogTestModels(ctx context.Context) error { +func (k8s *K8sClient) DeleteScenarioResources(ctx context.Context, labels client.MatchingLabels) error { list := &mlopsv1alpha1.ModelList{} err := k8s.KubeClient.List(ctx, list, client.InNamespace(k8s.namespace), - client.MatchingLabels{"test-suite": "godog"}, + labels, ) if err != nil { return err diff --git a/tests/integration/godog/k8sclient/watcher.go b/tests/integration/godog/k8sclient/watcher.go index 21e5c4f5f1..5ec02d6298 100644 --- a/tests/integration/godog/k8sclient/watcher.go +++ b/tests/integration/godog/k8sclient/watcher.go @@ -14,11 +14,11 @@ import ( "fmt" "sync" - mlopsv1alpha1 "github.com/seldonio/seldon-core/operator/v2/apis/mlops/v1alpha1" + "github.com/seldonio/seldon-core/operator/v2/pkg/generated/clientset/versioned/typed/mlops/v1alpha1" "k8s.io/apimachinery/pkg/api/meta" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/watch" - "sigs.k8s.io/controller-runtime/pkg/client" ) type WatcherStorage interface { @@ -31,9 +31,9 @@ type WatcherStorage interface { } type WatcherStore struct { - namespace string - label map[string]string - + namespace string + label string + mlopsClient v1alpha1.MlopsV1alpha1Interface modelWatcher watch.Interface mu sync.RWMutex @@ -52,13 +52,8 @@ type waiter struct { type ConditionFunc func(obj runtime.Object) (done bool, err error) // NewWatcherStore receives events that match on a particular object list and creates a database store to query crd state -func NewWatcherStore(namespace string, label map[string]string, w client.WithWatch) (*WatcherStore, error) { - modelWatcher, err := w.Watch( - context.Background(), - &mlopsv1alpha1.ModelList{}, - client.InNamespace(namespace), - client.MatchingLabels(label), - ) +func NewWatcherStore(namespace string, label string, mlopsClient v1alpha1.MlopsV1alpha1Interface) (*WatcherStore, error) { + modelWatcher, err := mlopsClient.Models(namespace).Watch(context.Background(), v1.ListOptions{LabelSelector: "test-suite=godog"}) if err != nil { return nil, fmt.Errorf("failed to create model watcher: %w", err) } @@ -66,6 +61,7 @@ func NewWatcherStore(namespace string, label map[string]string, w client.WithWat return &WatcherStore{ namespace: namespace, label: label, + mlopsClient: mlopsClient, modelWatcher: modelWatcher, store: make(map[string]runtime.Object), doneChan: make(chan struct{}), diff --git a/tests/integration/godog/scenario/scenario.go b/tests/integration/godog/scenario/scenario.go index 9b4bff15cd..fe7bbfeb40 100644 --- a/tests/integration/godog/scenario/scenario.go +++ b/tests/integration/godog/scenario/scenario.go @@ -12,16 +12,20 @@ package scenario import ( "context" "fmt" + "time" "github.com/cucumber/godog" + v "github.com/seldonio/seldon-core/operator/v2/pkg/generated/clientset/versioned" "github.com/seldonio/seldon-core/tests/integration/godog/k8sclient" "github.com/seldonio/seldon-core/tests/integration/godog/steps" "github.com/sirupsen/logrus" + controllerruntime "sigs.k8s.io/controller-runtime" ) type SuiteDeps struct { - K8sClient *k8sclient.K8sClient - WatcherStore k8sclient.WatcherStorage + k8sClient *k8sclient.K8sClient + mlopsClient *v.Clientset + watcherStore k8sclient.WatcherStorage } // might have to pass the suit struct and other config with closures to avoid having global vars @@ -41,41 +45,36 @@ type SuiteDeps struct { // } var suiteDeps SuiteDeps -// todo: think about how we can drive server config from a file -// - have a default server for test -// - but also have the posibility of specifying the servers deployed in the test suite - -// we create a server config or multiple servers -// server 1 caps mlserver -// default modelCapDeployment = mlserver - -// we can set the default model capabilities for models - -// if we have an scenario and it doesn't specify the server capabilities that it is deployed to -// it will deploy into the default server capability that might be in our server dpeloyment def - func InitializeTestSuite(ctx *godog.TestSuiteContext) { + // todo: we should bootstrap config here + // Create long-lived deps here k8sClient, err := k8sclient.New("seldon-mesh") if err != nil { panic(fmt.Errorf("failed to create k8s client: %w", err)) } - watchStore, err := k8sclient.NewWatcherStore("seldon-mesh", k8sclient.CRDLabels, k8sClient.KubeClient) + clientSet, err := v.NewForConfig(controllerruntime.GetConfigOrDie()) + if err != nil { + panic(fmt.Errorf("failed to mlops client: %w", err)) + } + + watchStore, err := k8sclient.NewWatcherStore("seldon-mesh", k8sclient.DefaultCRDLabel, clientSet.MlopsV1alpha1()) if err != nil { panic(fmt.Errorf("failed to create k8s watch store: %w", err)) } - suiteDeps.K8sClient = k8sClient - suiteDeps.WatcherStore = watchStore + suiteDeps.k8sClient = k8sClient + suiteDeps.mlopsClient = clientSet // todo: this clientSet might get use for get requests or for the mlops interface and could be passed to the world might be split up by type + suiteDeps.watcherStore = watchStore ctx.BeforeSuite(func() { - suiteDeps.WatcherStore.Start() + suiteDeps.watcherStore.Start() // e.g. create namespace, apply CRDs, etc. }) ctx.AfterSuite(func() { - suiteDeps.WatcherStore.Stop() + suiteDeps.watcherStore.Stop() // e.g. clean namespace, close clients if needed }) } @@ -85,8 +84,8 @@ func InitializeScenario(scenarioCtx *godog.ScenarioContext) { world, err := steps.NewWorld(steps.Config{ Namespace: "seldon-mesh", //TODO configurable Logger: logrus.New().WithField("test_type", "godog"), - KubeClient: suiteDeps.K8sClient, - WatcherStorage: suiteDeps.WatcherStore, + KubeClient: suiteDeps.k8sClient, + WatcherStorage: suiteDeps.watcherStore, IngressHost: "localhost", //TODO configurable HTTPPort: 9000, //TODO configurable GRPCPort: 9000, //TODO configurable @@ -100,11 +99,6 @@ func InitializeScenario(scenarioCtx *godog.ScenarioContext) { // Before: reset state and prep cluster before each scenario scenarioCtx.Before(func(ctx context.Context, scenario *godog.Scenario) (context.Context, error) { - if err := world.KubeClient.DeleteGodogTestModels(ctx); err != nil { - return ctx, fmt.Errorf("error when deleting models on before steps: %w", err) - } - - // Create a fresh model for THIS scenario world.CurrentModel.Reset(world) // Reset scenario-level state @@ -114,10 +108,11 @@ func InitializeScenario(scenarioCtx *godog.ScenarioContext) { // After: optional cleanup / rollback scenarioCtx.After(func(ctx context.Context, scenario *godog.Scenario, err error) (context.Context, error) { - // e.g. clean up any resources if needed - // if cleanupErr := world.KubeClient.DeleteGodogTestModels(); cleanupErr != nil && err == nil { - // err = cleanupErr - // } + time.Sleep(5 * time.Second) + if err := world.KubeClient.DeleteScenarioResources(ctx, world.Label); err != nil { + return ctx, fmt.Errorf("error when deleting models on before steps: %w", err) + } + return ctx, err }) diff --git a/tests/integration/godog/steps/model_steps.go b/tests/integration/godog/steps/model_steps.go index d444b4c4b1..5af4486f8f 100644 --- a/tests/integration/godog/steps/model_steps.go +++ b/tests/integration/godog/steps/model_steps.go @@ -20,6 +20,7 @@ import ( "github.com/seldonio/seldon-core/tests/integration/godog/scenario/assertions" "gopkg.in/yaml.v3" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" ) type Model struct { @@ -49,7 +50,9 @@ var testModels = map[string]TestModelConfig{ func LoadModelSteps(scenario *godog.ScenarioContext, w *World) { // Model Operations - scenario.Step(`^I have an? "([^"]+)" model$`, w.CurrentModel.IHaveAModel) + scenario.Step(`^I have an? "([^"]+)" model$`, func(modelName string) error { + return w.CurrentModel.IHaveAModel(modelName, w.Label) + }) scenario.Step(`^the model has "(\d+)" min replicas$`, w.CurrentModel.SetMinReplicas) scenario.Step(`^the model has "(\d+)" max replicas$`, w.CurrentModel.SetMaxReplicas) scenario.Step(`^the model has "(\d+)" replicas$`, w.CurrentModel.SetReplicas) @@ -61,7 +64,7 @@ func LoadModelSteps(scenario *godog.ScenarioContext, w *World) { scenario.Step(`^the model (?:should )?eventually become(?:s)? Ready$`, func() error { return w.CurrentModel.ModelReady(w.WatcherStorage) }) - scenario.Step(`^the model status message should be "([^"]+)"$`, w.CurrentModel.AssertModelStatus) + scenario.Step(`^the model status message should eventually be "([^"]+)"$`, w.CurrentModel.AssertModelStatus) } func LoadExplicitModelSteps(scenario *godog.ScenarioContext, w *World) { @@ -104,7 +107,7 @@ func (m *Model) deployModelSpec(spec *godog.DocString, namespace string, _ *k8sc return nil } -func (m *Model) IHaveAModel(model string) error { +func (m *Model) IHaveAModel(model string, label map[string]string) error { testModel, ok := testModels[model] if !ok { return fmt.Errorf("model %s not found", model) @@ -128,7 +131,7 @@ func (m *Model) IHaveAModel(model string) error { CreationTimestamp: metav1.Time{}, DeletionTimestamp: nil, DeletionGracePeriodSeconds: nil, - Labels: nil, + Labels: label, Annotations: nil, OwnerReferences: nil, Finalizers: nil, @@ -168,12 +171,16 @@ func (m *Model) Reset(world *World) { } func (m *Model) SetMinReplicas(replicas int) { - + m.model.Spec.ScalingSpec.MinReplicas = ptr.To(int32(replicas)) } -func (m *Model) SetMaxReplicas(replicas int) {} +func (m *Model) SetMaxReplicas(replicas int) { + m.model.Spec.ScalingSpec.MaxReplicas = ptr.To(int32(replicas)) +} -func (m *Model) SetReplicas(replicas int) {} +func (m *Model) SetReplicas(replicas int) { + m.model.Spec.ScalingSpec.Replicas = ptr.To(int32(replicas)) +} // ApplyModel model is aware of namespace and testsuite config and it might add extra information to the cr that the step hasn't added like namespace func (m *Model) ApplyModel(k *k8sclient.K8sClient) error { diff --git a/tests/integration/godog/steps/world.go b/tests/integration/godog/steps/world.go index 35fa240232..581992aa4f 100644 --- a/tests/integration/godog/steps/world.go +++ b/tests/integration/godog/steps/world.go @@ -14,6 +14,7 @@ import ( "net/http" "github.com/seldonio/seldon-core/apis/go/v2/mlops/v2_dataplane" + v "github.com/seldonio/seldon-core/operator/v2/pkg/generated/clientset/versioned" "github.com/seldonio/seldon-core/tests/integration/godog/k8sclient" "github.com/sirupsen/logrus" "google.golang.org/grpc" @@ -23,6 +24,7 @@ import ( type World struct { namespace string KubeClient *k8sclient.K8sClient + k8sClient v.Interface WatcherStorage k8sclient.WatcherStorage StartingClusterState string //todo: this will be a combination of starting state awareness of core 2 such as the //todo: server config,seldon config and seldon runtime to be able to reconcile to starting state should we change @@ -30,6 +32,7 @@ type World struct { CurrentModel *Model infer inference logger *logrus.Entry + Label map[string]string } type Config struct { @@ -65,6 +68,10 @@ func NewWorld(c Config) (*World, error) { } grpcClient := v2_dataplane.NewGRPCInferenceServiceClient(conn) + label := map[string]string{ + "scenario": randomSuffix(6), + } + w := &World{ namespace: c.Namespace, KubeClient: c.KubeClient, @@ -75,6 +82,7 @@ func NewWorld(c Config) (*World, error) { httpPort: c.HTTPPort, grpc: grpcClient, ssl: c.SSL}, + Label: label, } if c.Logger != nil {