Create basic integration test

.github/workflows/build.yml

@@ -5,26 +5,26 @@ on:
     branches:
       - main
 
+permissions:
+  contents: write
+
 jobs:
   build:
-    name: build
     runs-on: ubuntu-latest
-
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
-          persist-credentials: false
           fetch-depth: 0
-      - uses: conda-incubator/setup-miniconda@v3
+          ref: ${{ github.event.pull_request.head.ref }}
+      - uses: mamba-org/setup-micromamba@v2
         with:
-          python-version: 3.11
-          miniforge-version: latest
-          activate-environment: github-actions
           environment-file: workflow/envs/github-actions.yml
+          cache-environment: true
       - name: git config
         run: |
           git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com"
           git config --local user.name "github-actions[bot]"
+
       - name: Style R code
         shell: bash -el {0}
         run: |
@@ -54,6 +54,7 @@ jobs:
           directory: .
           snakefile: 'workflow/Snakefile'
           task: 'containerize'
+
       - name: Commit
         run: |
           git add figures/graphviz
@@ -64,8 +65,7 @@ jobs:
           git commit -m "🐳 Update Dockerfile" \
             || echo "No changes in Dockerfile to commit"
 
-      - name: Push
-        uses: ad-m/github-push-action@master
-        with:
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-          branch: ${{ github.head_ref }}
+          git push
+        env:
+            GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+

.github/workflows/tests.yml

@@ -10,36 +10,29 @@ on:
 
 jobs:
   tests:
-    name: tests
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: ["3.11"]
-
     steps:
-      - uses: actions/checkout@v3
-        with:
-          persist-credentials: false
-          fetch-depth: 0
-      - uses: actions/setup-python@v4
+      - uses: actions/checkout@v4
+      - uses: mamba-org/setup-micromamba@v2
         with:
-          python-version: ${{ matrix.python-version }}
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install pytest pytest-parallel
+          environment-file: workflow/envs/github-actions.yml
+          cache-environment: true
+
       - name: Lint workflow
         uses: snakemake/[email protected]
         with:
           directory: .
           snakefile: workflow/Snakefile
           args: "--lint"
+
       - name: Run test workflow
         uses: snakemake/[email protected]
         with:
-          directory: .
-          snakefile: workflow/Snakefile
-          args: "archive --forceall --cores 2 --use-conda --conda-frontend mamba --conda-cleanup-pkgs cache  --show-failed-logs --all-temp --configfile config/test.yaml"
-#      - name: Test with pytest
-#        run: |
-#          pytest --workers 2 .tests/
+            directory: tests/tmp
+            snakefile: ../../workflow/Snakefile
+            args: "--forceall --cores 2 --use-conda --conda-frontend mamba --conda-cleanup-pkgs cache  --show-failed-logs --all-temp --configfile ../config/test.yaml"
+
+      - name: Test with pytest
+        shell: bash -el {0}
+        run: |
+          pytest tests/

.gitignore

@@ -15,3 +15,4 @@ figures/otu*
 results/otu*
 report_otu*
 *.zip
+/tests/tmp

tests/config/test.yaml

@@ -0,0 +1,21 @@
+dataset_csv: ../../data/processed/otu-micro.csv
+dataset_name: otu-micro
+outcome_colname: dx
+ml_methods:
+ - glmnet
+kfold: 2
+ncores: 4
+nseeds: 2
+find_feature_importance: true
+hyperparams:
+  - glmnet:
+    - alpha: 
+      - 0
+    - lambda:
+      - 0.0001
+      - 0.001
+      - 0.01
+      - 0.1
+      - 1
+      - 10
+smk_path: '../../workflow/Snakefile'

tests/snap/figures/graphviz/rulegraph.dot

@@ -0,0 +1,43 @@
+digraph snakemake_dag {
+    graph[bgcolor=white, margin=0];
+    node[shape=box, style=rounded, fontname=sans,                 fontsize=10, penwidth=2];
+    edge[penwidth=2, color=grey];
+	0[label = "targets", color = "0.30 0.6 0.85", style="rounded"];
+	1[label = "render_report", color = "0.43 0.6 0.85", style="rounded"];
+	2[label = "plot_performance", color = "0.63 0.6 0.85", style="rounded"];
+	3[label = "combine_results", color = "0.20 0.6 0.85", style="rounded"];
+	4[label = "run_ml", color = "0.60 0.6 0.85", style="rounded"];
+	5[label = "preprocess_data", color = "0.00 0.6 0.85", style="rounded"];
+	6[label = "plot_feature_importance", color = "0.10 0.6 0.85", style="rounded"];
+	7[label = "find_feature_importance", color = "0.27 0.6 0.85", style="rounded"];
+	8[label = "plot_hp_performance", color = "0.57 0.6 0.85", style="rounded"];
+	9[label = "combine_hp_performance", color = "0.03 0.6 0.85", style="rounded"];
+	10[label = "plot_benchmarks", color = "0.50 0.6 0.85", style="rounded"];
+	11[label = "mutate_benchmark", color = "0.13 0.6 0.85", style="rounded"];
+	12[label = "plot_roc_curves", color = "0.33 0.6 0.85", style="rounded"];
+	13[label = "calc_model_sensspec", color = "0.07 0.6 0.85", style="rounded"];
+	14[label = "dot_to_png", color = "0.17 0.6 0.85", style="rounded"];
+	15[label = "write_graphviz", color = "0.40 0.6 0.85", style="rounded"];
+	1 -> 0
+	6 -> 1
+	8 -> 1
+	12 -> 1
+	14 -> 1
+	2 -> 1
+	10 -> 1
+	3 -> 2
+	7 -> 3
+	11 -> 3
+	13 -> 3
+	4 -> 3
+	5 -> 4
+	3 -> 6
+	4 -> 7
+	9 -> 8
+	4 -> 9
+	3 -> 10
+	4 -> 11
+	3 -> 12
+	4 -> 13
+	15 -> 14
+}            

tests/snap/results/otu-micro/benchmarks_results.csv

@@ -0,0 +1,3 @@
+s,h:m:s,max_rss,max_vms,max_uss,max_pss,io_in,io_out,mean_load,cpu_time,dataset,method,seed
+13.5081,00:00:13,0,0,0,0,0,0,146.64,0,otu-micro,glmnet,100
+15.8718,00:00:15,0,0,0,0,0,0,141.32,0,otu-micro,glmnet,101

tests/snap/results/otu-micro/feature-importance_results.csv

@@ -0,0 +1,12 @@
+perf_metric,perf_metric_diff,pvalue,lower,upper,feat,method,perf_metric_name,seed,dataset
+0.7108997588416731,9.886338797837624e-5,0.4752475247524752,0.7094310001404796,0.712218003774334,Otu00001|Otu00007,glmnet,logLoss,100,otu-micro
+0.7061344014695554,0.00486422076009607,0.504950495049505,0.6815637875730869,0.724593860029227,Otu00002|Otu00005,glmnet,logLoss,100,otu-micro
+0.7064500496672834,0.00454857256236806,0.22772277227722773,0.7000897918203043,0.7121198171352935,Otu00003|Otu00008,glmnet,logLoss,100,otu-micro
+0.707702497812103,0.00329612441754845,0.297029702970297,0.70067575077317,0.7138191216246346,Otu00004|Otu00009,glmnet,logLoss,100,otu-micro
+0.7107556336586557,2.4298857099580573e-4,0.0198019801980198,0.7104458713817189,0.7109985215280443,Otu00006,glmnet,logLoss,100,otu-micro
+0.7108849832277546,1.1363900189687026e-4,0.099009900990099,0.7107548139131115,0.7109996410476921,Otu00010,glmnet,logLoss,100,otu-micro
+0.692083807405421,8.298194941143144e-4,0.19801980198019803,0.6898300458170606,0.6945212631159176,Otu00001|Otu00002|Otu00007|Otu00010,glmnet,logLoss,101,otu-micro
+0.6931702321077426,-2.566052082073611e-4,0.5643564356435643,0.6927818751661532,0.6935964193487366,Otu00003|Otu00008,glmnet,logLoss,101,otu-micro
+0.6928331229823717,8.050391716363237e-5,0.4158415841584158,0.6919382843671043,0.6936662233410287,Otu00004|Otu00009,glmnet,logLoss,101,otu-micro
+0.6935807499810027,-6.671230814674356e-4,0.594059405940594,0.6924189081781714,0.6952762853469436,Otu00005,glmnet,logLoss,101,otu-micro
+0.6928464228710504,6.720402848492402e-5,0.36633663366336633,0.692418623430174,0.6932695538643108,Otu00006,glmnet,logLoss,101,otu-micro

tests/snap/results/otu-micro/performance_results.csv

@@ -0,0 +1,3 @@
+cv_metric_AUC,logLoss,AUC,prAUC,Accuracy,Kappa,F1,Sensitivity,Specificity,Pos_Pred_Value,Neg_Pred_Value,Precision,Recall,Detection_Rate,Balanced_Accuracy,method,seed,dataset
+0.55375,0.710998622229651,0.5,0.3125,0.75,0.5,0.666666666666667,0.5,1,1,0.666666666666667,1,0.5,0.25,0.75,glmnet,100,otu-micro
+0.4496875,0.692913626899535,0.5,0.3125,0.5,0,NA,0,1,NA,0.5,NA,0,0,0.5,glmnet,101,otu-micro

tests/snap/results/otu-micro/runs/glmnet_100_benchmarks.csv

@@ -0,0 +1,2 @@
+s,h:m:s,max_rss,max_vms,max_uss,max_pss,io_in,io_out,mean_load,cpu_time,dataset,method,seed
+13.5081,00:00:13,0,0,0,0,0,0,146.64,0,otu-micro,glmnet,100

tests/snap/results/otu-micro/runs/glmnet_100_feature-importance.csv

@@ -0,0 +1,7 @@
+perf_metric,perf_metric_diff,pvalue,lower,upper,feat,method,perf_metric_name,seed,dataset
+0.7108997588416731,9.886338797837624e-5,0.4752475247524752,0.7094310001404796,0.712218003774334,Otu00001|Otu00007,glmnet,logLoss,100,otu-micro
+0.7061344014695554,0.004864220760096072,0.504950495049505,0.6815637875730869,0.724593860029227,Otu00002|Otu00005,glmnet,logLoss,100,otu-micro
+0.7064500496672834,0.0045485725623680695,0.22772277227722773,0.7000897918203043,0.7121198171352935,Otu00003|Otu00008,glmnet,logLoss,100,otu-micro
+0.707702497812103,0.0032961244175484516,0.297029702970297,0.70067575077317,0.7138191216246346,Otu00004|Otu00009,glmnet,logLoss,100,otu-micro
+0.7107556336586557,2.4298857099580573e-4,0.019801980198019802,0.7104458713817189,0.7109985215280443,Otu00006,glmnet,logLoss,100,otu-micro
+0.7108849832277546,1.1363900189687026e-4,0.09900990099009901,0.7107548139131115,0.7109996410476921,Otu00010,glmnet,logLoss,100,otu-micro

tests/snap/results/otu-micro/runs/glmnet_100_performance.csv

@@ -0,0 +1,2 @@
+cv_metric_AUC,logLoss,AUC,prAUC,Accuracy,Kappa,F1,Sensitivity,Specificity,Pos_Pred_Value,Neg_Pred_Value,Precision,Recall,Detection_Rate,Balanced_Accuracy,method,seed,dataset
+0.55375,0.710998622229651,0.5,0.3125,0.75,0.5,0.666666666666667,0.5,1,1,0.666666666666667,1,0.5,0.25,0.75,glmnet,100,otu-micro

tests/snap/results/otu-micro/runs/glmnet_100_sensspec.csv

@@ -0,0 +1,5 @@
+cancer,normal,actual,tp,fp,sensitivity,fpr,specificity,precision,dataset,method,seed
+0.5016984653035406,0.49830153469645944,cancer,1,0,0.5,0,1,1,otu-micro,glmnet,100
+0.4931904396776823,0.5068095603223177,normal,1,1,0.5,0.5,0.5,0.5,otu-micro,glmnet,100
+0.48850729844505747,0.5114927015549425,normal,1,2,0.5,1,0,0.3333333333333333,otu-micro,glmnet,100
+0.44744728217268837,0.5525527178273116,cancer,2,2,1,1,0,0.5,otu-micro,glmnet,100

tests/snap/results/otu-micro/runs/glmnet_101_benchmarks.csv

@@ -0,0 +1,2 @@
+s,h:m:s,max_rss,max_vms,max_uss,max_pss,io_in,io_out,mean_load,cpu_time,dataset,method,seed
+15.8718,00:00:15,0,0,0,0,0,0,141.32,0,otu-micro,glmnet,101

tests/snap/results/otu-micro/runs/glmnet_101_feature-importance.csv

@@ -0,0 +1,6 @@
+perf_metric,perf_metric_diff,pvalue,lower,upper,feat,method,perf_metric_name,seed,dataset
+0.692083807405421,8.298194941143144e-4,0.19801980198019803,0.6898300458170606,0.6945212631159176,Otu00001|Otu00002|Otu00007|Otu00010,glmnet,logLoss,101,otu-micro
+0.6931702321077426,-2.566052082073611e-4,0.5643564356435643,0.6927818751661532,0.6935964193487366,Otu00003|Otu00008,glmnet,logLoss,101,otu-micro
+0.6928331229823717,8.050391716363237e-5,0.4158415841584158,0.6919382843671043,0.6936662233410287,Otu00004|Otu00009,glmnet,logLoss,101,otu-micro
+0.6935807499810027,-6.671230814674356e-4,0.594059405940594,0.6924189081781714,0.6952762853469436,Otu00005,glmnet,logLoss,101,otu-micro
+0.6928464228710504,6.720402848492402e-5,0.36633663366336633,0.692418623430174,0.6932695538643108,Otu00006,glmnet,logLoss,101,otu-micro

tests/snap/results/otu-micro/runs/glmnet_101_performance.csv

@@ -0,0 +1,2 @@
+cv_metric_AUC,logLoss,AUC,prAUC,Accuracy,Kappa,F1,Sensitivity,Specificity,Pos_Pred_Value,Neg_Pred_Value,Precision,Recall,Detection_Rate,Balanced_Accuracy,method,seed,dataset
+0.4496875,0.692913626899535,0.5,0.3125,0.5,0,NA,0,1,NaN,0.5,NA,0,0,0.5,glmnet,101,otu-micro

tests/snap/results/otu-micro/runs/glmnet_101_sensspec.csv

@@ -0,0 +1,5 @@
+cancer,normal,actual,tp,fp,sensitivity,fpr,specificity,precision,dataset,method,seed
+0.49745439360572086,0.5025456063942791,cancer,1,0,0.5,0,1,1,otu-micro,glmnet,101
+0.496304222842192,0.503695777157808,normal,1,1,0.5,0.5,0.5,0.5,otu-micro,glmnet,101
+0.49247447790883525,0.5075255220911647,normal,1,2,0.5,1,0,0.3333333333333333,otu-micro,glmnet,101
+0.49193335919162595,0.508066640808374,cancer,2,2,1,1,0,0.5,otu-micro,glmnet,101

tests/snap/results/otu-micro/sensspec_results.csv

@@ -0,0 +1,9 @@
+cancer,normal,actual,tp,fp,sensitivity,fpr,specificity,precision,dataset,method,seed
+0.5016984653035406,0.49830153469645944,cancer,1,0,0.5,0,1,1,otu-micro,glmnet,100
+0.4931904396776823,0.5068095603223177,normal,1,1,0.5,0.5,0.5,0.5,otu-micro,glmnet,100
+0.4885072984450574,0.5114927015549425,normal,1,2,0.5,1,0,0.3333333333333333,otu-micro,glmnet,100
+0.4474472821726884,0.5525527178273116,cancer,2,2,1,1,0,0.5,otu-micro,glmnet,100
+0.49745439360572086,0.5025456063942791,cancer,1,0,0.5,0,1,1,otu-micro,glmnet,101
+0.496304222842192,0.503695777157808,normal,1,1,0.5,0.5,0.5,0.5,otu-micro,glmnet,101
+0.4924744779088353,0.5075255220911647,normal,1,2,0.5,1,0,0.3333333333333333,otu-micro,glmnet,101
+0.4919333591916259,0.508066640808374,cancer,2,2,1,1,0,0.5,otu-micro,glmnet,101

tests/test_e2e.py

@@ -0,0 +1,29 @@
+import os
+import pathlib
+import subprocess
+import tempfile
+
+smk_cmd = ('snakemake -c 2 '
+        '-s ../../workflow/Snakefile '
+        '--configfile ../config/test.yaml '
+        '--use-conda --conda-frontend mamba '
+        )
+
+def compare_files(filename1, filename2):
+    with open(filename1, 'r') as infile:
+        f1 = infile.read()
+    with open(filename2, 'r') as infile:
+        f2 = infile.read()
+    return f1 == f2
+
+def test_snakemake():
+    #curr_wd = os.getcwd()
+    tmp_dir = pathlib.Path('tests/tmp')
+    #os.mkdir(tmp_dir)
+    #os.chdir(tmp_dir)
+    #out = subprocess.run(smk_cmd, shell=True, text=True, check=True, capture_output=True)
+    #os.chdir(curr_wd)
+    results = ('results/otu-micro/performance_results.csv',)
+    assertions = [compare_files(pathlib.Path('tests')/ 'snap'/ fname,
+                                tmp_dir / fname) for fname in results]
+    assert all(assertions)

workflow/Snakefile

@@ -41,6 +41,8 @@ find_feature_importance = config["find_feature_importance"]
 results_types = ["performance", "benchmarks", "sensspec"]
 if find_feature_importance:
     results_types.append("feature-importance")
+# for rule write_graphviz
+smk_path = f"-s {config['smk_path']}" if "smk_path" in config else ""
 
 
 include: "rules/learn.smk"

workflow/envs/github-actions.yml

@@ -6,8 +6,10 @@ channels:
   - r
 dependencies:
   - black
+  - pytest
   - python=3.11
   - r-base=4
   - r-styler
   - r-usethis
   - snakefmt
+

workflow/rules/plot.smk

@@ -104,9 +104,10 @@ rule write_graphviz:
         "../envs/smk.yml"
     params:
         config_path=config_path,
+        smk_path=smk_path,
     shell:
         """
-        snakemake --{wildcards.cmd} --configfile {params.config_path} 2> {log} > {output.dot}
+        snakemake {params.smk_path} --{wildcards.cmd} --configfile {params.config_path} 2> {log} > {output.dot}
         """