Merging development version of episomer into main

.github/workflows/check.yaml

@@ -0,0 +1,54 @@
+# Workflow derived from https://github.com/posit-dev/setup-air/tree/main/examples
+on:
+  pull_request:
+    branches: [main, master, dev]
+  push:
+    branches: [main, master, test-ci]
+
+name: check.yml
+
+permissions:
+  contents: write
+
+jobs:
+    R-CMD-check:
+      runs-on: ${{ matrix.config.os }}
+
+      name: ${{ matrix.config.os }} (${{ matrix.config.r }})
+
+      strategy:
+        fail-fast: false
+        matrix:
+          config:
+            - { os: ubuntu-latest, r: "release" }
+            - { os: windows-latest, r: "release" }
+            - { os: macos-latest, r: "release" }
+
+      env:
+        R_KEEP_PKG_SOURCE: yes
+
+      steps:
+        - uses: actions/checkout@v4
+
+        - uses: r-lib/actions/setup-pandoc@v2
+
+        - uses: quarto-dev/quarto-actions/setup@v2
+
+        - uses: r-lib/actions/setup-r@v2
+          with:
+            r-version: ${{ matrix.config.r }}
+            http-user-agent: ${{ matrix.config.http-user-agent }}
+            use-public-rspm: true
+
+        - uses: r-lib/actions/setup-r-dependencies@v2
+          with:
+            extra-packages: any::rcmdcheck
+            needs: check
+            working-directory: episomer
+
+        - uses: r-lib/actions/check-r-package@v2
+          with:
+            upload-snapshots: true
+            build_args: 'c("--no-manual","--compact-vignettes=gs+qpdf")'
+            working-directory: episomer
+

.gitignore

@@ -1,6 +1,8 @@
 recover/
 data/
-maual/
+manual/
+renv
+renv.lock
 *.swp
 *.swo
 *.class
@@ -12,10 +14,11 @@ scala/project/build.properties
 scala/project/target
 scala/target
 scala/lib_managed
+scala/.bsp
 scala/null
 *.jar
-epitweetr/java/ecdc-twitter-bundle_2.12-1.0-source.zip
-epitweetr_*.pdf
+episomer/java/ecdc-episomer-bundle_2.13-1.0-source.zip
+episomer_*.pdf
 vignette_*.pdf
 install/
 prerequisites
@@ -66,6 +69,5 @@ vignettes/*.pdf
 .Rprofile
 
 #Roxygen managed files
-*/man/*.Rd
 NAMESPACE
 /.vs

LICENSE.txt

@@ -284,4 +284,4 @@ the rights granted in Article 2 of this Licence and protect the covered Source
 Code from exclusive appropriation.
 
 All other changes or additions to this Appendix require the production of a new
-EUPL version.
+EUPL version.

Makefile

@@ -0,0 +1,53 @@
+SHELL = bash
+
+define check_epi_home
+	@if [ $(origin EPI_HOME) == undefined ]; then echo "please set EPI_HOME is unset"; exit 1; fi
+endef
+
+define set_kr_pass
+	@export pa=`pass epitools/ecdc_kr_pwd` && if [ -z `printenv pa` ]; then pass insert epitools/ecdc_kr_pwd; fi
+endef
+
+init:
+	rm -rf renv/
+	rm -f .Rprofile
+	rm -f renv.lock
+	Rscript devscripts/renv.R
+
+
+install-dependencies:
+	devscripts/install-dependencies.sh
+package:
+	devscripts/package.sh
+
+install-test:
+	Rscript devscripts/renv.test.R
+
+app:
+	devscripts/app.sh
+
+search:
+	devscripts/search.sh
+
+search-sandboxed:
+	$(call check_epi_home)
+	$(call set_kr_pass)
+	@export ecdc_wtitter_tool_kr_password=`pass epitools/ecdc_kr_pwd` && \
+	R -e "devtools::load_all('epitweetr');epitweetr::setup_config('$$EPI_HOME');epitweetr::search_loop(sandboxed=TRUE)"
+
+R:
+	devscripts/r-interpreter.sh
+
+R-test:
+	$(call check_epi_home)
+	$(call set_kr_pass)
+	devscripts/package.sh
+	Rscript devscripts/renv.test.R
+	@export ecdc_wtitter_tool_kr_password=`pass epitools/ecdc_kr_pwd` && \
+	cd test && R
+
+spark:
+	devscripts/spark-interpreter.sh
+
+scala:
+	devscripts/scala-interpreter.sh

NOTICE

@@ -1,12 +1,12 @@
 =========================================================================
-======================    ECDC epitweetr Notice    ======================
+======================    ECDC episomer Notice    ======================
 =========================================================================
 
-ECDC epitweetr package
-Copyright 2020 The European Center of Disease Control (ECDC)
+ECDC episomer package
+Copyright 2025 The European Center of Disease Control (ECDC)
 
 This product includes software developed at
 The Apache Software Foundation (http://www.apache.org/).
 
-Portions of this software were developed by Epiconcept.
+Part of the code was commissioned by ECDC and produced by Epiconcept.
 

README.md

@@ -1,15 +1,15 @@
-# epitweetr: Early Detection of Public Health Threats from Twitter Data
+# episomer: Early Detection of Public Health Threats from Social Media Data
 
-[epitweetr site](https://github.com/EU-ECDC/epitweetr)
+[episomer site](https://github.com/EU-ECDC/episomer)
 
-[Report bug & issues](https://github.com/EU-ECDC/epitweetr/issues)
+[Report bug & issues](https://github.com/EU-ECDC/episomer/issues)
 
-The `epitweetr` package allows you to automatically monitor trends of tweets by time, place and topic. This automated monitoring aims at early detecting public health threats through the detection of signals (e.g. an unusual increase in the number of tweets for a specific time, place and topic). The `epitweetr` package was designed to focus on infectious diseases, and it can be extended to all hazards or other fields of study by modifying the topics and keywords.
+The `episomer` package allows you to automatically monitor trends of tweets by time, place and topic. This automated monitoring aims at early detecting public health threats through the detection of signals (e.g. an unusual increase in the number of tweets for a specific time, place and topic). The `episomer` package was designed to focus on infectious diseases, and it can be extended to all hazards or other fields of study by modifying the topics and keywords.
 
-The general principle behind `epitweetr` is that it collects tweets and related metadata from the Twitter Standard API versions 1.1 (https://developer.twitter.com/en/docs/twitter-api/v1/tweets/search/overview/standard) and 2.0 (https://developer.twitter.com/en/docs/twitter-api/tweets/search/api-reference/get-tweets-search-recent) according to specified topics and stores these tweets on your computer on a database that can operate to calculate statistics or as a search engine. `epitweetr` geolocalises the tweets and collects information on key words, URLs, hashtags within a tweet but also entities and context detected by the Twitter API 2.0. Tweets are aggregated according to topic and geographical location. Next, a signal detection algorithm identifies the number of tweets (by topic and geographical location) that exceeds what is expected for a given day. If a number of tweets exceeds what is expected, `epitweetr` sends out email alerts to notify those who need to further investigate these signals following the epidemic intelligence processes (filtering, validation, analysis and preliminary assessment).
+The general principle behind `episomer` is that it collects posts and related metadata from the Social Media APIs.  Currently bluesky API is supportdd (https://docs.bsky.app/) according to specified topics and stores these tweets on your computer on a database that can operate to calculate statistics or as a search engine. `episomer` geolocalises the posts and collects information on key words, URLs, hashtags within a posts and are aggregated according to topic and geographical location. Next, a signal detection algorithm identifies the number of posts (by topic and geographical location) that exceeds what is expected for a given day. If a number of posts exceeds what is expected, `episomer` sends out email alerts to notify those who need to further investigate these signals following the epidemic intelligence processes (filtering, validation, analysis and preliminary assessment).
 
-The package includes  an interactive web application (Shiny app) with five pages: the dashboard, where a user can visualise and explore tweets (Fig 1), the alerts page, where you can view the current alerts and train machine learning models for alert classification on user defined categories (Fig 2), the geotag page, where you can evaluate the geolocation algorithm and provide annotations for improving its performance (Fig 3), the data protection page, where the user can search, anonymise and delete tweets from the epitweetr database to support data deletion requests (Fig 4), the configuration page, where you can change settings and check the status of the underlying processes (Fig 5), and the troubleshoot page, with automatic checks and hints for using `epitweetr` with all its functionalities (Fig 6). 
+The package includes  an interactive web application (Shiny app) with five pages: the dashboard, where a user can visualise and explore tweets (Fig 1), the alerts page, where you can view the current alerts and train machine learning models for alert classification on user defined categories (Fig 2), the geotag page, where you can evaluate the geolocation algorithm and provide annotations for improving its performance (Fig 3), the data protection page, where the user can search, anonymise and delete tweets from the episomer database to support data deletion requests (Fig 4), the configuration page, where you can change settings and check the status of the underlying processes (Fig 5), and the troubleshoot page, with automatic checks and hints for using `episomer` with all its functionalities (Fig 6). 
 
 On the dashboard, users can view the aggregated number of tweets over time, the location of these tweets on a map and different most frequent elements found in or extracted from these tweets (words, hashtags, URLs, contexts and entities). These visualisations can be filtered by the topic, location and time period you are interested in. Other filters are available and include the possibility to adjust the time unit of the timeline, whether retweets/quotes should be included, what kind of geolocation types you are interested in, the sensitivity of the prediction interval for the signal detection, and the number of days used to calculate the threshold for signals. This information is also downloadable directly from this interface in the form of data, pictures, and/or reports.
 
-More information is available in the epitweetr peer-review publication ( https://doi.org/10.2807/1560-7917.ES.2022.27.39.2200177).
+More information is available in the episomer peer-review publication ( https://doi.org/10.2807/1560-7917.ES.2022.27.39.2200177).

devscripts/R-dep-licenses.R

@@ -1,5 +1,5 @@
 # R script to get all R dependencies licenses
-deps <- gtools::getDependencies("epitweetr")
+deps <- gtools::getDependencies("episomer")
 licences <- lapply(deps, function(p) list(package = p, license = packageDescription(p, fields="License")))
 jsonlite::write_json(licences, "licenses/r-dep-licenses.json", pretty = T)
 

devscripts/app.bat

@@ -1 +1 @@
-R -e "epitweetr::epitweetr_app(Sys.getenv('EPI_HOME'))" 
+R -e "episomer::episomer_app(Sys.getenv('EPI_HOME'))" 

devscripts/app.sh

@@ -11,6 +11,7 @@ export ecdc_wtitter_tool_kr_password=$pass
 expect -c '
 spawn R
 expect ">"
-send "epitweetr::setup_config(\"'$EPI_HOME'\")\r" 
-send "epitweetr::epitweetr_app()\r"
+send "devtools::load_all(\"episomer\")\r"
+send "episomer::setup_config(\"'$EPI_HOME'\")\r" 
+send "episomer::episomer_app()\r"
 interact'

devscripts/build-manual.R

@@ -1 +1 @@
-devtools::build_manual("epitweetr")
+devtools::build_manual("episomer")

devscripts/check-cran-remote.R

@@ -1,3 +1,3 @@
-devtools::check_win_release("epitweetr")
-#devtools::check_win_devel("epitweetr")
-#devtools::check_win_old_release("epitweetr")
+devtools::check_win_release("episomer")
+#devtools::check_win_devel("episomer")
+#devtools::check_win_old_release("episomer")

devscripts/check-cran.R

@@ -1 +1 @@
-devtools::check("epitweetr")
+devtools::check("episomer")

devscripts/createPackage.R

@@ -1,5 +1,5 @@
-packageName <- "epitweetr" 
+packageName <- "episomer" 
 devtools::create(packageName)
-devtools::use_vignette('epitweetr-vignette')
+devtools::use_vignette('episomer-vignette')
 
 

devscripts/detect.bat

@@ -1,2 +1,2 @@
-R -e "epitweetr::detect_loop(Sys.getenv('EPI_HOME'))" 
+R -e "episomer::detect_loop(Sys.getenv('EPI_HOME'))" 
 

devscripts/detect.sh

@@ -11,6 +11,6 @@ export ecdc_wtitter_tool_kr_password=$pass
 expect -c '
 spawn R
 expect ">"
-send "epitweetr::setup_config(\"'$EPI_HOME'\")\r" 
-send "epitweetr::detect_loop()\r"
+send "episomer::setup_config(\"'$EPI_HOME'\")\r" 
+send "episomer::detect_loop()\r"
 interact'

devscripts/install-dependencies.sh

@@ -0,0 +1,14 @@
+sudo apt update
+sudo apt install r-base
+sudo apt install libcurl4-openssl-dev
+sudo apt install libsodium-dev
+sudo apt install pandoc
+sudo apt install texlive-latex-base
+sudo apt install texlive-fonts-recommended
+sudo apt install libharfbuzz-dev libfribidi-dev
+
+echo "deb https://repo.scala-sbt.org/scalasbt/debian all main" | sudo tee /etc/apt/sources.list.d/sbt.list
+echo "deb https://repo.scala-sbt.org/scalasbt/debian /" | sudo tee /etc/apt/sources.list.d/sbt_old.list
+curl -sL "https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x2EE0EA64E40A89B84B2DF73499E82A75642AC823" | sudo tee /etc/apt/trusted.gpg.d/sbt.asc
+sudo apt update
+sudo apt install sbt

devscripts/java_libs.sh

@@ -1,14 +1,13 @@
 # updating scala depedencies file
 export cdir=`pwd`
-export SPARK_VERSION=3.0.3
 
 cd "$cdir/scala"
 
 rm -r "$cdir/scala/lib_managed"
 sbt package
 cd "$cdir/scala/lib_managed"
 
-find . -type f > "$cdir/epitweetr/inst/extdata/sbt-deps.txt"
+find . -type f > "$cdir/episomer/inst/extdata/sbt-deps.txt"
 
 cd "$cdir"
 

devscripts/package.R

@@ -1,47 +1,52 @@
 # creating package and installing it
 print(getwd())
+repos = "https://cran.asnr.fr/"
 
 if(!("devtools" %in% installed.packages()[,"Package"]))
-  install.packages("devtools")
-
-#devtools::install_deps()
-devtools::document()
-#devtools::build_manual()
-#devtools::build_vignettes()
+  install.packages("devtools", repos = repos)
+fast <- FALSE
+
+if(!fast) {
+    devtools::install_deps("episomer")
+    devtools::document("episomer")
+    renv::snapshot()
+    devtools::build_manual("episomer")
+    devtools::build_vignettes("episomer")
+}
 
-if(!file.exists(file.path("..", "install"))){
-  dir.create(file.path("..", "install"), showWarnings = FALSE)
+if(!file.exists(file.path("install"))){
+  dir.create(file.path("install"), showWarnings = FALSE)
 }
-if(!file.exists(file.path("..", "manual"))){
-  dir.create(file.path("..", "manual"), showWarnings = FALSE)
+if(!file.exists(file.path("manual"))){
+  dir.create(file.path("manual"), showWarnings = FALSE)
 }
 
-if(any(grepl("epitweetr.*", sessionInfo())))
-  detach("package:epitweetr", unload=TRUE)
-
-
+if(any(grepl("episomer.*", sessionInfo())))
+  detach("package:episomer", unload=TRUE)
 
 installer_path <- c(
-  devtools::build(binary=FALSE, vignettes=TRUE, manual=TRUE)
-  ,devtools::build(binary=TRUE, vignettes=TRUE, manual=TRUE)
+  devtools::build("episomer", binary=FALSE, vignettes=TRUE, manual=TRUE)
+  ,devtools::build("episomer", binary=TRUE, vignettes=TRUE, manual=TRUE)
 )
-install.packages(file.path(installer_path[[2]]), dependencies = FALSE)
 
+#install.packages(file.path(installer_path[[2]]), dependencies = FALSE)
+devtools::load_all("episomer")
 
 #moving installer
 installer_name <- (
-  c(paste("epitweetr_",packageVersion("epitweetr"),".tar.gz", sep = ""),
+  c(paste("episomer_",packageVersion("episomer"),".tar.gz", sep = ""),
     if(.Platform$OS.type == "windows")
-      paste("epitweetr_",packageVersion("epitweetr"),".zip", sep = "") 
+      paste("episomer_",packageVersion("episomer"),".zip", sep = "") 
     else
-      paste("epitweetr_",packageVersion("epitweetr"),"_R_x86_64-pc-linux-gnu.tar.gz", sep = "") 
+      paste("episomer_",packageVersion("episomer"),"_R_x86_64-pc-linux-gnu.tar.gz", sep = "") 
   )
 )
-file.rename(file.path(installer_path), file.path("..", "install", installer_name))
-
-manual_name <- paste("epitweetr_",packageVersion("epitweetr"),".pdf", sep = "") 
-#moving manual
-#file.rename(file.path("..", manual_name), file.path("..", "manual", manual_name))
 
+file.rename(file.path(installer_path), file.path("install", installer_name))
+if(!fast) {
+    manual_name <- paste("episomer_",packageVersion("episomer"),".pdf", sep = "") 
+    #moving manual
+    file.rename(file.path(manual_name), file.path("manual", manual_name))
+}
 
 

devscripts/package.bat

@@ -9,12 +9,12 @@ set SBT_OPTS="-Xmx6G"
 cd "%cdir%\scala"
 call sbt package
 
-if not exist "%cdir%\epitweetr\inst\java" mkdir "%cdir%\epitweetr\inst\java"
-copy "%cdir%\scala\target\scala-2.12\ecdc-twitter-bundle_2.12-1.0.jar" "%cdir%\epitweetr\inst\java"
+if not exist "%cdir%\episomer\inst\java" mkdir "%cdir%\episomer\inst\java"
+copy "%cdir%\scala\target\scala-2.12\ecdc-twitter-bundle_2.12-1.0.jar" "%cdir%\episomer\inst\java"
 
-git archive --format zip --output "%cdir%/epitweetr/java/ecdc-twitter-bundle_2.12-1.0-source.zip" HEAD 
+git archive --format zip --output "%cdir%/episomer/java/ecdc-twitter-bundle_2.12-1.0-source.zip" HEAD 
 
-cd "%cdir%\epitweetr"
+cd "%cdir%\episomer"
 
 call Rscript.exe ..\devscripts\package.R
 

devscripts/package.sh

@@ -1,19 +1,17 @@
 # creating package and installing it
 export cdir=`pwd`
 export OPENBLAS_NUM_THREADS=1
-export SPARK_VERSION=3.0.3
-export JAVA_HOME="/usr/lib/jvm/java-8-openjdk-amd64"
-export PATH=$PATH:$JAVA_HOME/bin
 
 cd "$cdir/scala"
 sbt package
 
-mkdir -p "$cdir/epitweetr/inst/java"
-cp "$cdir/scala/target/scala-2.12/ecdc-twitter-bundle_2.12-1.0.jar" "$cdir/epitweetr/inst/java"
+mkdir -p "$cdir/episomer/inst/java"
+cp "$cdir/scala/target/scala-2.13/ecdc-episomer-bundle_2.13-1.0.jar" "$cdir/episomer/inst/java"
 
-git archive --format zip --output "$cdir/epitweetr/java/ecdc-twitter-bundle_2.12-1.0-source.zip" HEAD 
+git archive --format zip --output "$cdir/episomer/java/ecdc-episomer-bundle_2.13-1.0-source.zip" HEAD 
 
-cd "$cdir/epitweetr"
-Rscript ../devscripts/package.R
+cd "$cdir"
+Rscript devscripts/renv.R
+Rscript devscripts/package.R
 
 cd "$cdir"