diff --git a/all-in-one/pom.xml b/all-in-one/pom.xml new file mode 100644 index 000000000..061724ce4 --- /dev/null +++ b/all-in-one/pom.xml @@ -0,0 +1,213 @@ + + + + 4.0.0 + + cn.edu.tsinghua + iot-benchmark + 1.0.0 + ../pom.xml + + all-in-one + Benchmark All-in-One + + + + cn.edu.tsinghua + core + ${project.version} + + + cn.edu.tsinghua + influxdb + ${project.version} + + + cn.edu.tsinghua + influxdb-2.0 + ${project.version} + + + cn.edu.tsinghua + cnosdb + ${project.version} + + + cn.edu.tsinghua + kairosdb + ${project.version} + + + cn.edu.tsinghua + opentsdb + ${project.version} + + + cn.edu.tsinghua + tdengine + ${project.version} + + + cn.edu.tsinghua + tdengine-3.0 + ${project.version} + + + cn.edu.tsinghua + questdb + ${project.version} + + + cn.edu.tsinghua + dolphindb-3.0 + ${project.version} + + + cn.edu.tsinghua + dolphindb-2.0 + ${project.version} + + + cn.edu.tsinghua + timescaledb + ${project.version} + + + cn.edu.tsinghua + timescaledb-cluster + ${project.version} + + + cn.edu.tsinghua + victoriametrics + ${project.version} + + + cn.edu.tsinghua + iotdb-2.0 + ${project.version} + + + cn.edu.tsinghua + iotdb-1.3 + ${project.version} + + + cn.edu.tsinghua + mssqlserver + ${project.version} + + + cn.edu.tsinghua + sqlite + ${project.version} + + + + iot-benchmark-all-in-one + + + org.apache.maven.plugins + maven-assembly-plugin + 3.1.0 + + + + server-assembly + package + + single + + + + src/assembly/assembly.xml + + false + + + true + true + + + + + + + + org.codehaus.mojo + exec-maven-plugin + 3.1.0 + + + + dedup-lib + package + + java + + + cn.edu.tsinghua.iot.benchmark.allinone.DedupTool + + ${project.build.directory}/iot-benchmark-all-in-one/iot-benchmark-all-in-one/lib + + + + + + + org.apache.maven.plugins + maven-antrun-plugin + 3.1.0 + + + + repackage-zip + package + + run + + + + + + + + + + + + + + + + + diff --git a/all-in-one/src/assembly/assembly.xml b/all-in-one/src/assembly/assembly.xml new file mode 100644 index 000000000..0f3e6d272 --- /dev/null +++ b/all-in-one/src/assembly/assembly.xml @@ -0,0 +1,154 @@ + + + + + all-in-one + + + dir + + true + + + ${maven.multiModuleProjectDirectory}/core/target/iot-benchmark-core/iot-benchmark-core/lib + lib/core + + + ${maven.multiModuleProjectDirectory}/influxdb/target/iot-benchmark-influxdb/iot-benchmark-influxdb/lib + lib/influxdb + + + ${maven.multiModuleProjectDirectory}/influxdb-2.0/target/iot-benchmark-influxdb-2.0/iot-benchmark-influxdb-2.0/lib + lib/influxdb-2.0 + + + ${maven.multiModuleProjectDirectory}/cnosdb/target/iot-benchmark-cnosdb/iot-benchmark-cnosdb/lib + lib/cnosdb + + + ${maven.multiModuleProjectDirectory}/kairosdb/target/iot-benchmark-kairosdb/iot-benchmark-kairosdb/lib + lib/kairosdb + + + ${maven.multiModuleProjectDirectory}/opentsdb/target/iot-benchmark-opentsdb/iot-benchmark-opentsdb/lib + lib/opentsdb + + + ${maven.multiModuleProjectDirectory}/tdengine/target/iot-benchmark-tdengine/iot-benchmark-tdengine/lib + lib/tdengine + + + ${maven.multiModuleProjectDirectory}/tdengine-3.0/target/iot-benchmark-tdengine-3.0/iot-benchmark-tdengine-3.0/lib + lib/tdengine-3.0 + + + ${maven.multiModuleProjectDirectory}/questdb/target/iot-benchmark-questdb/iot-benchmark-questdb/lib + lib/questdb + + + ${maven.multiModuleProjectDirectory}/dolphindb-3.0/target/iot-benchmark-dolphindb-3.0/iot-benchmark-dolphindb-3.0/lib + lib/dolphindb-3.0 + + + ${maven.multiModuleProjectDirectory}/dolphindb-2.0/target/iot-benchmark-dolphindb-2.0/iot-benchmark-dolphindb-2.0/lib + lib/dolphindb-2.0 + + + ${maven.multiModuleProjectDirectory}/timescaledb/target/iot-benchmark-timescaledb/iot-benchmark-timescaledb/lib + lib/timescaledb + + + ${maven.multiModuleProjectDirectory}/timescaledb-cluster/target/iot-benchmark-timescaledb-cluster/iot-benchmark-timescaledb-cluster/lib + lib/timescaledb-cluster + + + ${maven.multiModuleProjectDirectory}/victoriametrics/target/iot-benchmark-victoriametrics/iot-benchmark-victoriametrics/lib + lib/victoriametrics + + + ${maven.multiModuleProjectDirectory}/iotdb-2.0/target/iot-benchmark-iotdb-2.0/iot-benchmark-iotdb-2.0/lib + lib/iotdb-2.0 + + + ${maven.multiModuleProjectDirectory}/iotdb-1.3/target/iot-benchmark-iotdb-1.3/iot-benchmark-iotdb-1.3/lib + lib/iotdb-1.3 + + + ${maven.multiModuleProjectDirectory}/mssqlserver/target/iot-benchmark-mssqlserver/iot-benchmark-mssqlserver/lib + lib/mssqlserver + + + ${maven.multiModuleProjectDirectory}/sqlite/target/iot-benchmark-sqlite/iot-benchmark-sqlite/lib + lib/sqlite + + + ${maven.multiModuleProjectDirectory}/configuration/bin/ + bin + 0755 + + + ${maven.multiModuleProjectDirectory}/configuration/conf/ + conf + + + + + ${maven.multiModuleProjectDirectory}/configuration/benchmark.bat + + + ${maven.multiModuleProjectDirectory}/configuration/benchmark.sh + 0755 + + + ${maven.multiModuleProjectDirectory}/configuration/rep-benchmark.sh + 0755 + + + + ${maven.multiModuleProjectDirectory}/configuration/cli-benchmark.sh + 0755 + + + ${maven.multiModuleProjectDirectory}/configuration/routine + + + ${maven.multiModuleProjectDirectory}/LICENSE + + + ${project.build.outputDirectory}/git.properties + + + diff --git a/all-in-one/src/main/java/cn/edu/tsinghua/iot/benchmark/allinone/DedupTool.java b/all-in-one/src/main/java/cn/edu/tsinghua/iot/benchmark/allinone/DedupTool.java new file mode 100644 index 000000000..feca5b221 --- /dev/null +++ b/all-in-one/src/main/java/cn/edu/tsinghua/iot/benchmark/allinone/DedupTool.java @@ -0,0 +1,190 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package cn.edu.tsinghua.iot.benchmark.allinone; + +import java.io.BufferedInputStream; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +/** + * All-in-one 发行包体积优化（构建期后处理）。 + * + *

各数据库模块的发行目录会各自带上 core 的全部依赖（fastutil 16MB 等在每个模块目录 重复一份，18 份总计约 490MB）。本工具对 lib/<db> + * 做两步收敛（判定均为文件名 + 逐字节比较，同名不同版本的 jar 保留）： + * + *

    + *
  1. 去重：删除与 lib/core 同名且内容完全一致的 jar（core 与模块目录各保留一份）。 + *
  2. 提升：lib/core 没有、但出现在 >=2 个模块目录、且各副本逐字节一致的 jar， 提升一份到 lib/core 并删除各模块目录副本。 + *
+ * + *

运行时 classpath 顺序是「专属目录在前、lib/core 在后」（见 bin/startup.sh），收敛后 运行时从 lib/core 加载同一份内容，行为等价。 + * + *

日志生态 jar（slf4j-*、logback-*、log4j-*、reload4j-* 前缀）不参与提升：绑定由 classpath 目录顺序决定「哪个目录在前哪个绑定生效」，提升进 + * core 会与 core 的唯一绑定 （slf4j-reload4j）同目录并存，JVM 通配符展开顺序未指定 → 绑定随机化。 + * + *

zip 由 maven-antrun-plugin 在两步之后重建（见 all-in-one/pom.xml）。 + * + *

用法（maven exec:java）：{@code DedupTool } + */ +public final class DedupTool { + + private static final int BUFFER_SIZE = 64 * 1024; + + private DedupTool() {} + + public static void main(String[] args) throws IOException { + if (args.length < 1) { + System.err.println("用法: DedupTool "); + System.exit(1); + } + Path libDir = Paths.get(args[0]); + Path coreDir = libDir.resolve("core"); + + Map coreJars = new HashMap<>(); + try (DirectoryStream jars = Files.newDirectoryStream(coreDir, "*.jar")) { + for (Path jar : jars) { + coreJars.put(jar.getFileName().toString(), jar); + } + } + System.out.println("[all-in-one dedup] lib/core: " + coreJars.size() + " jars"); + + // name -> 各模块目录中的副本（已存在的，随删除实时移除） + Map> moduleJars = new HashMap<>(); + try (DirectoryStream dirs = Files.newDirectoryStream(libDir)) { + for (Path dir : dirs) { + if (!Files.isDirectory(dir) || dir.equals(coreDir)) { + continue; + } + try (DirectoryStream jars = Files.newDirectoryStream(dir, "*.jar")) { + for (Path jar : jars) { + moduleJars + .computeIfAbsent(jar.getFileName().toString(), k -> new ArrayList<>()) + .add(jar); + } + } + } + } + + // 第一步：去重——删除与 lib/core 同名且逐字节一致的模块目录副本 + int removed = 0; + long removedBytes = 0; + for (Map.Entry> entry : moduleJars.entrySet()) { + Path coreJar = coreJars.get(entry.getKey()); + if (coreJar == null) { + continue; + } + Iterator it = entry.getValue().iterator(); + while (it.hasNext()) { + Path jar = it.next(); + if (!sameContent(jar, coreJar)) { + continue; + } + long size = Files.size(jar); + Files.delete(jar); + it.remove(); + removed++; + removedBytes += size; + } + } + System.out.printf( + "[all-in-one dedup] step1 removed %d jars (%d MB)%n", removed, removedBytes / 1024 / 1024); + + // 第二步：提升——core 没有、>=2 个模块目录同名且各副本逐字节一致、且非日志生态 + // 的 jar，复制一份到 lib/core 并删除各模块目录副本 + int promoted = 0; + long promotedBytes = 0; + for (Map.Entry> entry : moduleJars.entrySet()) { + String name = entry.getKey(); + List copies = entry.getValue(); + if (coreJars.containsKey(name) || copies.size() < 2 || isLoggingJar(name)) { + continue; + } + boolean allSame = true; + for (int i = 1; i < copies.size(); i++) { + if (!sameContent(copies.get(0), copies.get(i))) { + allSame = false; + break; + } + } + if (!allSame) { + System.out.println("[all-in-one dedup] step2 skip 同名不同版本: " + name); + continue; + } + long size = Files.size(copies.get(0)); + Files.copy(copies.get(0), coreDir.resolve(name)); + for (Path jar : copies) { + Files.delete(jar); + } + promoted++; + promotedBytes += size * (copies.size() - 1); + } + System.out.printf( + "[all-in-one dedup] step2 promoted %d jars to lib/core (saved %d MB)%n", + promoted, promotedBytes / 1024 / 1024); + } + + /** 日志生态 jar 不参与提升：绑定随 classpath 目录顺序生效，提升会破坏绑定确定性。 */ + private static boolean isLoggingJar(String name) { + return name.startsWith("slf4j-") + || name.startsWith("logback-") + || name.startsWith("log4j-") + || name.startsWith("reload4j-"); + } + + /** 先比大小、再流式逐块比较，避免一次读入 16MB 级 jar。 */ + private static boolean sameContent(Path a, Path b) throws IOException { + if (Files.size(a) != Files.size(b)) { + return false; + } + byte[] bufA = new byte[BUFFER_SIZE]; + byte[] bufB = new byte[BUFFER_SIZE]; + try (InputStream inA = new BufferedInputStream(new FileInputStream(a.toFile())); + InputStream inB = new BufferedInputStream(new FileInputStream(b.toFile()))) { + while (true) { + int n = inA.read(bufA); + if (n < 0) { + return true; + } + int off = 0; + while (off < n) { + int m = inB.read(bufB, off, n - off); + if (m < 0) { + return false; + } + off += m; + } + for (int i = 0; i < n; i++) { + if (bufA[i] != bufB[i]) { + return false; + } + } + } + } + } +} diff --git a/configuration/benchmark.bat b/configuration/benchmark.bat index 1cbcb650c..6edf5e619 100644 --- a/configuration/benchmark.bat +++ b/configuration/benchmark.bat @@ -70,12 +70,50 @@ set JAVA_OPTS=-ea^ @REM ***** CLASSPATH library setting ***** @REM Ensure that any user defined CLASSPATH variables are not used on startup -set CLASSPATH="%BENCHMARK_HOME%\lib\*" -@REM set CLASSPATH=%CLASSPATH%;iotdb.IoTDB +@REM all-in-one 布局（存在 lib\core 目录）时：先加载 DB_SWITCH 对应的专属目录 +@REM lib\，再加载 lib\core；旧布局（lib 直接放 jar）保持原行为。 +set CLASSPATH= +set DB_LIB_DIR= +if exist "%BENCHMARK_HOME%\lib\core\" ( + for /f "tokens=1,* delims==" %%a in ('findstr /b /c:"DB_SWITCH=" "%BENCHMARK_CONF%\config.properties" 2^>nul') do set "DB_SWITCH=%%b" + if not defined DB_SWITCH for /f "tokens=1,* delims==" %%a in ('findstr /b /c:"# DB_SWITCH=" "%BENCHMARK_CONF%\config.properties" 2^>nul') do set "DB_SWITCH=%%b" + if not defined DB_SWITCH for /f "tokens=1,* delims==" %%a in ('findstr /b /c:"#DB_SWITCH=" "%BENCHMARK_CONF%\config.properties" 2^>nul') do set "DB_SWITCH=%%b" + if defined DB_SWITCH set "DB_SWITCH=%DB_SWITCH: =%" + if defined DB_SWITCH call :select_db_lib + if defined DB_LIB_DIR if exist "%BENCHMARK_HOME%\lib\%DB_LIB_DIR%\" ( + for %%f in ("%BENCHMARK_HOME%\lib\%DB_LIB_DIR%\*.jar") do call :append "%%~f" + ) + for %%f in ("%BENCHMARK_HOME%\lib\core\*.jar") do call :append "%%~f" + @REM conf 也放入 classpath：SLF4J 选中 reload4j（log4j 1.2）binding 的模块 + @REM （如 iotdb-2.0/1.3）依赖 classpath 上的 log4j.properties，否则 LOGGER 输出被吞。 + call :append "%BENCHMARK_HOME%\conf" +) else ( + set CLASSPATH="%BENCHMARK_HOME%\lib\*" +) goto okClasspath +:select_db_lib +echo.%DB_SWITCH%|findstr /b /c:"IoTDB-200-" >nul && set "DB_LIB_DIR=iotdb-2.0" && goto :eof +echo.%DB_SWITCH%|findstr /b /c:"IoTDB-130-" >nul && set "DB_LIB_DIR=iotdb-1.3" && goto :eof +echo.%DB_SWITCH%|findstr /b /c:"InfluxDB-2" >nul && set "DB_LIB_DIR=influxdb-2.0" && goto :eof +echo.%DB_SWITCH%|findstr /b /c:"InfluxDB" >nul && set "DB_LIB_DIR=influxdb" && goto :eof +echo.%DB_SWITCH%|findstr /b /c:"OpenTSDB" >nul && set "DB_LIB_DIR=opentsdb" && goto :eof +echo.%DB_SWITCH%|findstr /b /c:"CnosDB" >nul && set "DB_LIB_DIR=cnosdb" && goto :eof +echo.%DB_SWITCH%|findstr /b /c:"KairosDB" >nul && set "DB_LIB_DIR=kairosdb" && goto :eof +echo.%DB_SWITCH%|findstr /b /c:"TimescaleDB-cluster" >nul && set "DB_LIB_DIR=timescaledb-cluster" && goto :eof +echo.%DB_SWITCH%|findstr /b /c:"TimescaleDB" >nul && set "DB_LIB_DIR=timescaledb" && goto :eof +echo.%DB_SWITCH%|findstr /b /c:"TDengine-3" >nul && set "DB_LIB_DIR=tdengine-3.0" && goto :eof +echo.%DB_SWITCH%|findstr /b /c:"TDengine" >nul && set "DB_LIB_DIR=tdengine" && goto :eof +echo.%DB_SWITCH%|findstr /b /c:"QuestDB" >nul && set "DB_LIB_DIR=questdb" && goto :eof +echo.%DB_SWITCH%|findstr /b /c:"MsSqlServer" >nul && set "DB_LIB_DIR=mssqlserver" && goto :eof +echo.%DB_SWITCH%|findstr /b /c:"VictoriaMetrics" >nul && set "DB_LIB_DIR=victoriametrics" && goto :eof +echo.%DB_SWITCH%|findstr /b /c:"DolphinDB-3" >nul && set "DB_LIB_DIR=dolphindb-3.0" && goto :eof +echo.%DB_SWITCH%|findstr /b /c:"DolphinDB-2" >nul && set "DB_LIB_DIR=dolphindb-2.0" && goto :eof +echo.%DB_SWITCH%|findstr /b /c:"SQLite" >nul && set "DB_LIB_DIR=sqlite" && goto :eof +goto :eof + :append -set CLASSPATH=%CLASSPATH%;%1 +set CLASSPATH=%CLASSPATH%;%~1 goto :eof @@ -84,7 +122,7 @@ goto :eof rem echo CLASSPATH: %CLASSPATH% -"%JAVA_HOME%\bin\java" %JAVA_OPTS% -cp .;./lib/* %MAIN_CLASS% -cf %BENCHMARK_HOME%/conf +"%JAVA_HOME%\bin\java" %JAVA_OPTS% -cp "%CLASSPATH%" %MAIN_CLASS% -cf %BENCHMARK_HOME%/conf goto finally :err diff --git a/configuration/bin/startup.sh b/configuration/bin/startup.sh index aaa3b9caa..483b8fccb 100755 --- a/configuration/bin/startup.sh +++ b/configuration/bin/startup.sh @@ -98,10 +98,78 @@ echo Using configuration file: "${benchmark_conf}" # set MAIN_CLASS MAIN_CLASS=cn.edu.tsinghua.iot.benchmark.App # set CLASSPATH +# all-in-one 布局（存在 lib/core 目录）时： +# 先加载 DB_SWITCH 对应的专属目录 lib/，再加载 lib/core， +# 保证各模块优先使用自己版本的同名依赖；FakeDB/SelfCheck 仅用 lib/core。 +# 用 classpath 通配符（lib/

/*）由 java 自行展开 jar； +# Windows（Git Bash/MSYS）下路径转 C:/ 形式、分隔符用分号，Linux/macOS 用冒号。 +# 旧布局（lib 直接放 jar）保持原行为。 CLASSPATH="" -for f in ${BENCHMARK_HOME}/lib/*.jar; do - CLASSPATH=${CLASSPATH}":"$f -done +case "$(uname -s 2>/dev/null)" in + MINGW*|MSYS*) + CP_SEP=";" + ;; + *) + CP_SEP=":" + ;; +esac +if [ -d "${BENCHMARK_HOME}/lib/core" ]; then + CONF_FILE="${benchmark_conf}" + if [ -d "${benchmark_conf}" ]; then + CONF_FILE="${benchmark_conf}/config.properties" + fi + DB_SWITCH=$(grep -E '^DB_SWITCH=' "${CONF_FILE}" 2>/dev/null | head -n1 | cut -d'=' -f2- | tr -d ' ') + if [ -z "${DB_SWITCH}" ]; then + DB_SWITCH=$(grep -E '^#[[:space:]]*DB_SWITCH=' "${CONF_FILE}" 2>/dev/null | head -n1 | sed 's/^#[[:space:]]*//' | cut -d'=' -f2- | tr -d ' ') + fi + DB_LIB_DIR="" + case "${DB_SWITCH}" in + IoTDB-200-*) DB_LIB_DIR="iotdb-2.0" ;; + IoTDB-130-*) DB_LIB_DIR="iotdb-1.3" ;; + InfluxDB-2*) DB_LIB_DIR="influxdb-2.0" ;; + InfluxDB*) DB_LIB_DIR="influxdb" ;; + OpenTSDB*) DB_LIB_DIR="opentsdb" ;; + CnosDB*) DB_LIB_DIR="cnosdb" ;; + KairosDB*) DB_LIB_DIR="kairosdb" ;; + TimescaleDB-cluster*) DB_LIB_DIR="timescaledb-cluster" ;; + TimescaleDB*) DB_LIB_DIR="timescaledb" ;; + TDengine-3*) DB_LIB_DIR="tdengine-3.0" ;; + TDengine*) DB_LIB_DIR="tdengine" ;; + QuestDB*) DB_LIB_DIR="questdb" ;; + MsSqlServer*) DB_LIB_DIR="mssqlserver" ;; + VictoriaMetrics*) DB_LIB_DIR="victoriametrics" ;; + DolphinDB-3*) DB_LIB_DIR="dolphindb-3.0" ;; + DolphinDB-2*) DB_LIB_DIR="dolphindb-2.0" ;; + SQLite*) DB_LIB_DIR="sqlite" ;; + *) DB_LIB_DIR="" ;; + esac + CP_HOME="${BENCHMARK_HOME}" + if [ "${CP_SEP}" = ";" ]; then + CP_HOME="$(cd "${BENCHMARK_HOME}" && pwd -W)" + fi + CP_DB_LIBS="" + if [ -n "${DB_LIB_DIR}" ] && [ -d "${BENCHMARK_HOME}/lib/${DB_LIB_DIR}" ]; then + CP_DB_LIBS="${CP_HOME}/lib/${DB_LIB_DIR}/*" + fi + # conf 目录也放入 classpath：SLF4J 全项目统一 2.x 生态后，默认绑定为 + # lib/core 的 slf4j-reload4j（配置走 conf/log4j.properties）；iotdb 模块 + # 自带 logback 1.3.16、dolphindb 模块自带 slf4j-log4j12，经「模块目录 + # 优先」规则确定性生效。core 目录刻意只携带一个绑定——多个绑定并存时 + # SLF4J 2.x 按 classpath 顺序选取，而 JVM 对通配符展开顺序不作保证。 + if [ -n "${CP_DB_LIBS}" ]; then + CLASSPATH="${CP_DB_LIBS}${CP_SEP}${CP_HOME}/lib/core/*${CP_SEP}${CP_HOME}/conf" + else + CLASSPATH="${CP_HOME}/lib/core/*${CP_SEP}${CP_HOME}/conf" + fi +else + CP_HOME="${BENCHMARK_HOME}" + if [ "${CP_SEP}" = ";" ]; then + CP_HOME="$(cd "${BENCHMARK_HOME}" && pwd -W)" + fi + for f in ${CP_HOME}/lib/*.jar; do + CLASSPATH=${CLASSPATH}${CP_SEP}$f + done +fi # set benchmark_parms benchmark_parms="$benchmark_parms -Duser.timezone=GMT+8" diff --git a/configuration/conf/log4j.properties b/configuration/conf/log4j.properties new file mode 100644 index 000000000..c334e9dd4 --- /dev/null +++ b/configuration/conf/log4j.properties @@ -0,0 +1,14 @@ +# log4j 1.2 configuration (compatible with reload4j). +# Used when SLF4J selects the reload4j binding on the classpath, e.g. the +# iotdb modules whose lib ships logback-classic 1.3.x (invisible to +# slf4j-api 1.7) so SLF4J falls back to slf4j-reload4j. Without this file +# log4j warns "No appenders could be found" and every LOGGER output +# (including the latency matrix) is silently dropped. +# The conf directory is on the classpath (see bin/startup.sh), so log4j 1.2 +# picks this file up automatically. +log4j.rootLogger=INFO, CONSOLE + +log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender +log4j.appender.CONSOLE.target=System.out +log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout +log4j.appender.CONSOLE.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %c{1}:%L - %m%n diff --git a/core/pom.xml b/core/pom.xml index eeaed16d1..1851f245e 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -92,6 +92,66 @@ opencsv 5.5.2 + + + org.slf4j + jcl-over-slf4j + ${org.slf4j.version} + + + org.slf4j + jul-to-slf4j + ${org.slf4j.version} + + + org.slf4j + slf4j-api + ${org.slf4j.version} + + + org.slf4j + slf4j-reload4j + ${org.slf4j.version} + + + iot-benchmark-core + + + org.apache.maven.plugins + maven-assembly-plugin + 3.1.0 + + + + server-assembly + package + + single + + + + src/assembly/assembly.xml + + false + + + true + true + + + + + + + + + diff --git a/core/src/assembly/assembly.xml b/core/src/assembly/assembly.xml new file mode 100644 index 000000000..d6d623635 --- /dev/null +++ b/core/src/assembly/assembly.xml @@ -0,0 +1,71 @@ + + + + core + + dir + zip + + true + + + lib + + + + + ${maven.multiModuleProjectDirectory}/configuration/bin/ + bin + 0755 + + + ${maven.multiModuleProjectDirectory}/configuration/conf/ + conf + + + + + ${maven.multiModuleProjectDirectory}/configuration/benchmark.bat + + + ${maven.multiModuleProjectDirectory}/configuration/benchmark.sh + 0755 + + + ${maven.multiModuleProjectDirectory}/configuration/rep-benchmark.sh + 0755 + + + + ${maven.multiModuleProjectDirectory}/configuration/cli-benchmark.sh + 0755 + + + ${maven.multiModuleProjectDirectory}/configuration/routine + + + ${maven.multiModuleProjectDirectory}/LICENSE + + + ${project.build.outputDirectory}/git.properties + + + diff --git a/dolphindb-2.0/pom.xml b/dolphindb-2.0/pom.xml index 62cba54e7..bdb3d6c9f 100644 --- a/dolphindb-2.0/pom.xml +++ b/dolphindb-2.0/pom.xml @@ -23,6 +23,14 @@ com.dolphindb dolphindb-javaapi 2.00.11.1 + + + + org.slf4j + slf4j-simple + + org.slf4j diff --git a/dolphindb-3.0/pom.xml b/dolphindb-3.0/pom.xml index 93575ed47..5a09bad39 100644 --- a/dolphindb-3.0/pom.xml +++ b/dolphindb-3.0/pom.xml @@ -23,6 +23,14 @@ com.dolphindb dolphindb-javaapi 3.00.0.2 + + + + org.slf4j + slf4j-simple + + org.slf4j diff --git a/iotdb-1.3/pom.xml b/iotdb-1.3/pom.xml index 0c3ed804d..f6e37c618 100644 --- a/iotdb-1.3/pom.xml +++ b/iotdb-1.3/pom.xml @@ -69,6 +69,14 @@ iotdb-session ${iotdb.version} + + + org.slf4j + slf4j-api + ${org.slf4j.version} + ch.qos.logback logback-classic diff --git a/iotdb-2.0/pom.xml b/iotdb-2.0/pom.xml index 49eaba286..fd6163723 100644 --- a/iotdb-2.0/pom.xml +++ b/iotdb-2.0/pom.xml @@ -69,6 +69,14 @@ iotdb-session ${iotdb.version} + + + org.slf4j + slf4j-api + ${org.slf4j.version} + ch.qos.logback logback-classic diff --git a/mssqlserver/src/main/java/cn.edu.tsinghua.iotdb.benchmark.mssqlserver/MsSQLServerDB.java b/mssqlserver/src/main/java/cn.edu.tsinghua.iotdb.benchmark.mssqlserver/MsSQLServerDB.java index 10d10c17d..ca9f65709 100644 --- a/mssqlserver/src/main/java/cn.edu.tsinghua.iotdb.benchmark.mssqlserver/MsSQLServerDB.java +++ b/mssqlserver/src/main/java/cn.edu.tsinghua.iotdb.benchmark.mssqlserver/MsSQLServerDB.java @@ -32,7 +32,9 @@ import java.sql.Statement; import java.sql.Timestamp; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; +import java.util.Set; public class MsSQLServerDB implements IDatabase { private static final Logger LOGGER = LoggerFactory.getLogger(MsSQLServerDB.class); @@ -68,9 +70,11 @@ public class MsSQLServerDB implements IDatabase { "SELECT * from %s where pk_fk_Id in (?) and pk_TimeStamp >= ? and pk_TimeStamp <= ? and value > ? order by pk_TimeStamp desc", }; - private PreparedStatement[] insertStatements = new PreparedStatement[6]; + // 槽位数 = SensorType 枚举值个数；此前写死 6，枚举扩展后 ordinal 越界（ArrayIndexOutOfBounds） + private PreparedStatement[] insertStatements = new PreparedStatement[SensorType.values().length]; // first: sensorType second: query index - private PreparedStatement[][] queryStatements = new PreparedStatement[6][10]; + private PreparedStatement[][] queryStatements = + new PreparedStatement[SensorType.values().length][10]; private static final String DELETE_TABLE = "drop table if exists %s_%s"; private DBConfig dbConfig; @@ -94,7 +98,10 @@ public void init() throws TsdbException { + ":" + dbConfig.getPORT().get(0) + ";DataBaseName=" - + dbConfig.getDB_NAME(), + + dbConfig.getDB_NAME() + // mssql-jdbc 12.2+ 默认 encrypt=true，容器/自签证书环境会因 PKIX 校验失败 + // 连不上；压测场景显式关闭加密。 + + ";encrypt=false", dbConfig.getUSERNAME(), dbConfig.getPASSWORD()); @@ -135,8 +142,13 @@ public void init() throws TsdbException { public void cleanup() throws TsdbException { try { Statement statement = connection.createStatement(); + // 类型映射存在多对一（如 TEXT 与 default 分支都映射 text），按唯一表名 drop 一次 + Set dropped = new HashSet<>(); for (SensorType sensorType : SensorType.values()) { - statement.execute(String.format(DELETE_TABLE, dbConfig.getDB_NAME(), typeMap(sensorType))); + if (dropped.add(typeMap(sensorType))) { + statement.execute( + String.format(DELETE_TABLE, dbConfig.getDB_NAME(), typeMap(sensorType))); + } } statement.close(); } catch (SQLException sqlException) { @@ -170,11 +182,17 @@ public Double registerSchema(List schemaList) throws TsdbException try { start = System.nanoTime(); Statement statement = connection.createStatement(); + // 类型映射存在多对一（TEXT 与 default 分支都映射 text、FLOAT 与 DOUBLE 都映射 float）， + // 按唯一表名建表，避免 "already an object named" 撞表 + Set created = new HashSet<>(); for (SensorType sensorType : SensorType.values()) { if (sensorType == SensorType.DOUBLE) { continue; } String sysType = typeMap(sensorType); + if (!created.add(sysType)) { + continue; + } String createSQL = String.format( CREATE_TABLE, diff --git a/opentsdb/src/main/java/cn/edu/tsinghua/iot/benchmark/opentsdb/HttpRequest.java b/opentsdb/src/main/java/cn/edu/tsinghua/iot/benchmark/opentsdb/HttpRequest.java index bf54b5662..fdc3b0b83 100644 --- a/opentsdb/src/main/java/cn/edu/tsinghua/iot/benchmark/opentsdb/HttpRequest.java +++ b/opentsdb/src/main/java/cn/edu/tsinghua/iot/benchmark/opentsdb/HttpRequest.java @@ -21,11 +21,13 @@ import java.io.BufferedReader; import java.io.IOException; +import java.io.InputStream; import java.io.InputStreamReader; import java.io.PrintWriter; import java.net.HttpURLConnection; import java.net.URL; import java.net.URLConnection; +import java.nio.charset.StandardCharsets; /** From https://www.cnblogs.com/zhuawang/archive/2012/12/08/2809380.html */ public class HttpRequest { @@ -94,7 +96,7 @@ public static String sendPost(String url, String param) throws IOException { try { URL realUrl = new URL(url); // 打开和URL之间的连接 - URLConnection conn = realUrl.openConnection(); + HttpURLConnection conn = (HttpURLConnection) realUrl.openConnection(); // 设置通用的请求属性 conn.setRequestProperty("accept", "*/*"); conn.setRequestProperty("connection", "Keep-Alive"); @@ -103,12 +105,13 @@ public static String sendPost(String url, String param) throws IOException { // 发送POST请求必须设置如下两行 conn.setDoOutput(true); conn.setDoInput(true); - // 获取URLConnection对象对应的输出流 - out = new PrintWriter(conn.getOutputStream()); - // 发送请求参数 - if (param != null) out.print(param); - // flush输出流的缓冲 - out.flush(); + // OpenTSDB 2.x 的 Netty HTTP 层不支持 chunked 传输编码：批量写入的 JSON 体 + // （约 90KB）超过 HttpURLConnection 内部缓冲后会隐式走 chunked，被服务端 + // 400 拒绝（Chunked request not supported）。显式固定长度强制 Content-Length， + // 并统一按 UTF-8 编码输出 + byte[] body = (param == null ? "" : param).getBytes(StandardCharsets.UTF_8); + conn.setFixedLengthStreamingMode(body.length); + conn.getOutputStream().write(body); // 定义BufferedReader输入流来读取URL的响应 in = new BufferedReader(new InputStreamReader(conn.getInputStream())); String line; @@ -153,7 +156,27 @@ public static String sendDelete(String url, String param) throws IOException { while ((line = in.readLine()) != null) { result += line; } - } catch (Exception e) { + } catch (IOException e) { + // 非 2xx 时 getInputStream 抛 IOException；把错误响应体拼进异常消息， + // 便于调用方按错误内容（如空库时的 No such name）做容错处理 + try { + HttpURLConnection errorConn = (HttpURLConnection) new URL(url).openConnection(); + InputStream errorStream = errorConn.getErrorStream(); + if (errorStream != null) { + StringBuilder body = new StringBuilder(); + BufferedReader errorReader = new BufferedReader(new InputStreamReader(errorStream)); + String line; + while ((line = errorReader.readLine()) != null) { + body.append(line); + } + errorReader.close(); + throw new IOException(e.getMessage() + " | body: " + body); + } + } catch (IOException inner) { + if (inner.getMessage() != null && inner.getMessage().contains(" | body: ")) { + throw inner; + } + } throw e; } // 使用finally块来关闭输出流、输入流 diff --git a/opentsdb/src/main/java/cn/edu/tsinghua/iot/benchmark/opentsdb/OpenTSDB.java b/opentsdb/src/main/java/cn/edu/tsinghua/iot/benchmark/opentsdb/OpenTSDB.java index fd67356d5..45597402a 100644 --- a/opentsdb/src/main/java/cn/edu/tsinghua/iot/benchmark/opentsdb/OpenTSDB.java +++ b/opentsdb/src/main/java/cn/edu/tsinghua/iot/benchmark/opentsdb/OpenTSDB.java @@ -61,7 +61,9 @@ public OpenTSDB(DBConfig dbConfig) { } sensorRandom = new Random(1 + config.getQUERY_SEED()); String openUrl = dbConfig.getHOST().get(0) + ":" + dbConfig.getPORT().get(0); - writeUrl = openUrl + "/api/put?summary "; + // ?summary 模式要求每个数据点带 summary 字段，而 OpenTSDBDataModel 没有该字段， + // 在 OpenTSDB 2.4.1 上会 400；普通 put 模式配合 auto_create_metrics 即可写入 + writeUrl = openUrl + "/api/put"; queryUrl = openUrl + "/api/query"; } @@ -84,6 +86,12 @@ public void cleanup() throws TsdbException { LOGGER.info("Delete old data of {} ...", metricName); LOGGER.debug("Delete request response: {}", response); } catch (IOException e) { + // 空库首跑时 metric 不存在，OpenTSDB 返回 400（No such name）， + // 此时没有旧数据可删，跳过即可，不应中断压测 + if (e.getMessage() != null && e.getMessage().contains("No such name")) { + LOGGER.warn("Metric {} does not exist yet, skip delete.", metricName); + continue; + } LOGGER.error("Delete old OpenTSDB metric {} failed. Error: {}", metricName, e.getMessage()); throw new TsdbException(e); } diff --git a/pom.xml b/pom.xml index 4f19476c3..0619fb4ef 100644 --- a/pom.xml +++ b/pom.xml @@ -29,6 +29,7 @@ mssqlserver sqlite verification + all-in-one @@ -40,14 +41,18 @@ ${targetJavaVersion} UTF-8 4.13.1 - 1.1.3 + + 1.3.16 1.3.1 false false false 2.43.0 1.22.0 - 1.7.36 + + 2.0.9 diff --git a/timescaledb-cluster/pom.xml b/timescaledb-cluster/pom.xml index d58f14c61..b0284c966 100644 --- a/timescaledb-cluster/pom.xml +++ b/timescaledb-cluster/pom.xml @@ -39,6 +39,11 @@ slf4j-reload4j ${org.slf4j.version} + + org.postgresql + postgresql + 42.7.11 + diff --git a/timescaledb/pom.xml b/timescaledb/pom.xml index 981ce4c4a..75aeeb290 100644 --- a/timescaledb/pom.xml +++ b/timescaledb/pom.xml @@ -39,6 +39,11 @@ slf4j-reload4j ${org.slf4j.version} + + org.postgresql + postgresql + 42.7.11 + diff --git a/verification/pom.xml b/verification/pom.xml index 08070c853..2dc1da8d4 100644 --- a/verification/pom.xml +++ b/verification/pom.xml @@ -27,9 +27,16 @@ + ch.qos.logback logback-classic - 1.2.13 + ${logback.version} + + + + org.slf4j + slf4j-api + ${org.slf4j.version}