From 2d557500eb01ba51a5537b61d44d5439c8b78bab Mon Sep 17 00:00:00 2001 From: gledis69 Date: Mon, 14 Mar 2022 18:32:25 +0300 Subject: [PATCH 1/5] Fix Variable Name For Release Testing --- download-hammerdb.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/download-hammerdb.sh b/download-hammerdb.sh index 604a710..3f763ae 100755 --- a/download-hammerdb.sh +++ b/download-hammerdb.sh @@ -5,8 +5,8 @@ cd "$(dirname "$0")" # no-op if the directory already exists -if test -d "HammerDB-$HAMMERDB_VERSION"; then - echo "HammerDB-$HAMMERDB_VERSION exists, skipping download" 1>&2 +if test -d "HammerDB-$1"; then + echo "HammerDB-$1 exists, skipping download" 1>&2 exit fi From 160aaed032ab9c76cf91c2d3639d0071f8e0c951 Mon Sep 17 00:00:00 2001 From: gledis69 Date: Mon, 14 Mar 2022 19:45:36 +0300 Subject: [PATCH 2/5] Improved feedback and remove if which will not age well --- download-hammerdb.sh | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/download-hammerdb.sh b/download-hammerdb.sh index 3f763ae..f49cd59 100755 --- a/download-hammerdb.sh +++ b/download-hammerdb.sh @@ -10,14 +10,6 @@ if test -d "HammerDB-$1"; then exit fi -# no-op if the directory already exists -if [ "$1" = "master" ]; then - ./download-hammerdb.sh 4.4 - git clone https://github.com/TPC-Council/HammerDB --branch master HammerDB-master - cp -R HammerDB-4.4/{lib,include,bin} HammerDB-master - exit -fi - if [ "$1" == 4.4 ] then SHA1=5347e06baad690336afa6d354330cd59d98343cc @@ -39,7 +31,7 @@ then OUTPUT=HammerDB-3.3-Linux.tar.gz URL=https://github.com/TPC-Council/HammerDB/releases/download/v3.3/$OUTPUT else - echo 'Expects version parameter. Supported versions: 3.3 4.0 4.3' 1>&2 + echo 'Expects version parameter. Supported versions: 3.3 4.0 4.3 4.4' 1>&2 exit 1 fi From 49c2b65e927d7eea2d288435fbb7695b9dc2772d Mon Sep 17 00:00:00 2001 From: gledis69 Date: Thu, 7 Apr 2022 13:21:33 +0300 Subject: [PATCH 3/5] Expose more parameters --- azure/cleanup.sh | 2 +- azure/driver-vm.bicep | 22 ++++++++++++++-------- azure/hyperscale-pre-created.bicep | 7 ++++++- azure/hyperscale.bicep | 6 ++++++ azure/vm.bicep | 5 ++++- 5 files changed, 31 insertions(+), 11 deletions(-) diff --git a/azure/cleanup.sh b/azure/cleanup.sh index d1ded67..177a2f5 100755 --- a/azure/cleanup.sh +++ b/azure/cleanup.sh @@ -3,4 +3,4 @@ set -euxo pipefail cd "$(dirname "$0")" RESOURCE_GROUP=$1 -az group delete -y --name "$RESOURCE_GROUP" --no-wait > /dev/null +# az group delete -y --name "$RESOURCE_GROUP" --no-wait > /dev/null diff --git a/azure/driver-vm.bicep b/azure/driver-vm.bicep index 300beb1..49be136 100644 --- a/azure/driver-vm.bicep +++ b/azure/driver-vm.bicep @@ -4,9 +4,11 @@ param adminUsername string param pgHost string param pgUser string param pgPassword string +param pgPort int param pgVersion string param location string +param zone string param size string param vmName string @@ -25,6 +27,7 @@ param runVirtualUsers int param allWarehouses bool param duration int param rampup int +param delay int param timeprofile bool // Interpolating booleans directly will result in capitalized words, i.e. True/False @@ -50,40 +53,42 @@ sed -i -e "s/vuset vu .*/vuset vu {3}/" run.tcl sed -i -e "s/diset tpcc pg_allwarehouse .*/diset tpcc pg_allwarehouse {4}/" run.tcl sed -i -e "s/diset tpcc pg_duration .*/diset tpcc pg_duration {5}/" run.tcl sed -i -e "s/diset tpcc pg_rampup .*/diset tpcc pg_rampup {6}/" run.tcl -sed -i -e "s/diset tpcc pg_timeprofile .*/diset tpcc pg_timeprofile {7}/" run.tcl +sed -i -e "s/vuset delay .*/vuset delay {7}/" run.tcl +sed -i -e "s/diset tpcc pg_timeprofile .*/diset tpcc pg_timeprofile {8}/" run.tcl ''' -var sedCommands = format(sedCommandsTemplate, buildWarehouses, runWarehouses, buildVirtualUsers, runVirtualUsers, allWarehousesString, duration, rampup, timeprofileString) - +var sedCommands = format(sedCommandsTemplate, buildWarehouses, runWarehouses, buildVirtualUsers, runVirtualUsers, allWarehousesString, duration, rampup, delay, timeprofileString) var driverBootTemplate = ''' echo export PGHOST='{0}' >> .bashrc echo export PGUSER={1} >> .bashrc echo export PGPASSWORD='{2}' >> .bashrc +echo export PGPORT={3} >> .bashrc # Use the same environment variables right now, sourcing bashrc doesn't work # since we are not in an interactive shell export PGHOST='{0}' export PGUSER={1} export PGPASSWORD='{2}' +export PGPORT={3} # Make sure we can open enough connections echo 'ulimit -n "$(ulimit -Hn)"' >> .bashrc cat >> .bashrc << '__ssh_connection_bashrc__' -{3} +{4} __ssh_connection_bashrc__ -sudo apt-get install -y postgresql-client-{4} +sudo apt-get install -y postgresql-client-{5} git clone https://github.com/citusdata/citus-benchmark.git cd citus-benchmark -{5} +{6} while ! psql -c 'select 1'; do echo failed; sleep 1; done -tmux new-session -d -t main -s cloud-init \; send-keys './build-and-run.sh {6}' Enter +tmux new-session -d -t main -s cloud-init \; send-keys './build-and-run.sh {7}' Enter ''' -var driverBootScript = format(driverBootTemplate, pgHost, pgUser, pgPassword, bashrcTmuxAutoAttach, pgVersion, sedCommands, replace(buildAndRunFlags, '\'', '\'\\\'\'')) +var driverBootScript = format(driverBootTemplate, pgHost, pgUser, pgPassword, pgPort, bashrcTmuxAutoAttach, pgVersion, sedCommands, replace(buildAndRunFlags, '\'', '\'\\\'\'')) module vm 'vm.bicep' = { name: '${vmName}-driver-module' @@ -91,6 +96,7 @@ module vm 'vm.bicep' = { adminPublicKey: adminPublicKey adminUsername: adminUsername location: location + zone: zone size: size vmName: vmName nicName: nicName diff --git a/azure/hyperscale-pre-created.bicep b/azure/hyperscale-pre-created.bicep index 1861b70..27846cb 100644 --- a/azure/hyperscale-pre-created.bicep +++ b/azure/hyperscale-pre-created.bicep @@ -4,9 +4,10 @@ param pgAdminPassword string param vmAdminPublicKey string param vmAdminUsername string = 'azureuser' param pgHost string - +param pgPort int = 5432 param location string = resourceGroup().location +param zone string = '1' param buildAndRunFlags string = '' param warehouses int = 1000 @@ -18,6 +19,7 @@ param allWarehouses bool = true param duration int = 60 param rampup int = 5 param timeprofile bool = true +param delay int = 20 // Configuration of the postgres server group @@ -61,7 +63,9 @@ module driverVm 'driver-vm.bicep' = { params: { adminPublicKey: vmAdminPublicKey adminUsername: vmAdminUsername + pgPort: pgPort location: location + zone: zone size: driverSize vmName: driverVmName nicName: driverNicName @@ -81,6 +85,7 @@ module driverVm 'driver-vm.bicep' = { allWarehouses: allWarehouses duration: duration rampup: rampup + delay: delay timeprofile: timeprofile } } diff --git a/azure/hyperscale.bicep b/azure/hyperscale.bicep index 51d9a5f..cfe9399 100644 --- a/azure/hyperscale.bicep +++ b/azure/hyperscale.bicep @@ -1,10 +1,12 @@ @secure() param pgAdminPassword string +param pgPort int = 5432 @secure() param vmAdminPublicKey string param vmAdminUsername string = 'azureuser' param location string = resourceGroup().location +param zone string = '1' param buildAndRunFlags string = '' param warehouses int = 1000 @@ -15,6 +17,7 @@ param runVirtualUsers int = 250 param allWarehouses bool = true param duration int = 60 param rampup int = 3 +param delay int = 20 param timeprofile bool = true @@ -106,6 +109,7 @@ module driverVm 'driver-vm.bicep' = { adminPublicKey: vmAdminPublicKey adminUsername: vmAdminUsername location: location + zone: zone size: driverSize vmName: driverVmName nicName: driverNicName @@ -117,6 +121,7 @@ module driverVm 'driver-vm.bicep' = { pgUser: 'citus' pgPassword: pgAdminPassword pgVersion: pgVersion + pgPort: pgPort buildAndRunFlags: '--name "${namePrefix}" ${buildAndRunFlags}' buildWarehouses: buildWarehouses runWarehouses: runWarehouses @@ -125,6 +130,7 @@ module driverVm 'driver-vm.bicep' = { allWarehouses: allWarehouses duration: duration rampup: rampup + delay: delay timeprofile: timeprofile } } diff --git a/azure/vm.bicep b/azure/vm.bicep index 6edb9da..f6c41e0 100644 --- a/azure/vm.bicep +++ b/azure/vm.bicep @@ -2,6 +2,7 @@ param adminPublicKey string param adminUsername string param location string +param zone string param size string param vmName string @@ -50,12 +51,13 @@ resource nic 'Microsoft.Network/networkInterfaces@2021-03-01' = { resource ip 'Microsoft.Network/publicIpAddresses@2019-02-01' = { name: ipName + zones: array( zone ) location: location properties: { publicIPAllocationMethod: 'Static' } sku: { - name: 'Basic' + name: 'Standard' } } @@ -136,6 +138,7 @@ var driverBootScript = format(bootTemplate, adminUsername, tmuxConf, bootScript) resource vm 'Microsoft.Compute/virtualMachines@2021-07-01' = { name: vmName location: location + zones: array( zone ) properties: { hardwareProfile: { vmSize: size From 87e6be31b8052941a135c845fa077c41d375352d Mon Sep 17 00:00:00 2001 From: gledis69 Date: Wed, 15 Jun 2022 17:07:28 +0300 Subject: [PATCH 4/5] Restore cleanup script --- azure/cleanup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure/cleanup.sh b/azure/cleanup.sh index 177a2f5..d1ded67 100755 --- a/azure/cleanup.sh +++ b/azure/cleanup.sh @@ -3,4 +3,4 @@ set -euxo pipefail cd "$(dirname "$0")" RESOURCE_GROUP=$1 -# az group delete -y --name "$RESOURCE_GROUP" --no-wait > /dev/null +az group delete -y --name "$RESOURCE_GROUP" --no-wait > /dev/null From 6467bce096ffc20e54fbd8094a7832840726d856 Mon Sep 17 00:00:00 2001 From: gledis69 Date: Fri, 18 Nov 2022 15:56:23 +0300 Subject: [PATCH 5/5] Update to v5 VM-s --- azure/cleanup.sh | 2 +- azure/hyperscale-pre-created.bicep | 2 +- azure/hyperscale.bicep | 2 +- azure/iaas-postgres.bicep | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/azure/cleanup.sh b/azure/cleanup.sh index d1ded67..177a2f5 100755 --- a/azure/cleanup.sh +++ b/azure/cleanup.sh @@ -3,4 +3,4 @@ set -euxo pipefail cd "$(dirname "$0")" RESOURCE_GROUP=$1 -az group delete -y --name "$RESOURCE_GROUP" --no-wait > /dev/null +# az group delete -y --name "$RESOURCE_GROUP" --no-wait > /dev/null diff --git a/azure/hyperscale-pre-created.bicep b/azure/hyperscale-pre-created.bicep index 27846cb..73047c9 100644 --- a/azure/hyperscale-pre-created.bicep +++ b/azure/hyperscale-pre-created.bicep @@ -27,7 +27,7 @@ param pgVersion string = '14' // Configuration of the VM that runs the benchmark (the driver) // This VM should be pretty big, to make sure it does not become the bottleneck -param driverSize string = 'Standard_D64s_v3' +param driverSize string = 'Standard_D64ds_v5' param sshAllowIpPrefix string = '*' // networking reletaed settings, usually you don't have to change this diff --git a/azure/hyperscale.bicep b/azure/hyperscale.bicep index cfe9399..b14bd3c 100644 --- a/azure/hyperscale.bicep +++ b/azure/hyperscale.bicep @@ -32,7 +32,7 @@ param enableHa bool = false // Configuration of the VM that runs the benchmark (the driver) // This VM should be pretty big, to make sure it does not become the bottleneck -param driverSize string = 'Standard_D64s_v3' +param driverSize string = 'Standard_D64ds_v5' param sshAllowIpPrefix string = '*' // networking reletaed settings, usually you don't have to change this diff --git a/azure/iaas-postgres.bicep b/azure/iaas-postgres.bicep index bb13f67..f867a4a 100644 --- a/azure/iaas-postgres.bicep +++ b/azure/iaas-postgres.bicep @@ -20,14 +20,14 @@ param timeprofile bool = true // Configuration of the postgres server param pgVersion string = '14' -param pgSize string = 'Standard_D8s_v3' +param pgSize string = 'Standard_D8ds_v5' param pgStorageSizeGB int = 512 param pgConfigOptions string = '' // Configuration of the VM that runs the benchmark (the driver) // This VM should be pretty big, to make sure it does not become the bottleneck -param driverSize string = 'Standard_D64s_v3' +param driverSize string = 'Standard_D64ds_v5' param sshAllowIpPrefix string = '*'