Skip to content

SPC Bin Counters and mmap #35

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
davideberius wants to merge 3 commits into
base: main
Choose a base branch
from
Open

SPC Bin Counters and mmap #35

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
483c3f9
This is an update to the SPC code to add two major features:
Aug 23, 2018
53a627e
This update introduces changes to how watermark counters
Jan 23, 2020
6fc0a12
This update reflects the recent removal of orte in
May 11, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 7 additions & 4 deletions examples/Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
# University Research and Technology
# Corporation. All rights reserved.
# Copyright (c) 2004-2018 The University of Tennessee and The University
# Copyright (c) 2004-2020 The University of Tennessee and The University
# of Tennessee Research Foundation. All rights
# reserved.
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
Expand Down Expand Up @@ -65,14 +65,15 @@ EXAMPLES = \
oshmem_max_reduction \
oshmem_strided_puts \
oshmem_symmetric_data \
spc_example
spc_mpit_example \
spc_mmap_example


# Default target. Always build the C MPI examples. Only build the
# others if we have the appropriate Open MPI / OpenSHMEM language
# bindings.

all: hello_c ring_c connectivity_c spc_example
all: hello_c ring_c connectivity_c spc_mpit_example spc_mmap_example
@ if which ompi_info >/dev/null 2>&1 ; then \
$(MAKE) mpi; \
fi
Expand Down Expand Up @@ -127,7 +128,9 @@ ring_c: ring_c.c
$(MPICC) $(CFLAGS) $(LDFLAGS) $? $(LDLIBS) -o $@
connectivity_c: connectivity_c.c
$(MPICC) $(CFLAGS) $(LDFLAGS) $? $(LDLIBS) -o $@
spc_example: spc_example.c
spc_mpit_example: spc_mpit_example.c
$(MPICC) $(CFLAGS) $(LDFLAGS) $? $(LDLIBS) -o $@
spc_mmap_example: spc_mmap_example.c
$(MPICC) $(CFLAGS) $(LDFLAGS) $? $(LDLIBS) -o $@

hello_mpifh: hello_mpifh.f
Expand Down
122 changes: 0 additions & 122 deletions examples/spc_example.c
View file
Open in desktop

This file was deleted.

222 changes: 222 additions & 0 deletions examples/spc_mmap_example.c
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,222 @@
/*
* Copyright (c) 2020 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
*
* Simple example usage of SPCs through an mmap'd file.
*/

#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <unistd.h>
#include <sys/mman.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <errno.h>

#include <mpi.h>

/* This structure will help us store all of the offsets for each
* counter that we want to print out.
*/
typedef struct spc_s {
char name[128];
int offset;
int rules_offset;
int bins_offset;
} spc_t;

int main(int argc, char **argv)
{
if(argc < 4) {
printf("Usage: ./spc_mmap_test [num_messages] [message_size] [XML string]\n");
return -1;
}

MPI_Init(NULL, NULL);

int i, num_messages = atoi(argv[1]), message_size = atoi(argv[2]), rank, shm_fd;
char *buf = (char*)malloc(message_size * sizeof(char));

MPI_Request *requests = (MPI_Request*)malloc(num_messages * sizeof(MPI_Request));
MPI_Status *statuses = (MPI_Status*)malloc(num_messages * sizeof(MPI_Status));
MPI_Status status;

MPI_Comm_rank(MPI_COMM_WORLD, &rank);

int retval, shm_file_size, num_counters, freq_mhz;
long long value;
char filename[128], shm_filename[128], line[128], *token;

char hostname[128];
gethostname(hostname, 128);

char *nodename;
nodename = strtok(hostname, ".");

char *xml_string = argv[3];
snprintf(filename, 128, "/dev/shm/spc_data.%s.%s.%d.xml", nodename, xml_string, rank);

FILE *fptr = NULL;
void *data_ptr;
spc_t *spc_data;

if(NULL == (fptr = fopen(filename, "r"))) {
printf("Couldn't open xml file.\n");
MPI_Finalize();
return -1;
} else {
printf("[%d] Successfully opened the XML file!\n", rank);
}

/* The following is to read the formatted XML file to get the basic
* information we need to read the shared memory file and properly
* format some counters.
*/
char tmp_filename[128];
fgets(line, 128, fptr);
fgets(line, 128, fptr);

fgets(line, 128, fptr);
token = strtok(line, ">");
token = strtok(NULL, "<");
sscanf(token, "%s", shm_filename);

if(rank == 0) {
printf("shm_filename: %s\n", shm_filename);
}

fgets(line, 128, fptr);
token = strtok(line, ">");
token = strtok(NULL, "<");
sscanf(token, "%d", &shm_file_size);
if(rank == 0) {
printf("shm_file_size: %d\n", shm_file_size);
}

fgets(line, 128, fptr);
token = strtok(line, ">");
token = strtok(NULL, "<");
sscanf(token, "%d", &num_counters);
if(rank == 0) {
printf("num_counters: %d\n", num_counters);
}

fgets(line, 128, fptr);
token = strtok(line, ">");
token = strtok(NULL, "<");
sscanf(token, "%d", &freq_mhz);
if(rank == 0) {
printf("freq_mhz: %d\n", freq_mhz);
}

if(-1 == (shm_fd = open(shm_filename, O_RDONLY))){
printf("\nCould not open file '%s'... Error String: %s\n", shm_filename, strerror(errno));
return -1;
} else {
if(MAP_FAILED == (data_ptr = mmap(0, 8192, PROT_READ, MAP_SHARED, shm_fd, 0))) {
printf("Map failed :(\n");
return -1;
}
printf("Successfully mmap'd file!\n");
}

spc_data = (spc_t*)malloc(num_counters * sizeof(spc_t));

for(i = 0; i < num_counters; i++) {
fgets(line, 128, fptr); /* Counter begin header */
/* This should never happen... */
if(strcmp(line,"</SPC>\n") == 0) {
printf("Parsing ended prematurely. There weren't enough counters.\n");
break;
}

fgets(line, 128, fptr); /* Counter name header */
token = strtok(line, ">");
token = strtok(NULL, "<");
sscanf(token, "%s", spc_data[i].name); /* Counter name */

fgets(line, 128, fptr); /* Counter value offset header */
token = strtok(line, ">");
token = strtok(NULL, "<");
sscanf(token, "%d", &spc_data[i].offset); /* Counter offset */

fgets(line, 128, fptr); /* Counter rules offset header */
token = strtok(line, ">");
token = strtok(NULL, "<");
sscanf(token, "%d", &spc_data[i].rules_offset); /* Counter rules offset */

fgets(line, 128, fptr); /* Counter bins offset header */
token = strtok(line, ">");
token = strtok(NULL, "<");
sscanf(token, "%d", &spc_data[i].bins_offset); /* Counter bins offset */

fgets(line, 128, fptr); /* Counter end header */
}

fclose(fptr);

/* The following communication pattern is intended to cause a certain
* number of unexpected messages.
*/
if(rank==0) {
for(i=num_messages; i > 0; i--) {
MPI_Isend(buf, message_size, MPI_BYTE, 1, i, MPI_COMM_WORLD, &requests[i-1]);
}
MPI_Send(buf, message_size, MPI_BYTE, 1, 0, MPI_COMM_WORLD);
MPI_Waitall(num_messages, requests, statuses);

MPI_Barrier(MPI_COMM_WORLD);

for(i = 0; i < num_counters; i++) {
if((0 == strcmp(spc_data[i].name, "OMPI_SPC_MATCH_TIME")) || (0 == strcmp(spc_data[i].name, "OMPI_SPC_MATCH_QUEUE_TIME"))) {
value = (*((long long*)(data_ptr+spc_data[i].offset))) / freq_mhz;
} else {
value = *((long long*)(data_ptr+spc_data[i].offset));
}
if(value > 0)
printf("[%d] %s\t%lld\n", rank, spc_data[i].name, value );
}
MPI_Barrier(MPI_COMM_WORLD);
} else {
MPI_Recv(buf, message_size, MPI_BYTE, 0, 0, MPI_COMM_WORLD, &status);
for(i=0; i < num_messages; i++) {
MPI_Recv(buf, message_size, MPI_BYTE, 0, i+1, MPI_COMM_WORLD, &statuses[i]);
}

MPI_Barrier(MPI_COMM_WORLD);
MPI_Barrier(MPI_COMM_WORLD);
for(i = 0; i < num_counters; i++) {
/* These counters are stored in cycles, so we convert them to microseconds.
*/
if((0 == strcmp(spc_data[i].name, "OMPI_SPC_MATCH_TIME")) || (0 == strcmp(spc_data[i].name, "OMPI_SPC_MATCH_QUEUE_TIME"))) {
value = (*((long long*)(data_ptr+spc_data[i].offset))) / freq_mhz;
} else {
value = *((long long*)(data_ptr+spc_data[i].offset));
}
if(value > 0) {
printf("[%d] %s\t%lld\n", rank, spc_data[i].name, value );
if(spc_data[i].rules_offset > 0) {
int j, *rules = (int*)(data_ptr+spc_data[i].rules_offset);
long long *bins = (long long*)(data_ptr+spc_data[i].bins_offset);

for(j = 0; j < rules[0]; j++) {
if(j == rules[0]-1) {
printf("\t> %d\t", rules[j]);
}
else {
printf("\t<= %d\t", rules[j+1]);
}
printf("%lld\n", bins[j]);
}
}
}
}
}

MPI_Finalize();

return 0;
}
Loading