From e3dc413426e87477553c38c25a6d907ce38e2a8f Mon Sep 17 00:00:00 2001 From: longd3 Date: Fri, 19 Jun 2015 16:44:20 -0700 Subject: [PATCH 1/2] Instrument dynamo queries on research MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit @pims, @benejoseph : can you review please I think we’re missing `research_db` config in all .yml files so this build will probably be broken. Can you fill the missing config @benejoseph, i’m not sure what should be in there ? --- suripu-research/pom.xml | 5 +++++ .../hello/suripu/research/SuripuResearch.java | 5 +++-- .../SuripuResearchConfiguration.java | 20 +++++++++++++++++++ .../suripu-research.dev.yml.example | 9 +++++++++ suripu-research/suripu-research.prod.yml | 9 +++++++++ suripu-research/suripu-research.staging.yml | 9 +++++++++ 6 files changed, 55 insertions(+), 2 deletions(-) diff --git a/suripu-research/pom.xml b/suripu-research/pom.xml index 71175114f..45a761add 100644 --- a/suripu-research/pom.xml +++ b/suripu-research/pom.xml @@ -42,6 +42,11 @@ 1.2.2 true + + com.yammer.metrics + metrics-graphite + 2.2.0 + diff --git a/suripu-research/src/main/java/com/hello/suripu/research/SuripuResearch.java b/suripu-research/src/main/java/com/hello/suripu/research/SuripuResearch.java index 36f828f7e..aa5be69a6 100644 --- a/suripu-research/src/main/java/com/hello/suripu/research/SuripuResearch.java +++ b/suripu-research/src/main/java/com/hello/suripu/research/SuripuResearch.java @@ -9,6 +9,7 @@ import com.hello.suripu.core.ObjectGraphRoot; import com.hello.suripu.core.bundles.KinesisLoggerBundle; import com.hello.suripu.core.clients.AmazonDynamoDBClientFactory; +import com.hello.suripu.core.configuration.DynamoDBTableName; import com.hello.suripu.core.configuration.KinesisLoggerConfiguration; import com.hello.suripu.core.configuration.QueueName; import com.hello.suripu.core.db.AccessTokenDAO; @@ -18,8 +19,8 @@ import com.hello.suripu.core.db.DeviceDAO; import com.hello.suripu.core.db.DeviceDataDAO; import com.hello.suripu.core.db.FeatureStore; -import com.hello.suripu.core.db.UserLabelDAO; import com.hello.suripu.core.db.TrackerMotionDAO; +import com.hello.suripu.core.db.UserLabelDAO; import com.hello.suripu.core.db.util.JodaArgumentFactory; import com.hello.suripu.core.db.util.PostgresIntegerArrayArgumentFactory; import com.hello.suripu.core.filters.CacheFilterFactory; @@ -112,7 +113,7 @@ public void run(final SuripuResearchConfiguration configuration, final Environme final DataLogger activityLogger = kinesisLoggerFactory.get(QueueName.ACTIVITY_STREAM); final AmazonDynamoDBClientFactory featureStoreDynamoDBClientFactory = AmazonDynamoDBClientFactory.create(awsCredentialsProvider); - final AmazonDynamoDB featureDynamoDB = featureStoreDynamoDBClientFactory.getForEndpoint(configuration.getFeaturesDynamoDBConfiguration().getEndpoint()); + final AmazonDynamoDB featureDynamoDB = featureStoreDynamoDBClientFactory.getInstrumented(DynamoDBTableName.FEATURES, FeatureStore.class); final String featureNamespace = (configuration.getDebug()) ? "dev" : "prod"; final FeatureStore featureStore = new FeatureStore(featureDynamoDB, "features", featureNamespace); diff --git a/suripu-research/src/main/java/com/hello/suripu/research/configuration/SuripuResearchConfiguration.java b/suripu-research/src/main/java/com/hello/suripu/research/configuration/SuripuResearchConfiguration.java index 1f5af67a9..4ea755d15 100644 --- a/suripu-research/src/main/java/com/hello/suripu/research/configuration/SuripuResearchConfiguration.java +++ b/suripu-research/src/main/java/com/hello/suripu/research/configuration/SuripuResearchConfiguration.java @@ -2,6 +2,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.hello.suripu.core.configuration.DynamoDBConfiguration; +import com.hello.suripu.core.configuration.GraphiteConfiguration; import com.hello.suripu.core.configuration.KinesisConfiguration; import com.hello.suripu.core.configuration.KinesisLoggerConfiguration; import com.yammer.dropwizard.config.Configuration; @@ -50,6 +51,25 @@ public Boolean getDebug() { return debug; } + @Valid + @NotNull + @JsonProperty("metrics_enabled") + private Boolean metricsEnabled; + + public Boolean getMetricsEnabled() { + return metricsEnabled; + } + + + @Valid + @NotNull + @JsonProperty("graphite") + private GraphiteConfiguration graphite; + + public GraphiteConfiguration getGraphite() { + return graphite; + } + @Valid @NotNull @JsonProperty("kinesis") diff --git a/suripu-research/suripu-research.dev.yml.example b/suripu-research/suripu-research.dev.yml.example index 78517f8f5..fc5e50d89 100644 --- a/suripu-research/suripu-research.dev.yml.example +++ b/suripu-research/suripu-research.dev.yml.example @@ -1,4 +1,13 @@ +metrics_enabled: false debug: true +graphite: + host: carbon.hostedgraphite.com + api_key: 7509c0ff-4db5-4cae-91ee-6e78ff13b336 + reporting_interval_in_seconds: 30 + include_metrics: + - org.skife + - xx-responses + - com.hello sensors_db: # the name of your JDBC driver diff --git a/suripu-research/suripu-research.prod.yml b/suripu-research/suripu-research.prod.yml index ffa425f63..050d01ffb 100644 --- a/suripu-research/suripu-research.prod.yml +++ b/suripu-research/suripu-research.prod.yml @@ -1,4 +1,13 @@ +metrics_enabled: true debug: false +graphite: + host: carbon.hostedgraphite.com + api_key: 7509c0ff-4db5-4cae-91ee-6e78ff13b336 + reporting_interval_in_seconds: 30 + include_metrics: + - org.skife + - xx-responses + - com.hello sensors_db: driverClass: org.postgresql.Driver diff --git a/suripu-research/suripu-research.staging.yml b/suripu-research/suripu-research.staging.yml index 7ef9753fd..6662b83d9 100644 --- a/suripu-research/suripu-research.staging.yml +++ b/suripu-research/suripu-research.staging.yml @@ -1,4 +1,13 @@ +metrics_enabled: false debug: false +graphite: + host: carbon.hostedgraphite.com + api_key: 7509c0ff-4db5-4cae-91ee-6e78ff13b336 + reporting_interval_in_seconds: 30 + include_metrics: + - org.skife + - xx-responses + - com.hello sensors_db: driverClass: org.postgresql.Driver From 42ab35a4b6c51feee4c10aad4aa42544748b076c Mon Sep 17 00:00:00 2001 From: longd3 Date: Fri, 19 Jun 2015 16:55:56 -0700 Subject: [PATCH 2/2] Add metrics config handler in research --- .../hello/suripu/research/SuripuResearch.java | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/suripu-research/src/main/java/com/hello/suripu/research/SuripuResearch.java b/suripu-research/src/main/java/com/hello/suripu/research/SuripuResearch.java index aa5be69a6..11a8a72e9 100644 --- a/suripu-research/src/main/java/com/hello/suripu/research/SuripuResearch.java +++ b/suripu-research/src/main/java/com/hello/suripu/research/SuripuResearch.java @@ -5,6 +5,7 @@ import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; import com.amazonaws.services.dynamodbv2.AmazonDynamoDB; import com.amazonaws.services.kinesis.AmazonKinesisAsyncClient; +import com.google.common.base.Joiner; import com.google.common.collect.ImmutableMap; import com.hello.suripu.core.ObjectGraphRoot; import com.hello.suripu.core.bundles.KinesisLoggerBundle; @@ -26,6 +27,7 @@ import com.hello.suripu.core.filters.CacheFilterFactory; import com.hello.suripu.core.logging.DataLogger; import com.hello.suripu.core.logging.KinesisLoggerFactory; +import com.hello.suripu.core.metrics.RegexMetricPredicate; import com.hello.suripu.core.oauth.OAuthAuthenticator; import com.hello.suripu.core.oauth.OAuthProvider; import com.hello.suripu.core.oauth.stores.PersistentAccessTokenStore; @@ -44,11 +46,15 @@ import com.yammer.dropwizard.jdbi.ImmutableSetContainerFactory; import com.yammer.dropwizard.jdbi.OptionalContainerFactory; import com.yammer.dropwizard.jdbi.bundles.DBIExceptionsBundle; +import com.yammer.metrics.Metrics; +import com.yammer.metrics.reporting.GraphiteReporter; import org.skife.jdbi.v2.DBI; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.List; import java.util.TimeZone; +import java.util.concurrent.TimeUnit; /** * Created by pangwu on 3/2/15. @@ -105,6 +111,27 @@ public void run(final SuripuResearchConfiguration configuration, final Environme clientConfiguration.withConnectionTimeout(200); // in ms clientConfiguration.withMaxErrorRetry(1); + if(configuration.getMetricsEnabled()) { + final String graphiteHostName = configuration.getGraphite().getHost(); + final String apiKey = configuration.getGraphite().getApiKey(); + final Integer interval = configuration.getGraphite().getReportingIntervalInSeconds(); + + final String env = (configuration.getDebug()) ? "dev" : "prod"; + + final String prefix = String.format("%s.%s.%s", apiKey, env, "suripu-research"); + + final List metrics = configuration.getGraphite().getIncludeMetrics(); + final RegexMetricPredicate predicate = new RegexMetricPredicate(metrics); + final Joiner joiner = Joiner.on(", "); + LOGGER.info("Logging the following metrics: {}", joiner.join(metrics)); + + GraphiteReporter.enable(Metrics.defaultRegistry(), interval, TimeUnit.SECONDS, graphiteHostName, 2003, prefix, predicate); + + LOGGER.info("Metrics enabled."); + } else { + LOGGER.warn("Metrics not enabled."); + } + final AWSCredentialsProvider awsCredentialsProvider = new DefaultAWSCredentialsProviderChain(); final AmazonKinesisAsyncClient kinesisClient = new AmazonKinesisAsyncClient(awsCredentialsProvider, clientConfiguration);