Deploying CDP DMP SST

This section provides detailed instructions on how to deploy HCL CDP DMP SST using the Devtron in the AWS.

Before you begin

Make sure to create dmp-sst secret with required data in the AWS secret manager before deploying CDP DMP SST.

To create the dmp-sst secret in the AWS secret manager, follow the steps below:

  1. Create a dmp-sst secret sample key and value in the dmp-sst secret, and update ConfigMaps data with actual values.
    {
  "KafkaBootstrapServer": "",
  "KafkaTopicList": "dmp_sst_nba,dmp_sst_nba_di_api,analyze_post,dmp_sst_data,dmp_wsap_subscription,webpush_cleanup,dmp_rt_segments,sms_email_events,webpush_sent_metrics,webpush_metrics",
  "KafkaTopicConsumers": "1,1,1,1,1,1,1,1,1,1",
  "KafkaTopicConsumerBatchSize": "100,100,100,100,100,100,100,100,100,100,100",
  "KafkaTopicConsumerDefaultBatchTimeoutMS": "5000,5000,5000,5000,5000,5000,5000,5000,5000,5000",
  "KafkaTopicConsumerDefaultBatchTimeoutTrigger": "60000,10000,10000,10000,10000,10000,10000,10000,10000,10000",
  "KafkaConsumerGroup": "dmp-sst-kafka-k8|max.poll.interval.ms::600000|session.timeout.ms::30000|heartbeat.interval.ms::10000|partition.assignment.strategy::org.apache.kafka.clients.consumer.CooperativeStickyAssignor",
  "KafkaUsername": "",
  "KafkaPassword": "",
  "KafkaProducerHostName": "",
  "DbUsername": "",
  "DbPassword": "",
  "DbHost": "",
  "DbPort": "3307",
  "AerospikeServerAddress": "",
  "AerospikeServerPort": "13000",
  "AerospikeTimeOut": "10000",
  "AerospikeNamespace": "cdpstore",
  "AerospikeTTL": "-1",
  "HbaseZookeeperQuorum": "",
  "HbaseZookeeperClientPort": "",
  "HBaseDBEnable": "false",
  "CookieCacheThreshold": "60",
  "GraphDBEndpoint": "",
  "GraphDBPort": "8182",
  "GraphDBEnable": "false",
  "ZookeeperLockAddress": "",
  "ZookeeperRetrySleepTime": "1000",
  "ZookeeperClients": "15",
  "AlertEmail": "",
  "TsdbHost": "",
  "TsdbPort": "80",
  "SstApiAuthKeys": "",
  "KafkaProducerTopic": "dmp_sst_rt_profile",
  "KafkaProfileProducerTopic": "dmp_sst_profile",
  "type": "realtime",
  "region": "ap-south-1",
  "ZookeeperLockAddressOld": "",
  "AWSRegion": "ap-south-1",
  "AlertFromEmail": "",
  "DbUrl": "",
  "DbDriver": "org.mariadb.jdbc.Driver",
  "AerospikeUsername": "",
  "AerospikePassword": ""
}

Procedure

  1. Navigate to the Devtron Chart Store, and select the cdp-dmp-sst chart to deploy.
  2. Now, configure and deploy the CDP DMP SST charts.
  3. In the YAML section, update the ConfigMap using below details, and deploy the charts. 
    SSTExecutorServiceMinThreads: "32"
SSTExecutorServiceMaxThreads: "64"
SSTExecutorServiceThreadTTL: "5"
SSTExecutorServiceQueueSize: "10000000"

SRExecutorServiceMinThreads: "128"
SRExecutorServiceMaxThreads: "256"
SRExecutorServiceThreadTTL: "5"
SRExecutorServiceQueueSize: "10000000"

ExecutorServiceMinThreads: "64"
ExecutorServiceMaxThreads: "256"
ExecutorServiceThreadTTL: "5"
ExecutorServiceQueueSize: "10000000"

MaxEventBatchLag: "64"
SRMaxEventBatchLag: "64"
MaxBatchExecuteRetries: "3"

KafkaSstEventsTopic: sst_events
SECURITY_PROTOCOL: SASL_SSL
SASL_MECHANISM: SCRAM-SHA-512

KafkaProducerEventType: "100"
KafkaProducerPoolSize: "16"

KafkaDmpSstBidTopic: dmp_sst_bid
KafkaDmpSstImprTopic: dmp_sst_impr
KafkaDmpSstTaxonomyTopic: dmp_sst_taxonomy
KafkaDmpSegmentsTopic: dmp-segments
KafkaDmpRtSegmentsRefreshTopic: dmp_rt_segments_refresh
KafkaDmpSstRtProfileTopic: dmp_sst_rt_profile
KafkaDmpSstProgImprTopic: dmp_sst_prog_impr
KafkaDmpSstProgClicksTopic: dmp_sst_prog_click
KafkaDmpSstSiteRecoImprTopic: onsite_reco_impr
KafkaDmpSstSiteRecoClicksTopic: onsite_reco_click
KafkaDmpSstSiteNotifImprTopic: onsite_notif_impr
KafkaDmpSstSiteNotifClicksTopic: onsite_notif_click
KafkaDmpSstWebPushMetricsTopic: webpush_metrics
KafkaDmpSstWebPushSntMetricsTopic: webpush_sent_metrics
KafkaDmpSstMetricsTopic: dmp_sst_metrics
KafkaSMSEmailMetricsTopic: sms_email_events
KafkaWebPushCleanupTopic: webpush_cleanup
KafkaAppPushCleanupTopic: apppush_cleanup
KafkaGeofenceTopic: geofence_events
KafkaAnalyzePostTopic: analyze_post
KafkaWhatsappSubscriptionTopic: dmp_wsap_subscription
KafkaDmpSstProfileTopic: dmp_sst_profile
KafkaSstMergeEventTopic: sst_merge_events
KafkaSstBlacklistEventTopic: sst_blacklist_events
KafkaDmpSstNbaDiApiTopic: dmp_sst_nba_di_api
KafkaDmpSstDataTopic: dmp_sst_data
KafkaDmpRtSegmentsTopic: dmp_rt_segments
KafkaDmpSstNbaTopic: dmp_sst_nba
KafkaSstDemoEventsTopic: sst_demo_events

DBDataOidsCapLimit: "25"
MetricLoggingFrequencyMs: "60000"
MaxWaitForConnectionReestablish: "5"

TSDBHostName: <internal-metrics-host>
GmondCollectors: ""
GmondPort: "8652"
HostNameOverride: <internal-host>
MetricPrefix: dmp

GeoLiteCityFile: /disk1/dmp-sst-maps/GeoLite2-City.mmdb
GeoISPFile: /disk1/dmp-sst-maps/GeoIPISP.dat

SSTCronTime: 30 * * * * ?
SSTValidValuesConfig: /mnt/sst/config/valid_values.json
SSTValuesMappingConfig: /mnt/sst/config/values_mapping.json

ValidValuesS3Path: s3://<masked-bucket>/dmp/config/predictivesegments/validvalues.tsv
ValuesMappingS3Path: s3://<masked-bucket>/dmp/config/predictivesegments/mappingvalues.tsv

EncryptionEnabled: "true"

fromEmailId: user@domain.com
toEmailId: user@domain.com

DemoCampaignList: VIZVRM6515,VIZVRM6489
CampaignIncludeList: VIZVRM6499,VIZVRM6529,VIZVRM6525,VIZVRM6533,VIZVRM6534,VIZVRM6530,VIZVRM6528,VIZVRM6527,VIZVRM6526,VIZVRM6536,VIZVRM6537,VIZVRM6538,VIZVRM6539,VIZVRM6540,VIZVRM6542,VIZVRM6543,VIZVRM6552,VIZVRM6553

Dbname: vrm

mapsGenLocation: /disk1/dmp-sst-maps
ValidValuesConfig: /disk1/config/dmp-sst-maps/valid_values.json
ValuesMappingConfig: /disk1/config/dmp-sst-maps/values_mapping.json

mapsCronTime: 0 0/1 * * * ?
deleteOlderBackupIntervalInDays: "2"

APP_LOG_LEVEL: debug
PROFILE_LOG_LEVEL: off
DELETEDOIDS_LOG_LEVEL: off
MERGEDPROFILE_LOG_LEVEL: off
BLACKLISTEDPROFILE_LOG_LEVEL: off
FAILED_LOG_LEVEL: off
INBOUND_LOG_LEVEL: off
OUTBOUND_LOG_LEVEL: off

KafkaS3InboundTopic: sst_inboundEvents
KafkaS3OutboundTopic: sst_outboundEvents
KafkaS3FailedTopic: sst_failedEvents
KafkaS3DeletedOidTopic: sst_deletedOid
KafkaS3ProfileTopic: sst_profile
KafkaS3MergeTopic: sst_mergeProfile
KafkaS3BlacklistTopic: sst_blacklistProfile

KMS_IMPLEMENTATION: AWS

LOG_LEVEL_APP: info
LOG_LEVEL_ROOT: info

ArrayDefaultLimit: "20"
ArrayMaxLimit: "50"

redisHost: <internal-redis-host>
redisPort: "6379"
redisPassword: "<masked_password>"
redisTimeout: "5000"
redisMaxAttempts: "5"
redisMaxWaitMs: "120000"
  4. On successful deployment, validate the deployment as shown below.