Deploying CDP DMP SST

This section provides detailed instructions on how to deploy HCL CDP DMP SST using the Devtron in the AWS.

Prerequisites:

Make sure to create dmp-sst secret with required data in the AWS secret manager before deploying CDP DMP SST.

To create the dmp-sst secret in the AWS secret manager, follow the steps below:

  1. Create a dmp-sst secret sample key and value in the dmp-sst secret, and update ConfigMaps data with actual values.
    {
  "KafkaBootstrapServer": "",
  "KafkaTopicList": "dmp_sst_nba,dmp_sst_nba_di_api,analyze_post,dmp_sst_data,dmp_wsap_subscription,webpush_cleanup,dmp_rt_segments,sms_email_events,webpush_sent_metrics,webpush_metrics",
  "KafkaTopicConsumers": "1,1,1,1,1,1,1,1,1,1",
  "KafkaTopicConsumerBatchSize": "100,100,100,100,100,100,100,100,100,100,100",
  "KafkaTopicConsumerDefaultBatchTimeoutMS": "5000,5000,5000,5000,5000,5000,5000,5000,5000,5000",
  "KafkaTopicConsumerDefaultBatchTimeoutTrigger": "60000,10000,10000,10000,10000,10000,10000,10000,10000,10000",
  "KafkaConsumerGroup": "dmp-sst-kafka-k8|max.poll.interval.ms::600000|session.timeout.ms::30000|heartbeat.interval.ms::10000|partition.assignment.strategy::org.apache.kafka.clients.consumer.CooperativeStickyAssignor",
  "KafkaUsername": "",
  "KafkaPassword": "",
  "KafkaProducerHostName": "",
  "DbUsername": "",
  "DbPassword": "",
  "DbHost": "",
  "DbPort": "3307",
  "AerospikeServerAddress": "",
  "AerospikeServerPort": "13000",
  "AerospikeTimeOut": "10000",
  "AerospikeNamespace": "cdpstore",
  "AerospikeTTL": "-1",
  "HbaseZookeeperQuorum": "",
  "HbaseZookeeperClientPort": "",
  "HBaseDBEnable": "false",
  "CookieCacheThreshold": "60",
  "GraphDBEndpoint": "",
  "GraphDBPort": "8182",
  "GraphDBEnable": "false",
  "ZookeeperLockAddress": "",
  "ZookeeperRetrySleepTime": "1000",
  "ZookeeperClients": "15",
  "AlertEmail": "",
  "TsdbHost": "",
  "TsdbPort": "80",
  "SstApiAuthKeys": "",
  "KafkaProducerTopic": "dmp_sst_rt_profile",
  "KafkaProfileProducerTopic": "dmp_sst_profile",
  "type": "realtime",
  "region": "ap-south-1",
  "ZookeeperLockAddressOld": "",
  "AWSRegion": "ap-south-1",
  "AlertFromEmail": "",
  "DbUrl": "",
  "DbDriver": "org.mariadb.jdbc.Driver",
  "AerospikeUsername": "",
  "AerospikePassword": ""
}

Deploying CDP DMP SST

To deploy the CDP DMP SST, follow the steps below:

  1. Navigate to the Devtron Chart Store, and select the cdp-dmp-sst chart to deploy.

  2. Now, configure and deploy the CDP DMP SST charts.

  3. In the YAML section, update the ConfigMap using below details, and deploy the charts.
    SSTExecutorServiceMinThreads: "32"
SSTExecutorServiceMaxThreads: "64"
SSTExecutorServiceThreadTTL: "5"
SSTExecutorServiceQueueSize: "10000000"

SRExecutorServiceMinThreads: "128"
SRExecutorServiceMaxThreads: "256"
SRExecutorServiceThreadTTL: "5"
SRExecutorServiceQueueSize: "10000000"

ExecutorServiceMinThreads: "64"
ExecutorServiceMaxThreads: "256"
ExecutorServiceThreadTTL: "5"
ExecutorServiceQueueSize: "10000000"

MaxEventBatchLag: "64"
SRMaxEventBatchLag: "64"
MaxBatchExecuteRetries: "3"

KafkaSstEventsTopic: sst_events
SECURITY_PROTOCOL: SASL_SSL
SASL_MECHANISM: SCRAM-SHA-512

KafkaProducerEventType: "100"
KafkaProducerPoolSize: "16"

KafkaDmpSstBidTopic: dmp_sst_bid
KafkaDmpSstImprTopic: dmp_sst_impr
KafkaDmpSstTaxonomyTopic: dmp_sst_taxonomy
KafkaDmpSegmentsTopic: dmp-segments
KafkaDmpRtSegmentsRefreshTopic: dmp_rt_segments_refresh
KafkaDmpSstRtProfileTopic: dmp_sst_rt_profile
KafkaDmpSstProgImprTopic: dmp_sst_prog_impr
KafkaDmpSstProgClicksTopic: dmp_sst_prog_click
KafkaDmpSstSiteRecoImprTopic: onsite_reco_impr
KafkaDmpSstSiteRecoClicksTopic: onsite_reco_click
KafkaDmpSstSiteNotifImprTopic: onsite_notif_impr
KafkaDmpSstSiteNotifClicksTopic: onsite_notif_click
KafkaDmpSstWebPushMetricsTopic: webpush_metrics
KafkaDmpSstWebPushSntMetricsTopic: webpush_sent_metrics
KafkaDmpSstMetricsTopic: dmp_sst_metrics
KafkaSMSEmailMetricsTopic: sms_email_events
KafkaWebPushCleanupTopic: webpush_cleanup
KafkaAppPushCleanupTopic: apppush_cleanup
KafkaGeofenceTopic: geofence_events
KafkaAnalyzePostTopic: analyze_post
KafkaWhatsappSubscriptionTopic: dmp_wsap_subscription
KafkaDmpSstProfileTopic: dmp_sst_profile
KafkaSstMergeEventTopic: sst_merge_events
KafkaSstBlacklistEventTopic: sst_blacklist_events
KafkaDmpSstNbaDiApiTopic: dmp_sst_nba_di_api
KafkaDmpSstDataTopic: dmp_sst_data
KafkaDmpRtSegmentsTopic: dmp_rt_segments
KafkaDmpSstNbaTopic: dmp_sst_nba
KafkaSstDemoEventsTopic: sst_demo_events

DBDataOidsCapLimit: "25"
MetricLoggingFrequencyMs: "60000"
MaxWaitForConnectionReestablish: "5"

TSDBHostName: <internal-metrics-host>
GmondCollectors: ""
GmondPort: "8652"
HostNameOverride: <internal-host>
MetricPrefix: dmp

GeoLiteCityFile: /disk1/dmp-sst-maps/GeoLite2-City.mmdb
GeoISPFile: /disk1/dmp-sst-maps/GeoIPISP.dat

SSTCronTime: 30 * * * * ?
SSTValidValuesConfig: /mnt/sst/config/valid_values.json
SSTValuesMappingConfig: /mnt/sst/config/values_mapping.json

ValidValuesS3Path: s3://<masked-bucket>/dmp/config/predictivesegments/validvalues.tsv
ValuesMappingS3Path: s3://<masked-bucket>/dmp/config/predictivesegments/mappingvalues.tsv

EncryptionEnabled: "true"

fromEmailId: user@domain.com
toEmailId: user@domain.com

DemoCampaignList: VIZVRM6515,VIZVRM6489
CampaignIncludeList: VIZVRM6499,VIZVRM6529,VIZVRM6525,VIZVRM6533,VIZVRM6534,VIZVRM6530,VIZVRM6528,VIZVRM6527,VIZVRM6526,VIZVRM6536,VIZVRM6537,VIZVRM6538,VIZVRM6539,VIZVRM6540,VIZVRM6542,VIZVRM6543,VIZVRM6552,VIZVRM6553

Dbname: vrm

mapsGenLocation: /disk1/dmp-sst-maps
ValidValuesConfig: /disk1/config/dmp-sst-maps/valid_values.json
ValuesMappingConfig: /disk1/config/dmp-sst-maps/values_mapping.json

mapsCronTime: 0 0/1 * * * ?
deleteOlderBackupIntervalInDays: "2"

APP_LOG_LEVEL: debug
PROFILE_LOG_LEVEL: off
DELETEDOIDS_LOG_LEVEL: off
MERGEDPROFILE_LOG_LEVEL: off
BLACKLISTEDPROFILE_LOG_LEVEL: off
FAILED_LOG_LEVEL: off
INBOUND_LOG_LEVEL: off
OUTBOUND_LOG_LEVEL: off

KafkaS3InboundTopic: sst_inboundEvents
KafkaS3OutboundTopic: sst_outboundEvents
KafkaS3FailedTopic: sst_failedEvents
KafkaS3DeletedOidTopic: sst_deletedOid
KafkaS3ProfileTopic: sst_profile
KafkaS3MergeTopic: sst_mergeProfile
KafkaS3BlacklistTopic: sst_blacklistProfile

KMS_IMPLEMENTATION: AWS

LOG_LEVEL_APP: info
LOG_LEVEL_ROOT: info

ArrayDefaultLimit: "20"
ArrayMaxLimit: "50"

redisHost: <internal-redis-host>
redisPort: "6379"
redisPassword: "<masked_password>"
redisTimeout: "5000"
redisMaxAttempts: "5"
redisMaxWaitMs: "120000"

  4. On successful deployment, validate the deployment as shown below.