using Amazon; using Amazon.CloudSearchDomain; using Amazon.Runtime; using Amazon.S3; using CacheCow.Common; using Couchbase; using Newtonsoft.Json; using PetaPoco; using RateGateAsync; using Sony.Filtr.AdminSearch; using Sony.Filtr.ApolloAPI; using Sony.Filtr.AppleMusic; using Sony.Filtr.Applications; using Sony.Filtr.Buzz; using Sony.Filtr.ContentManagement.Data; using Sony.Filtr.Contracts; using Sony.Filtr.Contracts.Abstractions; using Sony.Filtr.Contracts.Entities; using Sony.Filtr.Contracts.Entities.Buzz; using Sony.Filtr.Core.EditorialPlaylists; using Sony.Filtr.Core.Factory; using Sony.Filtr.Core.FastlyCache; using Sony.Filtr.Core.ResponseCollector; using Sony.Filtr.Core.SonyMusic; using Sony.Filtr.Core.SpotifyRegion; using Sony.Filtr.DeezerAPI; using Sony.Filtr.DistributedCaching; using Sony.Filtr.DspWebAPI; using Sony.Filtr.PlaylistGeneration.Autocomplete; using Sony.Filtr.PlaylistGeneration.MusicInfoStore; using Sony.Filtr.PlaylistSynchronization; using Sony.Filtr.Search; using Sony.Filtr.SpotifyAnalytics; using Sony.Filtr.SpotifyImages; using Sony.Filtr.SpotifyWebAPI; using Sony.Filtr.Tasks; using Sony.Filtr.Tasks.Tasks; using Sony.Filtr.Tasks.Tasks.AppleMusic; using Sony.Filtr.Tasks.Tasks.Spotify; using Sony.Filtr.Utility.Extensions; using Sony.Filtr.Utility.HttpHandlers; using Sony.Filtr.Utility.Logging; using Sony.Filtr.YouTube.Api; using StackExchange.Redis; using StructureMap; using StructureMap.Pipeline; using System; using System.Collections.Generic; using System.Net; using System.Net.Http; using System.Threading; using System.Threading.Tasks; namespace Sony.Filtr.Bootstrap { public class Bootstrapper { private static readonly SemaphoreSlim SyncLock = new SemaphoreSlim(1); private static bool _initialized = false; private static IContainer _container = null; public async Task InitializeAsync() { await SyncLock.WaitAsync(); try { if (_initialized) { return _container; } _container = BuildContainer(); ConfigurePetaPoco(); _initialized = true; } finally { SyncLock.Release(); } return _container; } private static void ConfigurePetaPoco() { Mappers.Register(typeof(Application), new FiltrMapper()); Mappers.Register(typeof(CarouselImage), new FiltrMapper()); Mappers.Register(typeof(ContentBlock), new FiltrMapper()); Mappers.Register(typeof(SpotifyLinkedTrack), new FiltrMapper()); Mappers.Register(typeof(SpotifyTrackPopularityLogValue), new FiltrMapper()); Mappers.Register(typeof(BuzzUser), new FiltrMapper()); } private Container BuildContainer() { var container = new Container(config => { config.Scan(scanner => { scanner.TheCallingAssembly(); scanner.WithDefaultConventions(); }); config.For().Use(); config.For().Use(); config.For().Use(); config.For().Use(); config.For().Use(); config.For().Use(); config.For().Use(); config.For().Use(); config.For().Use(); config.For().Use(); config.For().Use(); ApolloBootstrapperEventSource.Log.ReadSettingsStart(); var bootstrapperSettings = new BootstrapperSettings(); var settings = bootstrapperSettings.GetSettings(); ApolloBootstrapperEventSource.Log.ReadSettingsStop(); ApolloBootstrapperEventSource.Log.ConfigureCouchbaseStart(); ConfigureCouchbase(config, settings); ApolloBootstrapperEventSource.Log.ConfigureCouchbaseStop(); ApolloBootstrapperEventSource.Log.ConfigureLastFmStart(); ConfigureLastFm(config, settings); ApolloBootstrapperEventSource.Log.ConfigureLastFmStop(); ApolloBootstrapperEventSource.Log.ConfigureAmazonStart(); ConfigureAmazon(config, settings); ApolloBootstrapperEventSource.Log.ConfigureAmazonStop(); ApolloBootstrapperEventSource.Log.ConfigureSpotifyStart(); ConfigureSpotifyWebApi(config, settings); ApolloBootstrapperEventSource.Log.ConfigureSpotifyStop(); ApolloBootstrapperEventSource.Log.ConfigureYoutubeStart(); ConfigureYoutubeApi(config, settings); ApolloBootstrapperEventSource.Log.ConfigureYoutubeStop(); ApolloBootstrapperEventSource.Log.ConfigureDeezerStart(); ConfigureDeezerApi(config, settings); ApolloBootstrapperEventSource.Log.ConfigureDeezerStop(); ApolloBootstrapperEventSource.Log.ConfigureRedisStart(); ConfigureRedis(config, settings); ApolloBootstrapperEventSource.Log.ConfigureRedisStop(); ApolloBootstrapperEventSource.Log.ConfigureElasticSearchStart(); ConfigureAdminElasticSearch(config, settings); ApolloBootstrapperEventSource.Log.ConfigureElasticSearchStop(); ApolloBootstrapperEventSource.Log.ConfigureAppleMusicStart(); ConfigureAppleMusic(config, settings); ApolloBootstrapperEventSource.Log.ConfigureAppleMusicStop(); ApolloBootstrapperEventSource.Log.ConfigureTasksStart(); ConfigureTasks(config, settings); ApolloBootstrapperEventSource.Log.ConfigureTasksStop(); ApolloBootstrapperEventSource.Log.ConfigureDatadogStart(); ConfigureDatadog(config, settings); ApolloBootstrapperEventSource.Log.ConfigureDatadogStop(); ApolloBootstrapperEventSource.Log.ConfigureApolloApiStart(); ConfigureApolloApi(config, settings); ApolloBootstrapperEventSource.Log.ConfigureApolloApiStop(); ConfigureHealthCheck(config, settings); }); return container; } private void ConfigureAppleMusic(ConfigurationExpression config, Dictionary settings) { var privateKey = settings.GetValueOrDefault("AppleMusicPrivateKey"); var kid = settings.GetValueOrDefault("AppleMusicKid"); var iss = settings.GetValueOrDefault("AppleMusicIss"); config.ForConcreteType().Configure .Ctor("privateKey").Is(privateKey) .Ctor("kid").Is(kid) .Ctor("iss").Is(iss); } private void ConfigureHealthCheck(ConfigurationExpression config, Dictionary settings) { var jobIds = settings.GetValueOrDefault("HealthcheckJobIds"); var jobIdsDontAlarm = settings.GetValueOrDefault("HealthcheckJobIdsNoAlarm"); config.ForConcreteType().Configure .Ctor("jobIds").Is(jobIds) .Ctor("jobIdsDontAlarm").Is(jobIdsDontAlarm); } private void ConfigureCouchbase(ConfigurationExpression config, Dictionary settings) { config.For().Use(() => new CouchbaseClient()).Singleton().Named("couchbaseDefault"); config.For().Add(() => new CouchbaseClient("couchbaseReserve")).Singleton().Named("couchbaseReserve"); var cachePrefix = settings.GetValueOrDefault("Cache-Prefix"); config.ForConcreteType().Configure .Ctor().Is(cachePrefix); var cachePrefixReserver = settings.GetValueOrDefault("CouchBaseReserve_Cache-Prefix", "Reserve"); config.ForConcreteType().Configure .Ctor().Is(cachePrefixReserver) .Ctor().IsNamedInstance("couchbaseReserve"); } private static void ConfigureDeezerApi(ConfigurationExpression config, Dictionary settings) { var deezerApplicationId = settings.GetValueOrDefault("DeezerApplicationId"); var deezerSecretKey = settings.GetValueOrDefault("DeezerRedirectUrl"); var deezerCredentials = new DeezerApplicationCredentials() { ApplicationId = int.Parse(deezerApplicationId), SecretKey = deezerSecretKey, }; config.ForConcreteType() .Configure .Ctor().Is(deezerCredentials) .Ctor().Is(); } private static void ConfigureYoutubeApi(ConfigurationExpression config, Dictionary settings) { config.ForConcreteType().Configure.Ctor().Is().Ctor().Is("AIzaSyAp0nuSiXSq5rNh--_xUKfPJU_RKOxBkTY"); } private void ConfigureRedis(ConfigurationExpression config, Dictionary settings) { var redisHostname = settings.GetValueOrDefault("Redis-HostName"); if (!string.IsNullOrWhiteSpace(redisHostname)) { config.For().Use(ConnectionMultiplexer.Connect(redisHostname)).Singleton(); config.For().Use("Redis DB:0", c => c.GetInstance().GetDatabase(0, null)); } config.For().Use().Singleton(); } private static void ConfigureSpotifyWebApi(ConfigurationExpression config, Dictionary settings) { string spotifyWebApiClientId = settings.GetValueOrDefault("SpotifyWebApiClientId"); string spotifyWebApiClientSecret = settings.GetValueOrDefault("SpotifyWebApiClientSecret"); int occurencesLimit = Int32.Parse(settings.GetValueOrDefault("SpotifyRateLimitPerSecond", "30")); config.For().Use(() => new SpotifyRateLimiter(new RateGate(occurencesLimit, TimeSpan.FromSeconds(1)))).Singleton(); config.ForConcreteType() .Configure .Ctor("clientId").Is(spotifyWebApiClientId) .Ctor("clientSecret").Is(spotifyWebApiClientSecret); var baseHandler = new HttpClientHandler() { UseCookies = false, AutomaticDecompression = DecompressionMethods.Deflate | DecompressionMethods.GZip }; var reusableImageHttpClient = new HttpClient(baseHandler); config.ForConcreteType().Configure.Ctor().Is(reusableImageHttpClient); } private static void ConfigureApolloApi(ConfigurationExpression config, Dictionary settings) { string apolloUrl = settings.GetValueOrDefault("ApolloUrl"); string clientId = settings.GetValueOrDefault("ApolloClientId"); string clientSecret = settings.GetValueOrDefault("ApolloClientSecret"); string audience = settings.GetValueOrDefault("ApolloAudience"); //"https://apollo-private-api/";// "https://api.sma.stream/"; string grantType = "client_credentials"; int maxSpotifyIdsInIsSonyRequest = Int32.Parse(settings.GetValueOrDefault("ApolloIsSonyMaxIdsInRequest")); int maxIdsInGetTracksRequest = Int32.Parse(settings.GetValueOrDefault("ApolloGetTracksMaxIdsInRequest")); int maxAlbumsByIdsRequest = 20; int threadRateLimit = Int32.Parse(settings.GetValueOrDefault("UpdatePlaylistInfo_SpotifyWebApi_ThreadRateLimit", "5")); string tokenServiceUrl = settings.GetValueOrDefault("ApolloTokenServiceURL");// "https://prod-dna-apps.auth0.com/oauth/token"; string apolloAuthPostPayload = JsonConvert.SerializeObject(new { client_id = clientId, client_secret = clientSecret, audience = audience, grant_type = grantType }); config.For().Use("ApolloWebApi", c => { var baseHandler = new HttpClientHandler() { UseCookies = false, AutomaticDecompression = DecompressionMethods.Deflate | DecompressionMethods.GZip }; var loggingHandler = new ApolloLoggingMessageHandler(baseHandler); var rateLimitHandler = new ThreadRateLimitHttpHandler(threadRateLimit) { InnerHandler = loggingHandler }; var retryHandler = new RetryHttpHandler(2, TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(1), rateLimitHandler); var authHandler = new ApolloAuthTokenMessageHandler(apolloAuthPostPayload, tokenServiceUrl, retryHandler); var httpClient = new HttpClient(authHandler) { BaseAddress = new Uri(apolloUrl) }; return new ApolloWebApi( httpClient, new ApolloApiSettings( maxSpotifyIdsInIsSonyRequest, maxIdsInGetTracksRequest, maxAlbumsByIdsRequest)); }) .Singleton(); config.For().Use("ApolloAppleWebApi", c => { var baseHandler = new HttpClientHandler() { UseCookies = false, AutomaticDecompression = DecompressionMethods.Deflate | DecompressionMethods.GZip }; var loggingHandler = new ApolloLoggingMessageHandler(baseHandler); var rateLimitHandler = new ThreadRateLimitHttpHandler(threadRateLimit) { InnerHandler = loggingHandler }; var retryHandler = new RetryHttpHandler(2, TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(1), rateLimitHandler); var authHandler = new ApolloAuthTokenMessageHandler(apolloAuthPostPayload, tokenServiceUrl, retryHandler); var httpClient = new HttpClient(authHandler) { BaseAddress = new Uri(apolloUrl) }; return new ApolloAppleWebApi(httpClient); }) .Singleton(); config.For().Use("DspWebApi", c => { var baseHandler = new HttpClientHandler() { UseCookies = false, AutomaticDecompression = DecompressionMethods.Deflate | DecompressionMethods.GZip }; var loggingHandler = new ApolloLoggingMessageHandler(baseHandler); var rateLimitHandler = new ThreadRateLimitHttpHandler(threadRateLimit) { InnerHandler = loggingHandler }; var retryHandler = new RetryHttpHandler(2, TimeSpan.FromSeconds(60), TimeSpan.FromSeconds(1), rateLimitHandler); var authHandler = new ApolloAuthTokenMessageHandler(apolloAuthPostPayload, tokenServiceUrl, retryHandler); var httpClient = new HttpClient(authHandler) { BaseAddress = new Uri(apolloUrl) }; return new DspWebApi(httpClient); }) .Singleton(); } private static void ConfigureDatadog(ConfigurationExpression config, Dictionary settings) { string datadogKey = settings.GetValueOrDefault("DatadogKey"); string datadogLink = settings.GetValueOrDefault("DatadogLink"); string datadogPort = settings.GetValueOrDefault("DatadogPort"); string datadogService = settings.GetValueOrDefault("DatadogService"); string datadogHost = settings.GetValueOrDefault("DatadogHost"); config.For().Use() .Ctor("key").Is(datadogKey) .Ctor("url").Is(datadogLink) .Ctor("port").Is(datadogPort) .Ctor("service").Is(datadogService) .Ctor("host").Is(datadogHost); } private static void ConfigureAmazon(ConfigurationExpression config, Dictionary settings) { string amazonBucketPrefix = settings.GetValueOrDefault("AwsBucketPrefix") ?? string.Empty; config.For().Use().Ctor("bucketPrefix").Is(amazonBucketPrefix); config.ForConcreteType().Configure.Ctor("bucketPrefix").Is(amazonBucketPrefix); var awsAccessKey = settings.GetValueOrDefault("AWSAccessKey"); var awsSecretKey = settings.GetValueOrDefault("AWSSecretKey"); var awsRegion = settings.GetValueOrDefault("AWSRegion"); var cloudSearchSearchEndpoint = settings.GetValueOrDefault("CloudSearchSearchEndpoint"); config.For(new SingletonLifecycle()).Use(new AmazonS3Client(awsAccessKey, awsSecretKey, RegionEndpoint.GetBySystemName(awsRegion))); var awsCredentials = new BasicAWSCredentials(awsAccessKey, awsSecretKey); config.For().Use(awsCredentials); var amazonCloudSearchDomainConfig = new AmazonCloudSearchDomainConfig() { ServiceURL = cloudSearchSearchEndpoint }; config.For().Use(amazonCloudSearchDomainConfig); } private static void ConfigureLastFm(ConfigurationExpression config, Dictionary settings) { string lastFmApiKey = settings.GetValueOrDefault("LastFM-API-Key"); string lastFmSecretKey = settings.GetValueOrDefault("LastFM-API-SecretKey"); if (lastFmApiKey != null && lastFmSecretKey != null) { var baseHandler = new HttpClientHandler() { UseCookies = false, AutomaticDecompression = DecompressionMethods.Deflate | DecompressionMethods.GZip }; var retryHandler = new RetryHttpHandler(2, TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(1), baseHandler); var lastFMHttpClient = new HttpClient(retryHandler) { BaseAddress = new Uri("http://ws.audioscrobbler.com/2.0/") }; config.ForConcreteType().Configure .Ctor("lastFmApiKey").Is(lastFmApiKey) .Ctor("lastFmSecretKey").Is(lastFmSecretKey) .Ctor("httpClient").Is(lastFMHttpClient); config.ForConcreteType().Configure .Ctor("lastFmApiKey").Is(lastFmApiKey) .Ctor("lastFmSecretKey").Is(lastFmSecretKey); } } private void ConfigureAdminElasticSearch(ConfigurationExpression config, Dictionary settings) { var elasticEndpointUrl = settings.GetValueOrDefault("ElasticSearchAdminEndpointUrl"); var elasticIndexName = settings.GetValueOrDefault("ElasticSearchAdminIndexName"); config.ForConcreteType().Configure .Ctor("elasticEndpointUrl").Is(elasticEndpointUrl) .Ctor("elasticIndexName").Is(elasticIndexName); } private void ConfigureTasks(ConfigurationExpression config, Dictionary settings) { var bucket = settings["TracklistHistoryExport-Bucket"]; var bucketPath = settings["TracklistHistoryExport-BucketPath"]; var accessKey = settings["TracklistHistoryExport-AccessKey"]; var secretKey = settings["TracklistHistoryExport-SecretKey"]; var bucketReference = new S3BucketReference() { Bucket = bucket, BucketPath = bucketPath, AccessKey = accessKey, SecretKey = secretKey, }; config.ForConcreteType().Configure.Ctor("destinationBucket").Is(bucketReference); config.ForConcreteType().Configure.Ctor("destinationBucket").Is(bucketReference); var uploadPlaylistsAndUsersBucketReference = new S3BucketReference() { Bucket = settings["UploadPlaylistsAndUsers-Bucket"], BucketPath = settings["UploadPlaylistsAndUsers-BucketPath"], AccessKey = settings["UploadPlaylistsAndUsers-AccessKey"], SecretKey = settings["UploadPlaylistsAndUsers-SecretKey"], }; config.ForConcreteType().Configure.Ctor().Is(uploadPlaylistsAndUsersBucketReference); var uploadApiResponseReference = new S3BucketReferenceWithRegion() { Bucket = settings["UploadApiResponse-Bucket"], Region = settings["UploadApiResponse-Region"] }; config.ForConcreteType().Configure.Ctor().Is(uploadApiResponseReference); var exportSpotifyArtists = new S3BucketReference() { Bucket = settings["ExportSpotifyArtists-Bucket"], AccessKey = settings["ExportSpotifyArtists-AccessKey"], SecretKey = settings["ExportSpotifyArtists-SecretKey"], }; config.ForConcreteType().Configure.Ctor().Is(exportSpotifyArtists); config.ForConcreteType().Configure.Ctor().Is(exportSpotifyArtists); config.ForConcreteType().Configure.Ctor().Is(settings["AWS_TARGET_BUCKET"]); config.For().Use("Importer", c => { return new NewPlaylistsFromFolderReader(settings["NewPlaylistsFileImporter_Folder"], settings["NewPlaylistsFileImporter_FileSearchPattern"]); }); } } }