{PipelineObjects: [ {Id: ScheduleId2,Name: IngestionTime, Fields: [{Key: startDateTime,StringValue: 2013-09-06T00:00:00,}, {Key: period,StringValue: 30 Minutes,}, {Key: type,StringValue: Schedule,}]}, {Id: ActivityId2,Name: Transform & Upload Deezer Raw Data, Fields: [{Key: schedule,RefValue: ScheduleId2}, {Key: command,StringValue: /Analytics/RawFilesTransporter/RawFilesTransporter/TransformUploadRawDeezer/TransformUploadRawDeezer_run.sh --context=Production,}, {Key: workerGroup,StringValue: dev_server_one,}, {Key: type,StringValue: ShellCommandActivity,}]}, {Id: RegExId28,Name: MyInputDataType, Fields: [{Key: outputFormat,StringValue: %1$s %2$s %3$s %4$s %5$s %6$s %7$s %8$s %9$s,}, {Key: column,StringValue: host STRING,}, {Key: column,StringValue: identity STRING,}, {Key: column,StringValue: user STRING,}, {Key: column,StringValue: time STRING,}, {Key: column,StringValue: request STRING,}, {Key: column,StringValue: status STRING,}, {Key: column,StringValue: size STRING,}, {Key: column,StringValue: referer STRING,}, {Key: column,StringValue: agent STRING,}, {Key: inputRegEx,StringValue: ([^ ]*) ([^ ]*) ([^ ]*) (-|[[^]]*]) ([^ "]*|"[^"]*") (-|[0-9]*) (-|[0-9]*)(?: ([^ "]*|"[^"]*") ([^ "]*|"[^"]*"))?,}, {Key: type,StringValue: RegEx,}]}, {Id: ActivityId6,Name: Download Amazon raw file,Fields: [{Key: schedule,RefValue: ScheduleId2}, {Key: command,StringValue: /Analytics/RawFilesTransporter/RawFilesTransporter/IngestAmazon/IngestAmazon_run.sh --context=Production,}, {Key: onFail,RefValue: ActionId3}, {Key: workerGroup,StringValue: dev_server_one,}, {Key: type,StringValue: ShellCommandActivity,}]}, {Id: ActivityId1,Name: Transform & Upload Amazon Raw Data,Fields: [{Key: schedule,RefValue: ScheduleId2}, {Key: command,StringValue: /Analytics/RawFilesTransporter/RawFilesTransporter/TransformUploadRawAmazon/TransformUploadRawAmazon_run.sh --context=Production,}, {Key: workerGroup,StringValue: dev_server_one,}, {Key: type,StringValue: ShellCommandActivity,}]}, {Id: ActivityId5,Name: Download Spotify raw file,Fields: [{Key: schedule,RefValue: ScheduleId2}, {Key: command,StringValue: /Analytics/RawFilesTransporter/RawFilesTransporter/IngestSpotify/IngestSpotify_run.sh --context=Production,}, {Key: workerGroup,StringValue: dev_server_one,}, {Key: type,StringValue: ShellCommandActivity,}]}, {Id: ActivityId9,Name: Load Amazon Raw Data into Redshift,Fields: [{Key: schedule,RefValue: ScheduleId2}, {Key: command,StringValue: /Analytics/RawFilesTransporter/RawFilesTransporter/LoadRawFileToRedshift/LoadRawFileToRedshift_run.sh --context=Production --context_param raw_file_table=staging_raw_amazon --context_param store_s3_import_folder=/Amazon/to_be_imported_into_redshift --context_param storeId=187,}, {Key: workerGroup,StringValue: dev_server_one,}, {Key: type,StringValue: ShellCommandActivity,}]}, {Id: ActivityId8,Name: Load Spotify Raw Data into Redshift,Fields: [{Key: schedule,RefValue: ScheduleId2}, {Key: command,StringValue: /Analytics/RawFilesTransporter/RawFilesTransporter/LoadRawFileToRedshift/LoadRawFileToRedshift_run.sh --context=Production --context_param raw_file_table=staging_raw_spotify --context_param store_s3_import_folder=/Spotify/to_be_imported_into_redshift --context_param storeId=286,}, {Key: workerGroup,StringValue: dev_server_one,}, {Key: type,StringValue: ShellCommandActivity,}]}, {Id: Default,Name: Default,Fields: [{Key: scheduleType,StringValue: cron,}, {Key: role,StringValue: DataPipelineDefaultRole,}, {Key: resourceRole,StringValue: DataPipelineDefaultResourceRole,}]}, {Id: ActivityId4,Name: Download Deezer raw file,Fields: [{Key: schedule,RefValue: ScheduleId2}, {Key: command,StringValue: /Analytics/RawFilesTransporter/RawFilesTransporter/IngestDeezer/IngestDeezer_run.sh --context=Production,}, {Key: workerGroup,StringValue: dev_server_one,}, {Key: type,StringValue: ShellCommandActivity,}]}, {Id: ActivityId3,Name: Transform & Upload Spotify Raw Data,Fields: [{Key: schedule,RefValue: ScheduleId2}, {Key: command,StringValue: /Analytics/RawFilesTransporter/RawFilesTransporter/TransformUploadRawSpotify/TransformUploadRawSpotify_run.sh --context=Production,}, {Key: workerGroup,StringValue: dev_server_one,}, {Key: type,StringValue: ShellCommandActivity,}]}, {Id: ActionId5,Name: Deezer Ingestion Failed,Fields: [{Key: message,StringValue: Deezer Ingestion Failed,}, {Key: subject,StringValue: Deezer Ingestion Failed,}, {Key: role,StringValue: DataPipelineDefaultRole,}, {Key: topicArn,StringValue: arn:aws:sns:us-east-1:437795906767:data_pipeline_topic,}, {Key: type,StringValue: SnsAlarm,}]}, {Id: ActivityId7,Name: Load Deezer Raw Data into Redshift,Fields: [{Key: schedule,RefValue: ScheduleId2}, {Key: command,StringValue: /Analytics/RawFilesTransporter/RawFilesTransporter/LoadRawFileToRedshift/LoadRawFileToRedshift_run.sh --context=Production --context_param raw_file_table=staging_raw_deezer --context_param store_s3_import_folder=/Deezer/to_be_imported_into_redshift --context_param storeId=348,}, {Key: workerGroup,StringValue: dev_server_one,}, {Key: type,StringValue: ShellCommandActivity,}]}, {Id: ActionId4,Name: Spotify Ingestion Failed,Fields: [{Key: message,StringValue: Spotify Ingestion Failed,}, {Key: subject,StringValue: Spotify Ingestion Failed,}, {Key: role,StringValue: DataPipelineDefaultRole,}, {Key: topicArn,StringValue: arn:aws:sns:us-east-1:437795906767:data_pipeline_topic,}, {Key: type,StringValue: SnsAlarm,}]}, {Id: ActionId3,Name: Amazon Ingestion Failed,Fields: [{Key: message,StringValue: Amazon Ingestion Failed,}, {Key: subject,StringValue: Amazon Ingestion Failed,}, {Key: topicArn,StringValue: arn:aws:sns:us-east-1:437795906767:data_pipeline_topic,}, {Key: type,StringValue: SnsAlarm,}]}, {Id: CustomId30,Name: MyOutputDataType,Fields: [{Key: column,StringValue: host STRING,}, {Key: column,StringValue: user STRING,}, {Key: column,StringValue: time STRING,}, {Key: column,StringValue: request STRING,}, {Key: column,StringValue: status STRING,}, {Key: column,StringValue: size STRING,}, {Key: recordSeparator,StringValue: ,}, {Key: columnSeparator,StringValue: ,}, {Key: type,StringValue: Custom,}]}]}