local:
  temp_folder: /mnt/data/tmp/exportAnalyticsRawFilesDirectory/SpotifyV2
  import_folder: /mnt/data/tmp/importAnalyticsRawFilesDirectory/SpotifyV2
s3:
  bucket: cucumbers
  archives_folder: SpotifyV2/archives
  import_folder: SpotifyV2/to_be_imported_into_redshift
expected_number_of_files: 38
check_num_of_files: true
store_id: 286
raw_table_name: staging_raw_spotify_v2
api:
  oauth_url: https://ws.spotify.com/oauth/token
  base_url: https://ws.spotify.com/analytics/api/theorchard
  client_id: orchard
  client_secret: 4BVPsfKMVu4T6HSQ
import_options:
  - escape
  - removequotes
  - gzip
resources:
  tracks:
    first_row: 1
    format: json
    uri: tracks/{year}/{month}/{day}
    schema:
      - track_artists
      - track_name
      - uri
      - album_artist
      - track_id
      - version
      - isrc
      - message
      - album_name
      - album_code
  users:
    first_row: 1
    format: json
    uri: users/{year}/{month}/{day}
    schema:
      - product
      - user_id
      - country
      - region
      - zipcode
      - access
      - version
      - gender
      - partner
      - message
      - referral
      - type
      - birth_year
  streams:
    first_row: 1
    format: json
    uri: streams/{year}/{month}/{day}/{country}
    countries:
      - AD
      - AE
      - AF
      - AG
      - AI
      - AL
      - AM
      - AN
      - AO
      - AQ
      - AR
      - AS
      - AT
      - AU
      - AW
      - AX
      - AZ
      - BA
      - BB
      - BD
      - BE
      - BF
      - BG
      - BH
      - BI
      - BJ
      - BM
      - BN
      - BO
      - BR
      - BS
      - BT
      - BV
      - BW
      - BY
      - BZ
      - CA
      - CC
      - CD
      - CF
      - CG
      - CH
      - CI
      - CK
      - CL
      - CM
      - CN
      - CO
      - CR
      - CS
      - CU
      - CV
      - CX
      - CY
      - CZ
      - DE
      - DJ
      - DK
      - DM
      - DO
      - DZ
      - EC
      - EE
      - EG
      - EH
      - ER
      - ES
      - ET
      - FI
      - FJ
      - FK
      - FM
      - FO
      - FO
      - FR
      - GA
      - GB
      - GD
      - GE
      - GF
      - GG
      - GH
      - GI
      - GL
      - GM
      - GN
      - GP
      - GQ
      - GR
      - GS
      - GT
      - GU
      - GW
      - GY
      - HK
      - HM
      - HN
      - HR
      - HT
      - HU
      - ID
      - IE
      - IL
      - IM
      - IN
      - IO
      - IQ
      - IR
      - IS
      - IT
      - JE
      - JM
      - JO
      - JP
      - KE
      - KG
      - KH
      - KI
      - KM
      - KN
      - KP
      - KR
      - KW
      - KY
      - KZ
      - LA
      - LB
      - LC
      - LI
      - LK
      - LR
      - LS
      - LT
      - LU
      - LV
      - LY
      - MA
      - MC
      - MD
      - MG
      - MH
      - MK
      - ML
      - MM
      - MN
      - MO
      - MP
      - MQ
      - MR
      - MS
      - MT
      - MU
      - MV
      - MW
      - MX
      - MY
      - MZ
      - NA
      - NC
      - NE
      - NF
      - NG
      - NI
      - NL
      - NO
      - NP
      - NR
      - NU
      - NZ
      - OM
      - PA
      - PE
      - PF
      - PG
      - PH
      - PK
      - PL
      - PM
      - PN
      - PR
      - PS
      - PT
      - PW
      - PY
      - QA
      - RE
      - RO
      - RU
      - RW
      - SA
      - SB
      - SC
      - SD
      - SE
      - SG
      - SH
      - SI
      - SJ
      - SK
      - SL
      - SM
      - SN
      - SO
      - SR
      - ST
      - SV
      - SY
      - SZ
      - TC
      - TD
      - TF
      - TG
      - TH
      - TJ
      - TK
      - TL
      - TM
      - TN
      - TO
      - TR
      - TT
      - TV
      - TW
      - TZ
      - UA
      - UG
      - UM
      - US
      - UY
      - UZ
      - VA
      - VC
      - VE
      - VG
      - VI
      - VN
      - VU
      - WF
      - WS
      - YE
      - YT
      - ZA
      - ZM
      - ZW
    schema:
      - user_id
      - cached
      - timestamp
      - source_uri
      - track_id
      - source
      - length
      - version
      - device_type
      - offline_timestamp
      - message
      - os
raw_table_schema:
  - row_id
  - user_id
  - cached
  - tmstamp
  - source_uri
  - track_id
  - source
  - length
  - version
  - device_type
  - offline_timestamp
  - message
  - os
  - track_artists
  - track_name
  - track_uri
  - track_album_artist
  - isrc
  - track_message
  - album_name
  - album_code
  - upc
  - product
  - user_country
  - user_region
  - zipcode
  - user_access
  - gender
  - partner
  - user_message
  - referral
  - user_type
  - birth_year
#todo: validation on data type will be in the above schema

mapping:
  user_id:
    file: streams
    field: user_id
    type: varchar
  cached:
    file: streams
    field: cached
    type: varchar
  download_date:
    file: streams
    field: timestamp
    type: date
  timestamp:
    file: streams
    field: timestamp
    type: datetime
  source_uri:
    file: streams
    field: source_uri
    type: varchar
  track_id:
    file: streams
    field: track_id
    type: varchar
  source:
    file: streams
    field: source
    type: varchar
  length:
    file: streams
    field: length
    type: int
  version:
    file: streams
    field: version
    type: int
  device_type:
    file: streams
    field: device_type
    type: varchar
  offline_timestamp:
    file: streams
    field: offline_timestamp
    type: varchar
  message:
    file: streams
    field: message
    type: varchar
  os:
    file: streams
    field: os
    type: varchar
  track_artists:
    file: tracks
    field: track_artists
    type: varchar
  track_name:
    file: tracks
    field: track_name
    type: varchar
  track_uri:
    file: tracks
    field: uri
    type: varchar
  track_album_artist:
    file: tracks
    field: album_artist
    type: varchar
  isrc:
    file: tracks
    field: isrc
    type: varchar
  track_message:
    file: tracks
    field: message
    type: varchar
  album_name:
    file: tracks
    field: album_name
    type: varchar
  album_code:
    file: tracks
    field: album_code
    type: varchar
  upc:
    file: tracks
    field: album_code
    type: bigint
  product:
    file: users
    field: product
    type: varchar
  user_country:
    file: users
    field: country
    type: varchar
  user_region:
    file: users
    field: region
    type: varchar
  zipcode:
    file: users
    field: zipcode
    type: varchar
  user_access:
    file: users
    field: access
    type: varchar
  gender:
    file: users
    field: gender
    type: varchar
  partner:
    file: users
    field: partner
    type: varchar
  user_message:
    file: users
    field: message
    type: varchar
  referral:
    file: users
    field: referral
    type: varchar
  user_type:
    file: users
    field: type
    type: varchar
  birth_year:
    file: users
    field: birth_year
    type: varchar