<?xml version="1.0" encoding="UTF-8"?>
<changelog xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="http://www.liquigraph.org/schema/1.0/liquigraph.xsd">
    <changeset id="DS-3302_backfill_spotifyPopularity_3:1" author="borisuvarov">
        <query>
           <![CDATA[
                // Data is loaded by chunks
                // Get the total number of records
                WITH 'snowflake_orchard' AS url_alias,
                '
                    SELECT COUNT(*) AS COUNT FROM (
                      SELECT
                          UPPER(isrc) AS ISRC,
                          popularity_score AS POPULARITY_SCORE
                      FROM chartmetric.raw_data.spotify
                      WHERE popularity_score IS NOT NULL
                      QUALIFY ROW_NUMBER() OVER (PARTITION by isrc ORDER BY modified_at, created_at desc NULLS LAST) = 1
                    )
                '
                AS sql
                CALL apoc.load.jdbc(url_alias, sql)
                YIELD row
                // Calculate params for the chunking
                WITH toInteger(row.COUNT) AS total, 1000000 AS step
                WITH range(0, total/step) AS items, step
                UNWIND items AS i
                WITH step AS limit, i * step AS offset
                // Process each chunk
                CALL apoc.periodic.iterate(
                    "
                        WITH 'snowflake_orchard' AS url_alias,
                        '
                          SELECT
                              UPPER(isrc) AS ISRC,
                              popularity_score AS POPULARITY_SCORE
                          FROM chartmetric.raw_data.spotify
                          WHERE popularity_score IS NOT NULL
                          QUALIFY ROW_NUMBER() OVER (PARTITION by isrc ORDER BY modified_at, created_at desc NULLS LAST) = 1
                          ORDER BY ISRC
                          LIMIT ' + $limit +' OFFSET ' + $offset
                        AS sql
                        CALL apoc.load.jdbc(url_alias, sql)
                        YIELD row
                        RETURN row
                    ",
                    "
                        MATCH (gsr:GlobalSoundRecording {isrc: row.ISRC})
                        SET
                            gsr.spotifyPopularity = row.POPULARITY_SCORE,
                            gsr.lastModifiedAt = datetime(),
                            gsr.lastModifiedBy = 'DB PR 9004'
                    ",
                    {batchSize:10000, iterateList:true, parallel:false, params:{limit: limit, offset: offset}}
                )
                YIELD total
                RETURN total;
            ]]>
        </query>
    </changeset>
</changelog>
