{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "import sys" ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "import pandas as pd\n", "import numpy\n", "import requests\n", "from collections import namedtuple\n", "import random\n", "import datetime\n", "import matplotlib.pyplot as plt\n", "import matplotlib.dates as mdates\n", "\n", "import pymysql\n", "from contextlib import contextmanager\n", "import datetime\n", "\n", "conn_kwargs = {\n", " 'host': 'unicron.cluster-cbn1zk7uet6r.eu-west-1.rds.amazonaws.com', \n", " 'user': 'unicron', \n", " 'password': 'tydz5cy9xbvchqpa249rfekhcg9sey', \n", " 'db': 'unicron', \n", " 'charset': 'utf8mb4'\n", "}\n", "\n", "@contextmanager\n", "def get_cursor(commit_after=False):\n", " connection = pymysql.connect(**conn_kwargs)\n", " try:\n", " with connection.cursor() as cur:\n", " yield cur\n", " if commit_after:\n", " connection.commit()\n", " else:\n", " connection.rollback()\n", " finally:\n", " try:\n", " connection.close()\n", " except Exception as e:\n", " print(\"Warning, failed to close connection\")\n", " print(e)\n", "\n", "\n", "def _sql_to_dataframe(sql_statement, columns, sql_args=[], index_column=None):\n", " with get_cursor(False) as cur:\n", " cur.execute(sql_statement, sql_args)\n", " rows = cur.fetchall()\n", " df = pd.DataFrame(data=rows, columns=columns)\n", " if index_column:\n", " df = df.set_index(index_column)\n", " return df\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "df1." ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "vid_files = [\n", " '/Users/joel/src/thundr/tikscrape/music_1606403758182.csv',\n", " '/Users/joel/src/thundr/tikscrape/music_1606404754757.csv'\n", "]" ] }, { "cell_type": "code", "execution_count": 39, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "['/Users/joel/src/thundr/tikscrape/music_1606403758182.csv',\n", " '/Users/joel/src/thundr/tikscrape/music_1606404754757.csv']" ] }, "execution_count": 39, "metadata": {}, "output_type": "execute_result" } ], "source": [ "vid_files" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 40, "metadata": {}, "outputs": [], "source": [ "all_df = pd.concat([pd.read_csv(v) for v in vid_files], ignore_index=True)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 41, "metadata": {}, "outputs": [], "source": [ "def write_csv(fname, tsvname):\n", " \n", " df_ruin_fre = pd.read_csv(fname)\n", " \n", " df_ruin_fre['createTime'] = df_ruin_fre.createTime.apply(lambda x: pd.Timestamp(x, unit='s'))\n", "\n", " df_ruin_fre[\n", " ['id', \n", " 'createTime',\n", " 'authorMeta.name',\n", " 'diggCount',\n", " 'shareCount', 'playCount',\n", " 'commentCount',\n", " 'text', \n", " 'authorMeta.id', 'authorMeta.nickName', \n", " 'musicMeta.musicId', 'musicMeta.musicName', \n", " 'webVideoUrl', \n", " #'mentions', \n", " ]\n", " ].to_csv(tsvname, sep='\\t')" ] }, { "cell_type": "code", "execution_count": 42, "metadata": {}, "outputs": [], "source": [ "write_csv('/Users/joel/src/thundr/tikscrape/music_1606403758182.csv', '~/ruin_friendship.tsv')" ] }, { "cell_type": "code", "execution_count": 43, "metadata": {}, "outputs": [], "source": [ "write_csv('/Users/joel/src/thundr/tikscrape/music_1606404754757.csv', '~/jenny.tsv')" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 16, "metadata": {}, "outputs": [], "source": [ "all_df.to_csv('~/test.tsv', sep='\\t')" ] }, { "cell_type": "code", "execution_count": 12, "metadata": {}, "outputs": [], "source": [ "def set_for_keys(my_dict, key_arr, val):\n", " \"\"\"\n", " Set val at path in my_dict defined by the string (or serializable object) array key_arr\n", " \"\"\"\n", " current = my_dict\n", " for i in range(len(key_arr)):\n", " key = key_arr[i]\n", " if key not in current:\n", " if i==len(key_arr)-1:\n", " current[key] = val\n", " else:\n", " current[key] = {}\n", " else:\n", " if type(current[key]) is not dict:\n", " print(\"Given dictionary is not compatible with key structure requested\")\n", " raise ValueError(\"Dictionary key already occupied\")\n", "\n", " current = current[key]\n", "\n", " return my_dict\n", "\n", "def to_formatted_json(df, sep=\".\"):\n", " result = []\n", " for _, row in df.iterrows():\n", " parsed_row = {}\n", " for idx, val in row.iteritems():\n", " keys = idx.split(sep)\n", " parsed_row = set_for_keys(parsed_row, keys, val)\n", "\n", " result.append(parsed_row)\n", " return result" ] }, { "cell_type": "code", "execution_count": 13, "metadata": {}, "outputs": [], "source": [ "d = json.loads(all_df.head(10).to_json(orient='records'))" ] }, { "cell_type": "code", "execution_count": 14, "metadata": { "scrolled": false }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "id 6878128824723229957\n", "text And I have to go in a break out room with her tomorrow 😿\n", "createTime 1601439223\n", "authorMeta.id 6809716070158337030\n", "authorMeta.secUid MS4wLjABAAAAmVe5G790qRHo6VFjcUkSDAEueUPxvrOWwhhSLn9PWn9MuE_zHIeeb3NrL9m3XTXZ\n", "authorMeta.name isaac..thebully\n", "authorMeta.nickName Matching with a dummy 😏✨\n", "authorMeta.verified False\n", "authorMeta.signature I’m a “bully” to mae 😂\n", "—\n", "I took over Janelle’s discord 🙈\n", "—\n", "4000+ily all🦭✨\n", "authorMeta.avatar https://p16-sign-va.tiktokcdn.com/tos-maliva-avt-0068/e3fc6e4ed08e7cce77d0c057505ee50a~c5_1080x1080.jpeg?x-expires=1606489200&x-signature=HC766VoTCzUUIjR8pgX4R9Vb%2F8M%3D\n", "musicMeta.musicId 6878128942629358342\n", "musicMeta.musicName Ruin our friendship\n", "musicMeta.musicAuthor Matching with a dummy 😏✨\n", "musicMeta.musicOriginal True\n", "musicMeta.playUrl https://sf16-va.tiktokcdn.com/obj/musically-maliva-obj/6878128654828833541.mp3\n", "musicMeta.coverThumb https://p16-sign-va.tiktokcdn.com/tos-maliva-avt-0068/e3fc6e4ed08e7cce77d0c057505ee50a~c5_100x100.jpeg?x-expires=1606489200&x-signature=y5cqFz%2BgM2vgMEfxF%2BODzNRHP0Q%3D\n", "musicMeta.coverMedium https://p16-sign-va.tiktokcdn.com/tos-maliva-avt-0068/e3fc6e4ed08e7cce77d0c057505ee50a~c5_720x720.jpeg?x-expires=1606489200&x-signature=IHQh8KIVV1SUgwNjQmaHNrYXfK4%3D\n", "musicMeta.coverLarge https://p16-sign-va.tiktokcdn.com/tos-maliva-avt-0068/e3fc6e4ed08e7cce77d0c057505ee50a~c5_1080x1080.jpeg?x-expires=1606489200&x-signature=HC766VoTCzUUIjR8pgX4R9Vb%2F8M%3D\n", "covers.default https://p16-sign-va.tiktokcdn.com/obj/tos-maliva-p-0068/9ad087281cb7495abd38ebf9ec6260f8?x-expires=1606424400&x-signature=gYGLGOFPHaZhxkuVKN80Q5FppZc%3D\n", "covers.origin https://p16-sign-va.tiktokcdn.com/obj/tos-maliva-p-0068/6b22814f0ea34f579d4ef239db4350e4_1601439224?x-expires=1606424400&x-signature=sJxnCRzBqxdosd85AFZX9yczIW0%3D\n", "covers.dynamic https://p16-sign-va.tiktokcdn.com/obj/tos-maliva-p-0068/7a5ac8a92a3c45459d44a9cd35595615_1601439225?x-expires=1606424400&x-signature=4IFFRtJy2ATJnPuvz5FKjkb4lXg%3D\n", "webVideoUrl https://www.tiktok.com/@isaac..thebully/video/6878128824723229957\n", "videoUrl https://v16-web.tiktok.com/video/tos/useast2a/tos-useast2a-pve-0068/feb9b39825764701a8589c24e88ee592/?a=1988&br=612&bt=306&cr=0&cs=0&cv=1&dr=0&ds=3&er=&expire=1606425372&l=202011261515210101902190785F02177C&lr=tiktok_m&mime_type=video_mp4&policy=2&qs=0&rc=amZnZWp2bnUzdzMzODczM0ApNjs2Nmk5aTtoN2U5N2hlZWdpb2RvZmkvX25fLS0vMTZzc2NjMjUyLl4wLzZhYl5eXzU6Yw%3D%3D&signature=933721afd10d7bea26396240d7c1a9cd&tk=tt_webid_v2&vl=&vr=\n", "videoUrlNoWaterMark https://api2.musical.ly/aweme/v1/playwm/?video_id=awesome\n", "videoMeta.height 1024\n", "videoMeta.width 576\n", "videoMeta.duration 51\n", "diggCount 74200\n", "shareCount 2818\n", "playCount 795100\n", "commentCount 1802\n", "downloaded False\n", "mentions []\n", "hashtags []\n", "{\n", " \"id\": 6878128824723229957,\n", " \"text\": \"And I have to go in a break out room with her tomorrow \\ud83d\\ude3f\",\n", " \"createTime\": 1601439223,\n", " \"authorMeta\": {\n", " \"id\": 6809716070158337030,\n", " \"secUid\": \"MS4wLjABAAAAmVe5G790qRHo6VFjcUkSDAEueUPxvrOWwhhSLn9PWn9MuE_zHIeeb3NrL9m3XTXZ\",\n", " \"name\": \"isaac..thebully\",\n", " \"nickName\": \"Matching with a dummy \\ud83d\\ude0f\\u2728\",\n", " \"verified\": false,\n", " \"signature\": \"I\\u2019m a \\u201cbully\\u201d to mae \\ud83d\\ude02\\n\\u2014\\nI took over Janelle\\u2019s discord \\ud83d\\ude48\\n\\u2014\\n4000+ily all\\ud83e\\uddad\\u2728\",\n", " \"avatar\": \"https://p16-sign-va.tiktokcdn.com/tos-maliva-avt-0068/e3fc6e4ed08e7cce77d0c057505ee50a~c5_1080x1080.jpeg?x-expires=1606489200&x-signature=HC766VoTCzUUIjR8pgX4R9Vb%2F8M%3D\"\n", " },\n", " \"musicMeta\": {\n", " \"musicId\": 6878128942629358342,\n", " \"musicName\": \"Ruin our friendship\",\n", " \"musicAuthor\": \"Matching with a dummy \\ud83d\\ude0f\\u2728\",\n", " \"musicOriginal\": true,\n", " \"playUrl\": \"https://sf16-va.tiktokcdn.com/obj/musically-maliva-obj/6878128654828833541.mp3\",\n", " \"coverThumb\": \"https://p16-sign-va.tiktokcdn.com/tos-maliva-avt-0068/e3fc6e4ed08e7cce77d0c057505ee50a~c5_100x100.jpeg?x-expires=1606489200&x-signature=y5cqFz%2BgM2vgMEfxF%2BODzNRHP0Q%3D\",\n", " \"coverMedium\": \"https://p16-sign-va.tiktokcdn.com/tos-maliva-avt-0068/e3fc6e4ed08e7cce77d0c057505ee50a~c5_720x720.jpeg?x-expires=1606489200&x-signature=IHQh8KIVV1SUgwNjQmaHNrYXfK4%3D\",\n", " \"coverLarge\": \"https://p16-sign-va.tiktokcdn.com/tos-maliva-avt-0068/e3fc6e4ed08e7cce77d0c057505ee50a~c5_1080x1080.jpeg?x-expires=1606489200&x-signature=HC766VoTCzUUIjR8pgX4R9Vb%2F8M%3D\"\n", " },\n", " \"covers\": {\n", " \"default\": \"https://p16-sign-va.tiktokcdn.com/obj/tos-maliva-p-0068/9ad087281cb7495abd38ebf9ec6260f8?x-expires=1606424400&x-signature=gYGLGOFPHaZhxkuVKN80Q5FppZc%3D\",\n", " \"origin\": \"https://p16-sign-va.tiktokcdn.com/obj/tos-maliva-p-0068/6b22814f0ea34f579d4ef239db4350e4_1601439224?x-expires=1606424400&x-signature=sJxnCRzBqxdosd85AFZX9yczIW0%3D\",\n", " \"dynamic\": \"https://p16-sign-va.tiktokcdn.com/obj/tos-maliva-p-0068/7a5ac8a92a3c45459d44a9cd35595615_1601439225?x-expires=1606424400&x-signature=4IFFRtJy2ATJnPuvz5FKjkb4lXg%3D\"\n", " },\n", " \"webVideoUrl\": \"https://www.tiktok.com/@isaac..thebully/video/6878128824723229957\",\n", " \"videoUrl\": \"https://v16-web.tiktok.com/video/tos/useast2a/tos-useast2a-pve-0068/feb9b39825764701a8589c24e88ee592/?a=1988&br=612&bt=306&cr=0&cs=0&cv=1&dr=0&ds=3&er=&expire=1606425372&l=202011261515210101902190785F02177C&lr=tiktok_m&mime_type=video_mp4&policy=2&qs=0&rc=amZnZWp2bnUzdzMzODczM0ApNjs2Nmk5aTtoN2U5N2hlZWdpb2RvZmkvX25fLS0vMTZzc2NjMjUyLl4wLzZhYl5eXzU6Yw%3D%3D&signature=933721afd10d7bea26396240d7c1a9cd&tk=tt_webid_v2&vl=&vr=\",\n", " \"videoUrlNoWaterMark\": \"https://api2.musical.ly/aweme/v1/playwm/?video_id=awesome\",\n", " \"videoMeta\": {\n", " \"height\": 1024,\n", " \"width\": 576,\n", " \"duration\": 51\n", " },\n", " \"diggCount\": 74200,\n", " \"shareCount\": 2818,\n", " \"playCount\": 795100,\n", " \"commentCount\": 1802,\n", " \"downloaded\": false,\n", " \"mentions\": \"[]\",\n", " \"hashtags\": \"[]\"\n", "}\n" ] } ], "source": [ "res = {}\n", "for k,v in d[0].items():\n", " print(k,v)\n", " set_for_keys(res, k.split('.'), v)\n", "print(json.dumps(res, indent=2))" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [], "source": [ "import json" ] }, { "cell_type": "code", "execution_count": 108, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[[6800723781938695429,\n", " 'А ширинка то расстёгивается потихоньку',\n", " 1583416908,\n", " 6531363258450875407,\n", " 'katya_golyshevaа',\n", " 'Eкатерина💣',\n", " 46,\n", " 2809158,\n", " 95611465,\n", " 1202,\n", " 1848,\n", " True,\n", " False,\n", " 'Instagram : katya_golyshevaа\\nСотрудничество: reklama@influencestars.ru',\n", " 'https://p16.muscdn.com/img/musically-maliva-obj/1655093156957189~c5_720x720.jpeg',\n", " 6787053043994118917,\n", " 'Август',\n", " 'Intelligency',\n", " None,\n", " 'https://sf16-sg.muscdn.com/obj/tiktok-obj/a601c4d240ff83923830d3d929104ebd.mp3',\n", " 'https://p16-va-default.akamaized.net/obj/tos-maliva-p-0068/3c0e90300089917657e6d506e1db296c',\n", " 'https://p16-va-default.akamaized.net/obj/tos-maliva-p-0068/008a81cd7ee64f1382ce50a3accd17b2_1583416912',\n", " 'https://p16-va-default.akamaized.net/obj/tos-maliva-p-0068/cfde62e6c3154a5f8cea3181bf4203d3_1583416912',\n", " 'https://p16-va-default.akamaized.net/obj/tos-maliva-p-0068/008a81cd7ee64f1382ce50a3accd17b2_1583416912',\n", " 'https://www.tiktok.com/@katya_golyshevaа/video/6800723781938695429',\n", " 'https://v19.muscdn.com/5180bc8af2d9546e81750032792f9995/5ea0b677/video/tos/useast2a/tos-useast2a-pve-0068/7a307230ca7a4af3bba80866e99bf18f/?a=1233&br=2410&bt=1205&cr=0&cs=0&dr=0&ds=3&er=&l=2020042215260401018904915614282A09&lr=tiktok_m&qs=0&rc=anByN2lydmVqczMzPDczM0ApOmQ8aDY6PGVmNzQ8OGY5ZGdhL2hqYWBrYGRfLS0uMTZzczI1MjQzX2EyXy1jMTA2LWM6Yw%3D%3D&vl=&vr=',\n", " None,\n", " 540,\n", " 960,\n", " 11,\n", " 11,\n", " 457040,\n", " 2783,\n", " 3791988,\n", " 1375,\n", " False,\n", " '[{\"id\":\"85849860\",\"name\":\"неловкость\",\"title\":\"Используйте этот звук и фильтр с переключением света, чтобы рассказать о своих неловких моментах 😜\",\"cover\":[\"https://p16-va-default.akamaized.net/obj/musically-maliva-obj/7cbd30f5ecb299d068c975b4fd4c8dcc\"]}]',\n", " 'https://www.tiktok.com/share/user/6531363258450875407'],\n", " [6802185033667841285,\n", " 'Раскрыл все карты 😉 #рекомендации #хочуврекомендации #дедушкавтренде #титок #тренды #дед #хочуврек #рек #гитара #музыка #танцы #секреты #84 #чеснок',\n", " 1583757125,\n", " 6518024844586521615,\n", " 'super_dedok',\n", " '🌠супер дед 🌠',\n", " 136,\n", " 138077,\n", " 1475342,\n", " 65,\n", " 2500,\n", " False,\n", " False,\n", " '🌸Дед Альберт 🌸\\n🌸84 года 🌸',\n", " 'https://p16.muscdn.com/img/musically-maliva-obj/1640314389623814~c5_720x720.jpeg',\n", " 6787053043994118917,\n", " 'Август',\n", " 'Intelligency',\n", " None,\n", " 'https://sf16-sg.muscdn.com/obj/tiktok-obj/a601c4d240ff83923830d3d929104ebd.mp3',\n", " 'https://p16-va-default.akamaized.net/obj/tos-maliva-p-0068/e809d3361d8aecfb93f9533689386462',\n", " 'https://p16-va-default.akamaized.net/obj/tos-maliva-p-0068/951694a2a9fe456ba913e6459f40186a_1583757236',\n", " 'https://p16-va-default.akamaized.net/obj/tos-maliva-p-0068/08b97896d2064f7695e8981a92f5a32b_1583757211',\n", " 'https://p16-va-default.akamaized.net/obj/tos-maliva-p-0068/951694a2a9fe456ba913e6459f40186a_1583757236',\n", " 'https://www.tiktok.com/@super_dedok/video/6802185033667841285',\n", " 'https://v19.muscdn.com/7fde486f2806a9805e0995f2d25961d0/5ea0b677/video/tos/useast2a/tos-useast2a-ve-0068c001/c88ec03d119d4d20aa1df4726367baeb/?a=1233&br=2604&bt=1302&cr=0&cs=0&dr=0&ds=3&er=&l=2020042215260401018904915614282A09&lr=tiktok_m&qs=0&rc=MzxvbGdoZ3Q2czMzOzczM0ApNTlmNGdoZmU4NzxoMzw3OmctMGltNmJzYWdfLS0wMTZzc19eMl4zX18vNS0xLS9eYmM6Yw%3D%3D&vl=&vr=',\n", " None,\n", " 576,\n", " 1024,\n", " 11,\n", " 11,\n", " 438427,\n", " 2801,\n", " 3692800,\n", " 3648,\n", " False,\n", " '[{\"id\":\"49711363\",\"name\":\"рекомендации\",\"title\":\"\",\"cover\":[]},{\"id\":\"86025601\",\"name\":\"хочуврекомендации\",\"title\":\"\",\"cover\":[]},{\"id\":\"1620764276681734\",\"name\":\"дедушкавтренде\",\"title\":\"\",\"cover\":[]},{\"id\":\"79006121\",\"name\":\"титок\",\"title\":\"\",\"cover\":[]},{\"id\":\"44600146\",\"name\":\"тренды\",\"title\":\"\",\"cover\":[]},{\"id\":\"11046549\",\"name\":\"дед\",\"title\":\"\",\"cover\":[]},{\"id\":\"1596093786694662\",\"name\":\"хочуврек\",\"title\":\"Насколько сильно ты хочешь попасть в тренд в TikTok? Танцуй и снимай видео под этот заводной трек 🕺💃\",\"cover\":[\"https://p16-va-default.akamaized.net/obj/musically-maliva-obj/d44aeb6537fbec6abcd1e7670eddafaf\"]},{\"id\":\"82913700\",\"name\":\"рек\",\"title\":\"\",\"cover\":[]},{\"id\":\"5485036\",\"name\":\"гитара\",\"title\":\"\",\"cover\":[]},{\"id\":\"5008447\",\"name\":\"музыка\",\"title\":\"\",\"cover\":[]},{\"id\":\"82977745\",\"name\":\"танцы\",\"title\":\"\",\"cover\":[]},{\"id\":\"25400043\",\"name\":\"секреты\",\"title\":\"\",\"cover\":[]},{\"id\":\"12608231\",\"name\":\"84\",\"title\":\"\",\"cover\":[]},{\"id\":\"35459948\",\"name\":\"чеснок\",\"title\":\"\",\"cover\":[]},{\"id\":\"85849860\",\"name\":\"неловкость\",\"title\":\"Используйте этот звук и фильтр с переключением света, чтобы рассказать о своих неловких моментах 😜\",\"cover\":[\"https://p16-va-default.akamaized.net/obj/musically-maliva-obj/7cbd30f5ecb299d068c975b4fd4c8dcc\"]}]',\n", " 'https://www.tiktok.com/share/user/6518024844586521615'],\n", " [6803043414272232709,\n", " \"very sad novost' video was udaleno, I'm rasstroen(sad)((( #foryou #viral #рекомендации\",\n", " 1583956978,\n", " 190163290986258432,\n", " 'deepins',\n", " 'za4em',\n", " 45,\n", " 2225685,\n", " 23669376,\n", " 66,\n", " 4789,\n", " True,\n", " False,\n", " 'иван; а следующее видео нескоро 🤪\\nПочта:reklama@influencestars.ru \\nInst:deepins',\n", " 'https://p16.muscdn.com/img/musically-maliva-obj/1662594887153670~c5_720x720.jpeg',\n", " 6787053043994118917,\n", " 'Август',\n", " 'Intelligency',\n", " None,\n", " 'https://sf16-sg.muscdn.com/obj/tiktok-obj/a601c4d240ff83923830d3d929104ebd.mp3',\n", " 'https://p16-va-default.akamaized.net/obj/tos-maliva-p-0068/eff10556532eb3ee8f121250f2a5d654',\n", " 'https://p16-va-default.akamaized.net/obj/tos-maliva-p-0068/dc0cb215365c47de88d65ba9c57db618_1583956986',\n", " 'https://p16-va-default.akamaized.net/obj/tos-maliva-p-0068/9229430080a14fca8e3ee7544fd6b75d_1583956984',\n", " 'https://p16-va-default.akamaized.net/obj/tos-maliva-p-0068/dc0cb215365c47de88d65ba9c57db618_1583956986',\n", " 'https://www.tiktok.com/@deepins/video/6803043414272232709',\n", " 'https://v19.muscdn.com/7a15c1d18382146bd9256ffc08738698/5ea0b67b/video/tos/useast2a/tos-useast2a-ve-0068c003/235ea1e4a14847fe9c14070bcb4e5118/?a=1233&br=2494&bt=1247&cr=0&cs=0&dr=0&ds=3&er=&l=2020042215260401018904915614282A09&lr=tiktok_m&qs=0&rc=M3N4ZTUzaXVuczMzPDczM0ApZDk5ODc8Z2Q8Nzo4ZTNoOWdyLXBzYTZrM2hfLS0uMTZzczUvYWMvXzEuYC80MDBhMDM6Yw%3D%3D&vl=&vr=',\n", " None,\n", " 576,\n", " 1024,\n", " 15,\n", " 15,\n", " 435325,\n", " 5359,\n", " 2989302,\n", " 1967,\n", " False,\n", " '[{\"id\":\"42164\",\"name\":\"foryou\",\"title\":\"\",\"cover\":[]},{\"id\":\"20884\",\"name\":\"viral\",\"title\":\"\",\"cover\":[]},{\"id\":\"49711363\",\"name\":\"рекомендации\",\"title\":\"\",\"cover\":[]},{\"id\":\"85849860\",\"name\":\"неловкость\",\"title\":\"Используйте этот звук и фильтр с переключением света, чтобы рассказать о своих неловких моментах 😜\",\"cover\":[\"https://p16-va-default.akamaized.net/obj/musically-maliva-obj/7cbd30f5ecb299d068c975b4fd4c8dcc\"]}]',\n", " 'https://www.tiktok.com/share/user/190163290986258432'],\n", " [6813660307831966981,\n", " '#рекомендации',\n", " 1586428917,\n", " 6581170900103299077,\n", " 'anasteisha_54',\n", " 'anasteisha_54 ',\n", " 22,\n", " 25322,\n", " 515599,\n", " 125,\n", " 3261,\n", " False,\n", " False,\n", " '❤️🌸inst: anasteisha_54🌸❤️',\n", " 'https://p16.muscdn.com/img/musically-maliva-obj/1657180466114565~c5_720x720.jpeg',\n", " 6787053043994118917,\n", " 'Август',\n", " 'Intelligency',\n", " None,\n", " 'https://sf16-sg.muscdn.com/obj/tiktok-obj/a601c4d240ff83923830d3d929104ebd.mp3',\n", " 'https://p16-va-default.akamaized.net/obj/tos-maliva-p-0068/8ce4a8d47b6d2295f75bf8ff858d6aad',\n", " 'https://p16-va-default.akamaized.net/obj/tos-maliva-p-0068/a02a966eb9d14e9590319ff8997635f0_1586428919',\n", " 'https://p16-va-default.akamaized.net/obj/tos-maliva-p-0068/e20f9516266140409ea683d61ce92157_1586428919',\n", " 'https://p16-va-default.akamaized.net/obj/tos-maliva-p-0068/a02a966eb9d14e9590319ff8997635f0_1586428919',\n", " 'https://www.tiktok.com/@anasteisha_54/video/6813660307831966981',\n", " 'https://v19.muscdn.com/7ac273c61108e6278122a70c6d2063ba/5ea0b67b/video/tos/useast2a/tos-useast2a-pve-0068/c9b3e5fdb24f47d4b640521afcbb6cf7/?a=1233&br=4434&bt=2217&cr=0&cs=0&dr=0&ds=3&er=&l=2020042215260401018904915614282A09&lr=tiktok_m&qs=0&rc=M3lkdWpxd3JpdDMzaDczM0ApPDo2OzU6NGU1N2llNjxlOWcxMjBybjRnazRfLS1gMTZzczE0MzNgYl41YDItYTMtNi46Yw%3D%3D&vl=&vr=',\n", " None,\n", " 540,\n", " 960,\n", " 15,\n", " 15,\n", " 433588,\n", " 1041,\n", " 6487612,\n", " 3149,\n", " False,\n", " '[{\"id\":\"49711363\",\"name\":\"рекомендации\",\"title\":\"\",\"cover\":[]}]',\n", " 'https://www.tiktok.com/share/user/6581170900103299077'],\n", " [6800722838371306758,\n", " '🐛даже я в шоке',\n", " 1583416683,\n", " 6612761740421087238,\n", " 'neetarmy',\n", " 'neetarmy',\n", " 137,\n", " 839445,\n", " 30096211,\n", " 1367,\n", " 5982,\n", " False,\n", " False,\n", " 'vk.com/n.e.e.t.army\\nTikTok верни ✔️\\nInstagram👇👇👇',\n", " 'https://p16.muscdn.com/img/musically-maliva-obj/1663629360751621~c5_720x720.jpeg',\n", " 6787053043994118917,\n", " 'Август',\n", " 'Intelligency',\n", " None,\n", " 'https://sf16-sg.muscdn.com/obj/tiktok-obj/a601c4d240ff83923830d3d929104ebd.mp3',\n", " 'https://p16-va-default.akamaized.net/obj/tos-maliva-p-0068/c430e389af00adb6e5accbf855be01b8',\n", " 'https://p16-va-default.akamaized.net/obj/tos-maliva-p-0068/879ee693a0ec47d3a39af4af7d5403f2_1583416687',\n", " 'https://p16-va-default.akamaized.net/obj/tos-maliva-p-0068/08fde4c1dda740548720f3c92abf1395_1583416687',\n", " 'https://p16-va-default.akamaized.net/obj/tos-maliva-p-0068/879ee693a0ec47d3a39af4af7d5403f2_1583416687',\n", " 'https://www.tiktok.com/@neetarmy/video/6800722838371306758',\n", " 'https://v16.muscdn.com/0158c2f9e943904574b0c9f950774da9/5ea0b675/video/tos/useast2a/tos-useast2a-ve-0068c001/ea1708ccaf0947abae7479fc223c11fa/?a=1233&br=2368&bt=1184&cr=0&cs=0&dr=0&ds=3&er=&l=2020042215260401018904915614282A09&lr=tiktok_m&qs=0&rc=MzRnbWczam1qczMzOjczM0ApOWllaTRnOWQ6Nzg7aGRlNGdtY2wvNjUyXmRfLS00MTZzc2BiNjBgXmBfNjUxM2JhYjE6Yw%3D%3D&vl=&vr=',\n", " None,\n", " 540,\n", " 960,\n", " 9,\n", " 9,\n", " 409698,\n", " 43901,\n", " 4877150,\n", " 10482,\n", " False,\n", " '[{\"id\":\"85849860\",\"name\":\"неловкость\",\"title\":\"Используйте этот звук и фильтр с переключением света, чтобы рассказать о своих неловких моментах 😜\",\"cover\":[\"https://p16-va-default.akamaized.net/obj/musically-maliva-obj/7cbd30f5ecb299d068c975b4fd4c8dcc\"]}]',\n", " 'https://www.tiktok.com/share/user/6612761740421087238'],\n", " [6801131413618953478,\n", " 'Нет, не месячные😂 продолжение на 10к лайков ☝🏽 #рекомендации #откройпреобразись #pov',\n", " 1583511807,\n", " 6546832067560046597,\n", " 'teolena16',\n", " '𝔞𝔩𝔢𝔫𝔞𝔱𝔢𝔬𝔬𝔲',\n", " 88,\n", " 181570,\n", " 4943892,\n", " 540,\n", " 18981,\n", " False,\n", " False,\n", " '😳НЕ ПОДПИСАЛСЯ-ПЛЮС 10 КГ😳\\n💞Алёночка👾\\n🇷🇺я живу в Перми🇷🇺\\n🔻ЖМИ 🔻',\n", " 'https://p16.muscdn.com/img/musically-maliva-obj/1662244845227013~c5_720x720.jpeg',\n", " 6787053043994118917,\n", " 'Август',\n", " 'Intelligency',\n", " None,\n", " 'https://sf16-sg.muscdn.com/obj/tiktok-obj/a601c4d240ff83923830d3d929104ebd.mp3',\n", " 'https://p16-va-default.akamaized.net/obj/tos-maliva-p-0068/39f3cffcb187d69a0dc2d8b7dd1c9dac',\n", " 'https://p16-va-default.akamaized.net/obj/tos-maliva-p-0068/011e0ed079cb44d0b3cd97d4c046dddb_1583511813',\n", " 'https://p16-va-default.akamaized.net/obj/tos-maliva-p-0068/56dc34cdb6d04dccbfa4e732a86948e2_1583511813',\n", " 'https://p16-va-default.akamaized.net/obj/tos-maliva-p-0068/011e0ed079cb44d0b3cd97d4c046dddb_1583511813',\n", " 'https://www.tiktok.com/@teolena16/video/6801131413618953478',\n", " 'https://v16.muscdn.com/a32f65f25e57ada00b934d53a90364b2/5ea0b677/video/tos/useast2a/tos-useast2a-pve-0068/debbf79d1f19429eb6fce055fc277740/?a=1233&br=1750&bt=875&cr=0&cs=0&dr=0&ds=3&er=&l=2020042215260401018904915614282A09&lr=tiktok_m&qs=0&rc=M3Y8cDtveHA6czMzOjczM0ApNzk7NjY8NDxlNztpZmg3NWdobnBwY25uZWVfLS00MTZzc2NeMjA1YGEvNS1eYl82MzY6Yw%3D%3D&vl=&vr=',\n", " None,\n", " 540,\n", " 960,\n", " 11,\n", " 11,\n", " 389340,\n", " 2492,\n", " 4782474,\n", " 3125,\n", " False,\n", " '[{\"id\":\"49711363\",\"name\":\"рекомендации\",\"title\":\"\",\"cover\":[]},{\"id\":\"1659396882963461\",\"name\":\"ОткройПреобразись\",\"title\":\"Включайся в челлендж #ОткройПреобразись 😍 Покажи своё преображение с Даниссимо под официальный трек челленджа и выиграй персональный шопинг тур или забери 300 000 рублей деньгами 💃🏼 Для участия в розыгрыше не забудь купить Даниссимо в любом магазине «Магнит» и зарегистрировать чек на сайте (переходи по ссылке ниже) — там же читай подробные правила. Акция проходит с 28 февраля по 3 апреля 2020 года.\",\"cover\":[\"https://p16-va-default.akamaized.net/obj/musically-maliva-obj/202002249cc647703ecd5a4a4db4858a\"]},{\"id\":\"180049\",\"name\":\"pov\",\"title\":\"Is this what we really look like??? Show us what life looks like when the tables are turned with #POV.\",\"cover\":[\"https://p16-va-default.akamaized.net/obj/musically-maliva-obj/1632487505818629.PNG\"]},{\"id\":\"85849860\",\"name\":\"неловкость\",\"title\":\"Используйте этот звук и фильтр с переключением света, чтобы рассказать о своих неловких моментах 😜\",\"cover\":[\"https://p16-va-default.akamaized.net/obj/musically-maliva-obj/7cbd30f5ecb299d068c975b4fd4c8dcc\"]}]',\n", " 'https://www.tiktok.com/share/user/6546832067560046597'],\n", " [6801096837991386373,\n", " 'Проблемки дев4онок🤡✌🏼#varzarevaa #рек #рекомендации #хочуврек',\n", " 1583503758,\n", " 6579127294735302662,\n", " 'varzarevaa_',\n", " 'varzarevaa',\n", " 46,\n", " 10770,\n", " 398108,\n", " 74,\n", " 514,\n", " False,\n", " False,\n", " 'показала талию в инсте\\n \\n 🌺inst: varzarevaa🌺',\n", " 'https://p16.muscdn.com/img/musically-maliva-obj/1638492613711878~c5_720x720.jpeg',\n", " 6787053043994118917,\n", " 'Август',\n", " 'Intelligency',\n", " None,\n", " 'https://sf16-sg.muscdn.com/obj/tiktok-obj/a601c4d240ff83923830d3d929104ebd.mp3',\n", " 'https://p16-va-default.akamaized.net/obj/tos-maliva-p-0068/71082067fdc834f3f04e75bb15378ff6',\n", " 'https://p16-va-default.akamaized.net/obj/tos-maliva-p-0068/e5eb3625788b415eb16168ca1fec99ce_1583503763',\n", " 'https://p16-va-default.akamaized.net/obj/tos-maliva-p-0068/8963e8e0240e43adbea8d837151cea3b_1583503763',\n", " 'https://p16-va-default.akamaized.net/obj/tos-maliva-p-0068/e5eb3625788b415eb16168ca1fec99ce_1583503763',\n", " 'https://www.tiktok.com/@varzarevaa_/video/6801096837991386373',\n", " 'https://v19.muscdn.com/d9c31153563010a31c7edd0565c02088/5ea0b677/video/tos/useast2a/tos-useast2a-pve-0068/9dcbbbdd309a40e1b95c52fa368a0130/?a=1233&br=1938&bt=969&cr=0&cs=0&dr=0&ds=3&er=&l=2020042215260401018904915614282A09&lr=tiktok_m&qs=0&rc=M2l3Z2Z1ZHM4czMzZjczM0ApN2lkZ2g7OTtkNzM7ZTRlM2dfX3MxZTZqZmVfLS0xMTZzczRfXl9fMWJeNjMuNTI2MC86Yw%3D%3D&vl=&vr=',\n", " None,\n", " 540,\n", " 960,\n", " 11,\n", " 11,\n", " 352612,\n", " 6349,\n", " 3354996,\n", " 4367,\n", " False,\n", " '[{\"id\":\"1638489839970310\",\"name\":\"varzarevaa\",\"title\":\"\",\"cover\":[]},{\"id\":\"82913700\",\"name\":\"рек\",\"title\":\"\",\"cover\":[]},{\"id\":\"49711363\",\"name\":\"рекомендации\",\"title\":\"\",\"cover\":[]},{\"id\":\"1596093786694662\",\"name\":\"хочуврек\",\"title\":\"Насколько сильно ты хочешь попасть в тренд в TikTok? Танцуй и снимай видео под этот заводной трек 🕺💃\",\"cover\":[\"https://p16-va-default.akamaized.net/obj/musically-maliva-obj/d44aeb6537fbec6abcd1e7670eddafaf\"]},{\"id\":\"85849860\",\"name\":\"неловкость\",\"title\":\"Используйте этот звук и фильтр с переключением света, чтобы рассказать о своих неловких моментах 😜\",\"cover\":[\"https://p16-va-default.akamaized.net/obj/musically-maliva-obj/7cbd30f5ecb299d068c975b4fd4c8dcc\"]}]',\n", " 'https://www.tiktok.com/share/user/6579127294735302662'],\n", " [6801426600211631366,\n", " 'Нас уже 2 миллиона 🍋!',\n", " 1583580535,\n", " 6691161048949015557,\n", " 'alicaponee',\n", " 'Ali',\n", " 14,\n", " 14217,\n", " 716449,\n", " 15,\n", " 1471,\n", " False,\n", " False,\n", " 'Пока что залёг на дно',\n", " 'https://p16.muscdn.com/img/musically-maliva-obj/1643972378278918~c5_720x720.jpeg',\n", " 6787053043994118917,\n", " 'Август',\n", " 'Intelligency',\n", " None,\n", " 'https://sf16-sg.muscdn.com/obj/tiktok-obj/a601c4d240ff83923830d3d929104ebd.mp3',\n", " 'https://p16-va-default.akamaized.net/obj/tos-maliva-p-0068/00ee77914a9c49b5a00fadfe7be15f10',\n", " 'https://p16-va-default.akamaized.net/obj/tos-maliva-p-0068/946bb6b557ab411a9c75da3ef6106100_1583580544',\n", " 'https://p16-va-default.akamaized.net/obj/tos-maliva-p-0068/9c2bc21584044b7ab7ca933c7ac50f10_1583580544',\n", " 'https://p16-va-default.akamaized.net/obj/tos-maliva-p-0068/946bb6b557ab411a9c75da3ef6106100_1583580544',\n", " 'https://www.tiktok.com/@alicaponee/video/6801426600211631366',\n", " 'https://v19.muscdn.com/73d406c49ed85583c71a933f613d04a1/5ea0b67c/video/tos/useast2a/tos-useast2a-ve-0068c002/04648e02c4d747d3bedeecab0d006592/?a=1233&br=1180&bt=590&cr=0&cs=0&dr=0&ds=3&er=&l=2020042215260401018904915614282A09&lr=tiktok_m&qs=0&rc=anl5ZTUzaW5yczMzPDczM0ApMzZoZWQ3ZmQ2N2lkNjdnNGcxZ3NzYTY0NWVfLS0uMTZzcy9hMl8vMmMwNTNhX18uXi06Yw%3D%3D&vl=&vr=',\n", " None,\n", " 540,\n", " 960,\n", " 16,\n", " 16,\n", " 340246,\n", " 21249,\n", " 2588879,\n", " 1124,\n", " False,\n", " '[]',\n", " 'https://www.tiktok.com/share/user/6691161048949015557'],\n", " [6800334757877058822,\n", " 'Подпишись на инсту satanoki🌸🤰🏻#рекомендации #хочуврекомендации #хочуврек',\n", " 1583326324,\n", " 6762097102354400261,\n", " 'satanoki',\n", " 'Satanoki🌸',\n", " 137,\n", " 242511,\n", " 7869851,\n", " 145,\n", " 2319,\n", " False,\n", " False,\n", " 'Подписывайся на инстаграм satanoki💕\\nСлежу за новыми подписчиками\\nКалининград🌸',\n", " 'https://p16.muscdn.com/img/musically-maliva-obj/1663036220518405~c5_720x720.jpeg',\n", " 6787053043994118917,\n", " 'Август',\n", " 'Intelligency',\n", " None,\n", " 'https://sf16-sg.muscdn.com/obj/tiktok-obj/a601c4d240ff83923830d3d929104ebd.mp3',\n", " 'https://p16-va-default.akamaized.net/obj/tos-maliva-p-0068/09bc7a91977d9397417412292083decb',\n", " 'https://p16-va-default.akamaized.net/obj/tos-maliva-p-0068/03862c7306484c589b22a9fc4f5db403_1583326329',\n", " 'https://p16-va-default.akamaized.net/obj/tos-maliva-p-0068/f658392105b1423f8353e09ae4a85c98_1583326330',\n", " 'https://p16-va-default.akamaized.net/obj/tos-maliva-p-0068/03862c7306484c589b22a9fc4f5db403_1583326329',\n", " 'https://www.tiktok.com/@satanoki/video/6800334757877058822',\n", " 'https://v16.muscdn.com/ed6782295efc9915ee243916b993392c/5ea0b676/video/tos/useast2a/tos-useast2a-pve-0068/1352995dfe89467aa3c0c2fb503bb93b/?a=1233&br=1674&bt=837&cr=0&cs=0&dr=0&ds=3&er=&l=2020042215260401018904915614282A09&lr=tiktok_m&qs=0&rc=ajNnPHB3ZXN0czMzZDczM0ApZjw0OjMzZmQ0NzY8ZDk4PGczMjYyXzZhNWNfLS1gMTZzczUwMmE1YzE0My01MDYtMi46Yw%3D%3D&vl=&vr=',\n", " None,\n", " 540,\n", " 960,\n", " 10,\n", " 10,\n", " 336498,\n", " 7062,\n", " 1852912,\n", " 2487,\n", " False,\n", " '[{\"id\":\"49711363\",\"name\":\"рекомендации\",\"title\":\"\",\"cover\":[]},{\"id\":\"86025601\",\"name\":\"хочуврекомендации\",\"title\":\"\",\"cover\":[]},{\"id\":\"1596093786694662\",\"name\":\"хочуврек\",\"title\":\"Насколько сильно ты хочешь попасть в тренд в TikTok? Танцуй и снимай видео под этот заводной трек 🕺💃\",\"cover\":[\"https://p16-va-default.akamaized.net/obj/musically-maliva-obj/d44aeb6537fbec6abcd1e7670eddafaf\"]},{\"id\":\"85849860\",\"name\":\"неловкость\",\"title\":\"Используйте этот звук и фильтр с переключением света, чтобы рассказать о своих неловких моментах 😜\",\"cover\":[\"https://p16-va-default.akamaized.net/obj/musically-maliva-obj/7cbd30f5ecb299d068c975b4fd4c8dcc\"]}]',\n", " 'https://www.tiktok.com/share/user/6762097102354400261'],\n", " [6803319238758092038,\n", " 'дев@4ки..🐝💓',\n", " 1584021204,\n", " 310271783625179136,\n", " 'quidlir',\n", " 'ликвидатор что ли ',\n", " 21,\n", " 1495046,\n", " 43247733,\n", " 578,\n", " 3916,\n", " True,\n", " False,\n", " 'Вы 🐝\\n❕Inst: quidlir❕\\nУчастник: @dream_team_house\\nPr⬇️\\nquidlir@dreamteamhouse.ru',\n", " 'https://mpak-odec1.akamaized.net/res/user_v_icon/867/icon_310271783625179136-APHXniVQBJ.jpg',\n", " 6787053043994118917,\n", " 'Август',\n", " 'Intelligency',\n", " None,\n", " 'https://sf16-sg.muscdn.com/obj/tiktok-obj/a601c4d240ff83923830d3d929104ebd.mp3',\n", " 'https://p16-va-default.akamaized.net/obj/tos-maliva-p-0068/c4783ba693529e72e2408c3733fb17ed',\n", " 'https://p16-va-default.akamaized.net/obj/tos-maliva-p-0068/1f10e7c359a14e358eed95181487b8da_1584021210',\n", " 'https://p16-va-default.akamaized.net/obj/tos-maliva-p-0068/a1166d901c564153bfde080561b404f9_1584021210',\n", " 'https://p16-va-default.akamaized.net/obj/tos-maliva-p-0068/1f10e7c359a14e358eed95181487b8da_1584021210',\n", " 'https://www.tiktok.com/@quidlir/video/6803319238758092038',\n", " 'https://v19.muscdn.com/6b98b860fb3c0cce062483bbe3fa32c9/5ea0b677/video/tos/useast2a/tos-useast2a-ve-0068c002/f7f790b062b742c69c7b6d47a396ed4c/?a=1233&br=2298&bt=1149&cr=0&cs=0&dr=0&ds=3&er=&l=2020042215260401018904915614282A09&lr=tiktok_m&qs=0&rc=Mzg1dnZ2cTw2czMzPDczM0ApOmQzZDk2PDs7Nzg1NmRkaWdlMW9pajNfcWlfLS01MTZzczYyMTFjNWEtLzVgNl4wLWM6Yw%3D%3D&vl=&vr=',\n", " None,\n", " 540,\n", " 960,\n", " 11,\n", " 11,\n", " 331706,\n", " 817,\n", " 2020239,\n", " 554,\n", " False,\n", " '[]',\n", " 'https://www.tiktok.com/share/user/310271783625179136']]" ] }, "execution_count": 108, "metadata": {}, "output_type": "execute_result" } ], "source": [ "json.loads(d)" ] }, { "cell_type": "code", "execution_count": 15, "metadata": {}, "outputs": [ { "ename": "KeyError", "evalue": "'authorMeta.fans'", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mKeyError\u001b[0m Traceback (most recent call last)", "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 13\u001b[0m \u001b[0mmultitoks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0magg\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0maggs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msort_values\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'musicMeta.musicName'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mascending\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 14\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m \u001b[0mmdf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmultitoks\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mmultitoks\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'musicMeta.musicId'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msort_values\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'authorMeta.fans'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mascending\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 16\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 17\u001b[0m \u001b[0mmdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'Scout'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'authorMeta.name'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m~/src/thundr/tracker/venv/lib/python3.7/site-packages/pandas/core/frame.py\u001b[0m in \u001b[0;36msort_values\u001b[0;34m(self, by, axis, ascending, inplace, kind, na_position, ignore_index)\u001b[0m\n\u001b[1;32m 4925\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 4926\u001b[0m \u001b[0mby\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mby\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 4927\u001b[0;31m \u001b[0mk\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_label_or_level_values\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mby\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0maxis\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 4928\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 4929\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mascending\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mtuple\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m~/src/thundr/tracker/venv/lib/python3.7/site-packages/pandas/core/generic.py\u001b[0m in \u001b[0;36m_get_label_or_level_values\u001b[0;34m(self, key, axis)\u001b[0m\n\u001b[1;32m 1690\u001b[0m \u001b[0mvalues\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maxes\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0maxis\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_level_values\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_values\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1691\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1692\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1693\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1694\u001b[0m \u001b[0;31m# Check for duplicates\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;31mKeyError\u001b[0m: 'authorMeta.fans'" ] } ], "source": [ "g = all_df.groupby(['authorMeta.name'])\n", "\n", "aggs = {c: 'first' for c in all_df.columns}\n", "\n", "aggs.update({\n", " 'createTime': 'min',\n", " 'diggCount': 'max',\n", " 'playCount': 'max',\n", " 'musicMeta.musicId': 'nunique',\n", " 'musicMeta.musicName': set\n", "})\n", "\n", "multitoks = g.agg(aggs).sort_values('musicMeta.musicName', ascending=False)\n", "\n", "mdf = multitoks[multitoks['musicMeta.musicId'] > 1].sort_values('authorMeta.fans', ascending=False)\n", "\n", "mdf['Scout'] = mdf['authorMeta.name']\n", "\n", "from datetime import datetime\n", "\n", "mdf['Created'] = mdf.createTime.apply(datetime.fromtimestamp)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.7.1" } }, "nbformat": 4, "nbformat_minor": 2 }