{ "cells": [ { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "## Set up Preview SDK" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "# Configures your AWS CLI to now understand our up and coming service Amazon Forecast\n", "!aws configure add-model --service-model file://../sdk/forecastquery-2018-06-26.normal.json --service-name forecastquery\n", "!aws configure add-model --service-model file://../sdk/forecast-2018-06-26.normal.json --service-name forecast" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Requirement already satisfied: boto3 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (1.9.69)\n", "Requirement already satisfied: s3transfer<0.2.0,>=0.1.10 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from boto3) (0.1.13)\n", "Requirement already satisfied: jmespath<1.0.0,>=0.7.1 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from boto3) (0.9.3)\n", "Requirement already satisfied: botocore<1.13.0,>=1.12.69 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from boto3) (1.12.69)\n", "Requirement already satisfied: python-dateutil<3.0.0,>=2.1; python_version >= \"2.7\" in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from botocore<1.13.0,>=1.12.69->boto3) (2.7.3)\n", "Requirement already satisfied: docutils>=0.10 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from botocore<1.13.0,>=1.12.69->boto3) (0.14)\n", "Requirement already satisfied: urllib3<1.25,>=1.20; python_version >= \"3.4\" in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from botocore<1.13.0,>=1.12.69->boto3) (1.22)\n", "Requirement already satisfied: six>=1.5 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from python-dateutil<3.0.0,>=2.1; python_version >= \"2.7\"->botocore<1.13.0,>=1.12.69->boto3) (1.11.0)\n", "\u001b[33mYou are using pip version 10.0.1, however version 18.1 is available.\n", "You should consider upgrading via the 'pip install --upgrade pip' command.\u001b[0m\n", "Requirement already satisfied: pandas in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (0.22.0)\n", "Requirement already satisfied: numpy>=1.9.0 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from pandas) (1.14.5)\n", "Requirement already satisfied: pytz>=2011k in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from pandas) (2018.4)\n", "Requirement already satisfied: python-dateutil>=2 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from pandas) (2.7.3)\n", "Requirement already satisfied: six>=1.5 in /home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages (from python-dateutil>=2->pandas) (1.11.0)\n", "\u001b[33mYou are using pip version 10.0.1, however version 18.1 is available.\n", "You should consider upgrading via the 'pip install --upgrade pip' command.\u001b[0m\n" ] } ], "source": [ "# Prerequisites : 1 time install only, remove the comments to execute the lines.\n", "!pip install boto3\n", "!pip install pahttps://ttsangforecastdemo.notebook.us-west-2.sagemaker.aws/notebooks/amazon-forecast-samples/notebooks/predict-amz-streams-forecast.ipynb#ndas" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "import boto3\n", "from time import sleep\n", "import subprocess" ] }, { "cell_type": "code", "execution_count": 65, "metadata": {}, "outputs": [], "source": [ "session = boto3.Session(region_name='us-west-2') #us-east-1 is also supported\n", "\n", "forecast = session.client(service_name='forecast')\n", "forecastquery = session.client(service_name='forecastquery')\n", "s3 = session.client('s3')\n", "accountId = boto3.client('sts').get_caller_identity().get('Account')\n", "\n", "bucketName = 'orch-sme-hackathon-012019'\n", "key=\"gravyforce/amzunlimited/amz_u_data_10440_v2.csv\"\n" ] }, { "cell_type": "code", "execution_count": 54, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "'orch-sme-hackathon-012019'" ] }, "execution_count": 54, "metadata": {}, "output_type": "execute_result" } ], "source": [ "bucketName" ] }, { "cell_type": "code", "execution_count": 78, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "'gravyforce/amzunlimited/amz_u_data_10440_v2.csv'" ] }, "execution_count": 78, "metadata": {}, "output_type": "execute_result" } ], "source": [ "key" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [], "source": [ "# One time setup only, uncomment the following command to create the role to provide to Amazon Forecast. \n", "# Save the generated role for all future calls to use for importing or exporting data. \n", "\n", "cmd = 'python ../setup_forecast_permissions.py '+bucketName\n", "p = subprocess.Popen(cmd.split(' '), stdout=subprocess.PIPE, stderr=subprocess.PIPE)\n", "roleArn = 'arn:aws:iam::%s:role/amazonforecast'%accountId" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### CreateDataset" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "More details about `Domain` and dataset type can be found on the [documentation](https://docs.aws.amazon.com/forecast/latest/dg/howitworks-domains-ds-types.html) . For this example, we are using [CUSTOM](https://docs.aws.amazon.com/forecast/latest/dg/custom-domain.html) domain with 3 required attributes `timestamp`, `target_value` and `item_id`. Also for your project name, update it to reflect your name in a lowercase format." ] }, { "cell_type": "code", "execution_count": 82, "metadata": {}, "outputs": [], "source": [ "DATASET_FREQUENCY = \"1D\" \n", "TIMESTAMP_FORMAT = \"yyyy-MM-dd\"\n", "project = 'amz_u_streams_v6' # Replace this with a unique name here, make sure the entire name is < 30 characters.\n", "datasetName= project+'_ds'\n", "datasetGroupName= project +'_gp'\n", "s3DataPath = \"s3://\"+bucketName+\"/\"+key" ] }, { "cell_type": "code", "execution_count": 83, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "'amz_u_streams_v6_ds'" ] }, "execution_count": 83, "metadata": {}, "output_type": "execute_result" } ], "source": [ "datasetName" ] }, { "cell_type": "code", "execution_count": 79, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "'s3://orch-sme-hackathon-012019/gravyforce/amzunlimited/amz_u_data_10440_v2.csv'" ] }, "execution_count": 79, "metadata": {}, "output_type": "execute_result" } ], "source": [ "s3DataPath" ] }, { "cell_type": "code", "execution_count": 84, "metadata": {}, "outputs": [], "source": [ "# Specify the schema of your dataset here. Make sure the order of columns matches the raw data files.\n", "schema ={\n", " \"Attributes\":[\n", " {\n", " \"AttributeName\":\"timestamp\",\n", " \"AttributeType\":\"timestamp\"\n", " },\n", " {\n", " \"AttributeName\":\"item_id\",\n", " \"AttributeType\":\"string\"\n", " },\n", " {\n", " \"AttributeName\":\"s_streams\",\n", " \"AttributeType\":\"integer\"\n", " },\n", " {\n", " \"AttributeName\":\"m_streams\",\n", " \"AttributeType\":\"integer\"\n", " },\n", " {\n", " \"AttributeName\":\"target_value\",\n", " \"AttributeType\":\"float\"\n", " }\n", " ]\n", "}\n", "\n", "response=forecast.create_dataset(\n", " Domain=\"CUSTOM\",\n", " DatasetType='TARGET_TIME_SERIES',\n", " DataFormat='CSV',\n", " DatasetName=datasetName,\n", " DataFrequency=DATASET_FREQUENCY, \n", " TimeStampFormat=TIMESTAMP_FORMAT,\n", " Schema = schema\n", " )" ] }, { "cell_type": "code", "execution_count": 85, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'DatasetName': 'amz_u_streams_v6_ds',\n", " 'DatasetType': 'TARGET_TIME_SERIES',\n", " 'DataFormat': 'CSV',\n", " 'Domain': 'CUSTOM',\n", " 'ScheduleExpression': 'none',\n", " 'DatasetArn': 'arn:aws:forecast:us-west-2:073138737237:ds/amz_u_streams_v6_ds',\n", " 'Status': 'ACTIVE',\n", " 'ResponseMetadata': {'RequestId': '1431be1c-4cc1-415a-9df7-63fdd9dffda2',\n", " 'HTTPStatusCode': 200,\n", " 'HTTPHeaders': {'content-type': 'application/x-amz-json-1.1',\n", " 'date': 'Fri, 11 Jan 2019 06:36:12 GMT',\n", " 'x-amzn-requestid': '1431be1c-4cc1-415a-9df7-63fdd9dffda2',\n", " 'content-length': '233',\n", " 'connection': 'keep-alive'},\n", " 'RetryAttempts': 0}}" ] }, "execution_count": 85, "metadata": {}, "output_type": "execute_result" } ], "source": [ "forecast.describe_dataset(DatasetName=datasetName)" ] }, { "cell_type": "code", "execution_count": 86, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'DatasetGroupName': 'amz_u_streams_v6_gp',\n", " 'DatasetGroupArn': 'arn:aws:forecast:us-west-2:073138737237:dsgroup/amz_u_streams_v6_gp',\n", " 'ResponseMetadata': {'RequestId': '27aabc0b-1d77-4dd8-9df5-d84e75a32186',\n", " 'HTTPStatusCode': 200,\n", " 'HTTPHeaders': {'content-type': 'application/x-amz-json-1.1',\n", " 'date': 'Fri, 11 Jan 2019 06:36:16 GMT',\n", " 'x-amzn-requestid': '27aabc0b-1d77-4dd8-9df5-d84e75a32186',\n", " 'content-length': '130',\n", " 'connection': 'keep-alive'},\n", " 'RetryAttempts': 0}}" ] }, "execution_count": 86, "metadata": {}, "output_type": "execute_result" } ], "source": [ "forecast.create_dataset_group(DatasetGroupName=datasetGroupName,RoleArn=roleArn,DatasetNames=[datasetName])" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "If you have an existing datasetgroup, you can update it" ] }, { "cell_type": "code", "execution_count": 87, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'DatasetGroupName': 'amz_u_streams_v6_gp',\n", " 'DatasetGroupArn': 'arn:aws:forecast:us-west-2:073138737237:dsgroup/amz_u_streams_v6_gp',\n", " 'Datasets': ['amz_u_streams_v6_ds'],\n", " 'RoleArn': 'arn:aws:iam::073138737237:role/amazonforecast',\n", " 'ResponseMetadata': {'RequestId': '18a3d3e9-b163-40e0-a4e7-67dc9e507a35',\n", " 'HTTPStatusCode': 200,\n", " 'HTTPHeaders': {'content-type': 'application/x-amz-json-1.1',\n", " 'date': 'Fri, 11 Jan 2019 06:36:20 GMT',\n", " 'x-amzn-requestid': '18a3d3e9-b163-40e0-a4e7-67dc9e507a35',\n", " 'content-length': '223',\n", " 'connection': 'keep-alive'},\n", " 'RetryAttempts': 0}}" ] }, "execution_count": 87, "metadata": {}, "output_type": "execute_result" } ], "source": [ "forecast.describe_dataset_group(DatasetGroupName=datasetGroupName)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Create Data Import Job\n", "Brings the data into Amazon Forecast system ready to forecast from raw data. " ] }, { "cell_type": "code", "execution_count": 88, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "afb7e470\n" ] } ], "source": [ "ds_import_job_response=forecast.create_dataset_import_job(DatasetName=datasetName,Delimiter=',', DatasetGroupName =datasetGroupName ,S3Uri= s3DataPath)\n", "ds_versionId=ds_import_job_response['VersionId']\n", "print(ds_versionId)" ] }, { "cell_type": "code", "execution_count": 89, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "CREATING\n", "0\n", "CREATING\n", "1\n", "CREATING\n", "2\n", "CREATING\n", "3\n", "CREATING\n", "4\n", "CREATING\n", "5\n", "CREATING\n", "6\n", "CREATING\n", "7\n", "CREATING\n", "8\n", "CREATING\n", "9\n", "CREATING\n", "10\n", "CREATING\n", "11\n", "CREATING\n", "12\n", "CREATING\n", "13\n", "CREATING\n", "14\n", "CREATING\n", "15\n", "CREATING\n", "16\n", "CREATING\n", "17\n", "CREATING\n", "18\n", "CREATING\n", "19\n", "CREATING\n", "20\n", "CREATING\n", "21\n", "ACTIVE\n", "22\n", "total time 230\n" ] } ], "source": [ "i = 0\n", "while True:\n", " dataImportStatus = forecast.describe_dataset_import_job(DatasetName=datasetName,VersionId=ds_versionId)['Status']\n", " print(dataImportStatus)\n", " print(i)\n", " i=i+1\n", " if dataImportStatus != 'ACTIVE' and dataImportStatus != 'FAILED':\n", " sleep(10)\n", " else:\n", " break\n", "print('total time {}'.format(i*10))" ] }, { "cell_type": "code", "execution_count": 100, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'DatasetArn': 'arn:aws:forecast:us-west-2:073138737237:ds/amz_u_streams_v6_ds',\n", " 'DatasetName': 'amz_u_streams_v6_ds',\n", " 'VersionId': 'afb7e470',\n", " 'Status': 'ACTIVE',\n", " 'FieldStatistics': {'date': {'Count': 26,\n", " 'CountDistinct': 26,\n", " 'CountNull': 0,\n", " 'Min': '2018-12-15T00:00:00Z',\n", " 'Max': '2019-01-09T00:00:00Z'},\n", " 'item': {'Count': 26, 'CountDistinct': 1, 'CountNull': 0},\n", " 'm_streams': {'Count': 26,\n", " 'CountDistinct': 24,\n", " 'CountNull': 0,\n", " 'Min': '199',\n", " 'Max': '522',\n", " 'Avg': 312.65384615384613,\n", " 'Stddev': 75.19199016261895},\n", " 's_streams': {'Count': 26,\n", " 'CountDistinct': 26,\n", " 'CountNull': 0,\n", " 'Min': '3655',\n", " 'Max': '14882',\n", " 'Avg': 11550.115384615385,\n", " 'Stddev': 2343.175662675303},\n", " 'target': {'Count': 26,\n", " 'CountDistinct': 26,\n", " 'CountNull': 0,\n", " 'Min': '3918.0',\n", " 'Max': '15158.0',\n", " 'Avg': 11862.76923076923,\n", " 'Stddev': 2342.4957683239013}},\n", " 'ResponseMetadata': {'RequestId': '80590e57-1782-4793-a941-b99690e06261',\n", " 'HTTPStatusCode': 200,\n", " 'HTTPHeaders': {'content-type': 'application/x-amz-json-1.1',\n", " 'date': 'Fri, 11 Jan 2019 07:07:27 GMT',\n", " 'x-amzn-requestid': '80590e57-1782-4793-a941-b99690e06261',\n", " 'content-length': '748',\n", " 'connection': 'keep-alive'},\n", " 'RetryAttempts': 0}}" ] }, "execution_count": 100, "metadata": {}, "output_type": "execute_result" } ], "source": [ "forecast.describe_dataset_import_job(DatasetName=datasetName,VersionId=ds_versionId)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Recipe" ] }, { "cell_type": "code", "execution_count": 101, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'RecipeNames': ['forecast_ARIMA',\n", " 'forecast_DEEP_AR',\n", " 'forecast_DEEP_AR_PLUS',\n", " 'forecast_ETS',\n", " 'forecast_MDN',\n", " 'forecast_MQRNN',\n", " 'forecast_NPTS',\n", " 'forecast_PROPHET',\n", " 'forecast_SQF'],\n", " 'ResponseMetadata': {'RequestId': '0e056f29-8613-4992-bf1c-d053060b87c7',\n", " 'HTTPStatusCode': 200,\n", " 'HTTPHeaders': {'content-type': 'application/x-amz-json-1.1',\n", " 'date': 'Fri, 11 Jan 2019 07:07:27 GMT',\n", " 'x-amzn-requestid': '0e056f29-8613-4992-bf1c-d053060b87c7',\n", " 'content-length': '174',\n", " 'connection': 'keep-alive'},\n", " 'RetryAttempts': 0}}" ] }, "execution_count": 101, "metadata": {}, "output_type": "execute_result" } ], "source": [ "recipesResponse=forecast.list_recipes()\n", "recipesResponse" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Get details about each recipe." ] }, { "cell_type": "code", "execution_count": 102, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'Recipe': {'Name': 'forecast_MQRNN',\n", " 'Train': [{'TrainingInfo': {'TrainedModelName': 'algorithm_MQRNN',\n", " 'AlgorithmName': 'MQRNN',\n", " 'TrainingParameters': {'epochs': '60',\n", " 'learning_rate': '3E-3',\n", " 'mini_batch_size': '32',\n", " 'quantiles': '[0.1,0.5,0.9]'}},\n", " 'BackTestWindowCount': 1,\n", " 'MetricsBuckets': []}]},\n", " 'ResponseMetadata': {'RequestId': '8c885f1c-c487-43f3-97ee-07e67afaa058',\n", " 'HTTPStatusCode': 200,\n", " 'HTTPHeaders': {'content-type': 'application/x-amz-json-1.1',\n", " 'date': 'Fri, 11 Jan 2019 07:07:27 GMT',\n", " 'x-amzn-requestid': '8c885f1c-c487-43f3-97ee-07e67afaa058',\n", " 'content-length': '281',\n", " 'connection': 'keep-alive'},\n", " 'RetryAttempts': 0}}" ] }, "execution_count": 102, "metadata": {}, "output_type": "execute_result" } ], "source": [ "forecast.describe_recipe(RecipeName='forecast_MQRNN')" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Create Solution with customer forecast horizon" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Forecast horizon is how long in future the forecast should be predicting. For weekly data, a value of 12 means 1 weeks. Our example is hourly data, we try forecast the next day, so we can set to 24." ] }, { "cell_type": "code", "execution_count": 93, "metadata": {}, "outputs": [], "source": [ "predictorName= project+'_mqrnn_v9'" ] }, { "cell_type": "code", "execution_count": 94, "metadata": {}, "outputs": [], "source": [ "forecastHorizon = 2" ] }, { "cell_type": "code", "execution_count": 95, "metadata": {}, "outputs": [], "source": [ "createPredictorResponse=forecast.create_predictor(RecipeName='forecast_MQRNN',DatasetGroupName= datasetGroupName ,PredictorName=predictorName, \n", " ForecastHorizon = forecastHorizon)" ] }, { "cell_type": "code", "execution_count": 96, "metadata": {}, "outputs": [], "source": [ "predictorVerionId=createPredictorResponse['VersionId']" ] }, { "cell_type": "code", "execution_count": 97, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'PredictorVersions': [{'PredictorName': 'amz_u_streams_v6_mqrnn_v9',\n", " 'VersionId': '7e4c4828'}],\n", " 'ResponseMetadata': {'RequestId': '1b35d2ee-5f51-4223-875d-39e23013ea6e',\n", " 'HTTPStatusCode': 200,\n", " 'HTTPHeaders': {'content-type': 'application/x-amz-json-1.1',\n", " 'date': 'Fri, 11 Jan 2019 06:42:50 GMT',\n", " 'x-amzn-requestid': '1b35d2ee-5f51-4223-875d-39e23013ea6e',\n", " 'content-length': '92',\n", " 'connection': 'keep-alive'},\n", " 'RetryAttempts': 0}}" ] }, "execution_count": 97, "metadata": {}, "output_type": "execute_result" } ], "source": [ "forecast.list_predictor_versions(PredictorName=predictorName)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Check the status of solutions, when the status change from **CREATING** to **ACTIVE**, we can continue to next steps. Depending on data size, model selection and hyper parameters，it can take 10 mins to more than one hour to be **ACTIVE**." ] }, { "cell_type": "code", "execution_count": 99, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "CREATING\n", "0\n", "CREATING\n", "10\n", "CREATING\n", "20\n", "CREATING\n", "30\n", "CREATING\n", "40\n", "CREATING\n", "50\n", "CREATING\n", "60\n", "CREATING\n", "70\n", "CREATING\n", "80\n", "CREATING\n", "90\n", "CREATING\n", "100\n", "CREATING\n", "110\n", "CREATING\n", "120\n", "CREATING\n", "130\n", "CREATING\n", "140\n", "CREATING\n", "150\n", "CREATING\n", "160\n", "CREATING\n", "170\n", "CREATING\n", "180\n", "CREATING\n", "190\n", "CREATING\n", "200\n", "CREATING\n", "210\n", "CREATING\n", "220\n", "CREATING\n", "230\n", "CREATING\n", "240\n", "CREATING\n", "250\n", "CREATING\n", "260\n", "CREATING\n", "270\n", "CREATING\n", "280\n", "CREATING\n", "290\n", "CREATING\n", "300\n", "CREATING\n", "310\n", "CREATING\n", "320\n", "CREATING\n", "330\n", "CREATING\n", "340\n", "CREATING\n", "350\n", "CREATING\n", "360\n", "CREATING\n", "370\n", "CREATING\n", "380\n", "CREATING\n", "390\n", "CREATING\n", "400\n", "CREATING\n", "410\n", "CREATING\n", "420\n", "CREATING\n", "430\n", "CREATING\n", "440\n", "CREATING\n", "450\n", "CREATING\n", "460\n", "CREATING\n", "470\n", "CREATING\n", "480\n", "CREATING\n", "490\n", "CREATING\n", "500\n", "CREATING\n", "510\n", "CREATING\n", "520\n", "CREATING\n", "530\n", "CREATING\n", "540\n", "CREATING\n", "550\n", "CREATING\n", "560\n", "CREATING\n", "570\n", "CREATING\n", "580\n", "CREATING\n", "590\n", "CREATING\n", "600\n", "CREATING\n", "610\n", "CREATING\n", "620\n", "CREATING\n", "630\n", "CREATING\n", "640\n", "CREATING\n", "650\n", "CREATING\n", "660\n", "CREATING\n", "670\n", "CREATING\n", "680\n", "CREATING\n", "690\n", "CREATING\n", "700\n", "CREATING\n", "710\n", "CREATING\n", "720\n", "CREATING\n", "730\n", "CREATING\n", "740\n", "CREATING\n", "750\n", "CREATING\n", "760\n", "CREATING\n", "770\n", "CREATING\n", "780\n", "CREATING\n", "790\n", "CREATING\n", "800\n", "CREATING\n", "810\n", "CREATING\n", "820\n", "CREATING\n", "830\n", "CREATING\n", "840\n", "CREATING\n", "850\n", "CREATING\n", "860\n", "CREATING\n", "870\n", "CREATING\n", "880\n", "CREATING\n", "890\n", "CREATING\n", "900\n", "CREATING\n", "910\n", "CREATING\n", "920\n", "CREATING\n", "930\n", "CREATING\n", "940\n", "CREATING\n", "950\n", "CREATING\n", "960\n", "CREATING\n", "970\n", "CREATING\n", "980\n", "CREATING\n", "990\n", "CREATING\n", "1000\n", "CREATING\n", "1010\n", "CREATING\n", "1020\n", "CREATING\n", "1030\n", "CREATING\n", "1040\n", "CREATING\n", "1050\n", "CREATING\n", "1060\n", "CREATING\n", "1070\n", "CREATING\n", "1080\n", "CREATING\n", "1090\n", "CREATING\n", "1100\n", "CREATING\n", "1110\n", "CREATING\n", "1120\n", "CREATING\n", "1130\n", "CREATING\n", "1140\n", "CREATING\n", "1150\n", "CREATING\n", "1160\n", "CREATING\n", "1170\n", "CREATING\n", "1180\n", "CREATING\n", "1190\n", "CREATING\n", "1200\n", "CREATING\n", "1210\n", "CREATING\n", "1220\n", "CREATING\n", "1230\n", "CREATING\n", "1240\n", "CREATING\n", "1250\n", "CREATING\n", "1260\n", "CREATING\n", "1270\n", "CREATING\n", "1280\n", "CREATING\n", "1290\n", "CREATING\n", "1300\n", "CREATING\n", "1310\n", "CREATING\n", "1320\n", "CREATING\n", "1330\n", "CREATING\n", "1340\n", "CREATING\n", "1350\n", "CREATING\n", "1360\n", "CREATING\n", "1370\n", "CREATING\n", "1380\n", "CREATING\n", "1390\n", "ACTIVE\n", "1400\n", "total time 1410\n" ] } ], "source": [ "i = 0\n", "while True:\n", " predictorStatus = forecast.describe_predictor(PredictorName=predictorName,VersionId=predictorVerionId)['Status']\n", " print(predictorStatus)\n", " print(i*10)\n", " i=i+1\n", " if predictorStatus != 'ACTIVE' and predictorStatus != 'FAILED':\n", " sleep(10)\n", " else:\n", " break\n", "print('total time {}'.format(i*10))" ] }, { "cell_type": "code", "execution_count": 103, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'PredictorName': 'amz_u_streams_v6_mqrnn_v9',\n", " 'VersionId': '7e4c4828',\n", " 'Status': 'ACTIVE',\n", " 'CreationStartTime': '2019-01-11T06:42:42.130Z',\n", " 'CreationCompleteTime': '2019-01-11T07:07:17.942Z',\n", " 'LastModificationTime': '2019-01-11T07:07:17.942Z',\n", " 'PredictorArn': 'arn:aws:forecast:us-west-2:073138737237:predictor/amz_u_streams_v6_mqrnn_v9',\n", " 'RecipeName': 'forecast_MQRNN',\n", " 'DatasetGroup': 'amz_u_streams_v6_gp',\n", " 'RecipeParameters': {},\n", " 'ResponseMetadata': {'RequestId': '3c95e8f2-2151-4fcc-a343-d4d1d29bffb7',\n", " 'HTTPStatusCode': 200,\n", " 'HTTPHeaders': {'content-type': 'application/x-amz-json-1.1',\n", " 'date': 'Fri, 11 Jan 2019 07:11:31 GMT',\n", " 'x-amzn-requestid': '3c95e8f2-2151-4fcc-a343-d4d1d29bffb7',\n", " 'content-length': '415',\n", " 'connection': 'keep-alive'},\n", " 'RetryAttempts': 0}}" ] }, "execution_count": 103, "metadata": {}, "output_type": "execute_result" } ], "source": [ "forecast.describe_predictor(PredictorName=predictorName,VersionId=predictorVerionId)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Get Error Metrics" ] }, { "cell_type": "code", "execution_count": 104, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'ModelMetrics': {'MQRNN': {'Metrics': {'p10': '0.19795040168368416',\n", " 'p50': '0.9894149153489091',\n", " 'p90': '1.779871650055208',\n", " 'rmse': '10792.663844695682'},\n", " 'MetricsByBucket': []}},\n", " 'ResponseMetadata': {'RequestId': 'cc64ddfa-9bab-4cf9-b7d0-d7d201d0dda6',\n", " 'HTTPStatusCode': 200,\n", " 'HTTPHeaders': {'content-type': 'application/x-amz-json-1.1',\n", " 'date': 'Fri, 11 Jan 2019 07:11:37 GMT',\n", " 'x-amzn-requestid': 'cc64ddfa-9bab-4cf9-b7d0-d7d201d0dda6',\n", " 'content-length': '170',\n", " 'connection': 'keep-alive'},\n", " 'RetryAttempts': 0}}" ] }, "execution_count": 104, "metadata": {}, "output_type": "execute_result" } ], "source": [ "forecastquery.get_accuracy_metrics(PredictorName=predictorName)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Deploy Predictor" ] }, { "cell_type": "code", "execution_count": 105, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "'amz_u_streams_v6_mqrnn_v9'" ] }, "execution_count": 105, "metadata": {}, "output_type": "execute_result" } ], "source": [ "predictorName" ] }, { "cell_type": "code", "execution_count": 106, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'PredictorName': 'amz_u_streams_v6_mqrnn_v9',\n", " 'VersionId': '7e4c4828',\n", " 'PredictorArn': 'arn:aws:forecast:us-west-2:073138737237:predictor/amz_u_streams_v6_mqrnn_v9',\n", " 'ResponseMetadata': {'RequestId': 'e6b4b859-a153-48d1-b759-a53b003b4967',\n", " 'HTTPStatusCode': 200,\n", " 'HTTPHeaders': {'content-type': 'application/x-amz-json-1.1',\n", " 'date': 'Fri, 11 Jan 2019 07:14:49 GMT',\n", " 'x-amzn-requestid': 'e6b4b859-a153-48d1-b759-a53b003b4967',\n", " 'content-length': '161',\n", " 'connection': 'keep-alive'},\n", " 'RetryAttempts': 0}}" ] }, "execution_count": 106, "metadata": {}, "output_type": "execute_result" } ], "source": [ "forecast.deploy_predictor(PredictorName=predictorName)" ] }, { "cell_type": "code", "execution_count": 107, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "{'PredictorNames': ['amz_u_streams_v6_mqrnn_v9', 'forecastdemo_mattmel_mqrnn', 'meghan_erdman_mqrnn', 'rishisatsangi_forecastdemo_mqrnn', 'stefanduberg_forecastdemo_mqrnn', 'ttsang_forecastdemo_mqrnn', 'workshop_forecastdemo_relv_mqrnn'], 'ResponseMetadata': {'RequestId': 'acfa9cae-bfc7-4045-bf89-bc417168bff8', 'HTTPStatusCode': 200, 'HTTPHeaders': {'content-type': 'application/x-amz-json-1.1', 'date': 'Fri, 11 Jan 2019 07:14:54 GMT', 'x-amzn-requestid': 'acfa9cae-bfc7-4045-bf89-bc417168bff8', 'content-length': '231', 'connection': 'keep-alive'}, 'RetryAttempts': 0}}\n" ] } ], "source": [ "deployedPredictorsResponse=forecast.list_deployed_predictors()\n", "print(deployedPredictorsResponse)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Please note that the following cell can also take 10 minutes or more to be fully operational. There's no output here, but that is fine as long as the * is there." ] }, { "cell_type": "code", "execution_count": 108, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "CREATING\n", "CREATING\n", "CREATING\n", "CREATING\n", "CREATING\n", "CREATING\n", "CREATING\n", "CREATING\n", "CREATING\n", "CREATING\n", "CREATING\n", "CREATING\n", "CREATING\n", "CREATING\n", "CREATING\n", "CREATING\n", "CREATING\n", "CREATING\n", "CREATING\n", "CREATING\n", "CREATING\n", "CREATING\n", "CREATING\n", "CREATING\n", "CREATING\n", "CREATING\n", "CREATING\n", "CREATING\n", "CREATING\n", "CREATING\n", "CREATING\n", "CREATING\n", "CREATING\n", "CREATING\n", "CREATING\n", "CREATING\n", "CREATING\n", "CREATING\n", "CREATING\n", "CREATING\n", "CREATING\n", "CREATING\n", "CREATING\n", "CREATING\n", "CREATING\n", "CREATING\n", "ACTIVE\n", "ACTIVE\n" ] } ], "source": [ "while True:\n", " deployedPredictorStatus = forecast.describe_deployed_predictor(PredictorName=predictorName)['Status']\n", " print(deployedPredictorStatus)\n", " if deployedPredictorStatus != 'ACTIVE' and deployedPredictorStatus != 'FAILED':\n", " sleep(30)\n", " else:\n", " break\n", "print(deployedPredictorStatus)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Get Forecast" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "When the solution is deployed and forecast results are ready, you can view them. " ] }, { "cell_type": "code", "execution_count": 113, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "{'Forecast': {'ForecastId': '1547192132_827e9b7a',\n", " 'Predictions': {'mean': [{'Date': '2019-01-10T00:00:00',\n", " 'Val': 97.82083892822266},\n", " {'Date': '2019-01-11T00:00:00',\n", " 'Val': 101.11492156982422}],\n", " 'p10': [{'Date': '2019-01-10T00:00:00',\n", " 'Val': 96.27088165283203},\n", " {'Date': '2019-01-11T00:00:00',\n", " 'Val': 96.3293228149414}],\n", " 'p50': [{'Date': '2019-01-10T00:00:00',\n", " 'Val': 97.82083892822266},\n", " {'Date': '2019-01-11T00:00:00',\n", " 'Val': 101.11492156982422}],\n", " 'p90': [{'Date': '2019-01-10T00:00:00',\n", " 'Val': 102.67658233642578},\n", " {'Date': '2019-01-11T00:00:00',\n", " 'Val': 107.48529815673828}]}},\n", " 'ResponseMetadata': {'HTTPHeaders': {'connection': 'keep-alive',\n", " 'content-length': '541',\n", " 'content-type': 'application/x-amz-json-1.1',\n", " 'date': 'Fri, 11 Jan 2019 14:17:10 GMT',\n", " 'x-amzn-requestid': '937bb46f-1458-4b2b-9ef6-951ccb2da400'},\n", " 'HTTPStatusCode': 200,\n", " 'RequestId': '937bb46f-1458-4b2b-9ef6-951ccb2da400',\n", " 'RetryAttempts': 0}}\n" ] } ], "source": [ "forecastResponse = forecastquery.get_forecast(\n", " PredictorName=predictorName,\n", " Interval=\"day\",\n", " Filters={\"item_id\":\"10440\"}\n", ")\n", "import pprint\n", "pprint.pprint(forecastResponse)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Export Forecast" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "###### You can batch export forecast to s3 bucket. To do so an role with s3 put access is needed, but this has already been created." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "forecastInfoList= forecast.list_forecasts(PredictorName=predictorName)['ForecastInfoList']\n", "forecastId= forecastInfoList[0]['ForecastId']" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "outputPath=\"s3://\"+bucketName+\"/output\"" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "forecastExportResponse = forecast.create_forecast_export_job(ForecastId=forecastId, OutputPath={\"S3Uri\": outputPath,\"RoleArn\":roleArn})" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "forecastExportJobId = forecastExportResponse['ForecastExportJobId']" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "while True:\n", " forecastExportStatus = forecast.describe_forecast_export_job(ForecastExportJobId=forecastExportJobId)['Status']\n", " print(forecastExportStatus)\n", " if forecastExportStatus != 'ACTIVE' and forecastExportStatus != 'FAILED':\n", " sleep(30)\n", " else:\n", " break" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Check s3 bucket for results" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "s3.list_objects(Bucket=bucketName,Prefix=\"output\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Cleanup\n", "\n", "While Forecast is in preview there are no charges for using it, but to future proof this work below are the instructions to cleanup your work space." ] }, { "cell_type": "code", "execution_count": 31, "metadata": {}, "outputs": [ { "ename": "ResourceNotFoundException", "evalue": "An error occurred (ResourceNotFoundException) when calling the DeleteDeployedPredictor operation: Deployed solution not found", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mResourceNotFoundException\u001b[0m Traceback (most recent call last)", "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0;31m# Delete Deployed Predictor\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mforecast\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdelete_deployed_predictor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mPredictorName\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpredictorName\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", "\u001b[0;32m~/anaconda3/envs/python3/lib/python3.6/site-packages/botocore/client.py\u001b[0m in \u001b[0;36m_api_call\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 318\u001b[0m \"%s() only accepts keyword arguments.\" % py_operation_name)\n\u001b[1;32m 319\u001b[0m \u001b[0;31m# The \"self\" in this scope is referring to the BaseClient.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 320\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_api_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moperation_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 321\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 322\u001b[0m \u001b[0m_api_call\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__name__\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpy_operation_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m~/anaconda3/envs/python3/lib/python3.6/site-packages/botocore/client.py\u001b[0m in \u001b[0;36m_make_api_call\u001b[0;34m(self, operation_name, api_params)\u001b[0m\n\u001b[1;32m 622\u001b[0m \u001b[0merror_code\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mparsed_response\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Error\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Code\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 623\u001b[0m \u001b[0merror_class\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexceptions\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfrom_code\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0merror_code\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 624\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0merror_class\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mparsed_response\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moperation_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 625\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 626\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mparsed_response\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;31mResourceNotFoundException\u001b[0m: An error occurred (ResourceNotFoundException) when calling the DeleteDeployedPredictor operation: Deployed solution not found" ] } ], "source": [ "# Delete Deployed Predictor \n", "forecast.delete_deployed_predictor(PredictorName=predictorName)" ] }, { "cell_type": "code", "execution_count": 32, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'PredictorName': 'amz_u_streams_mqrnn_v3',\n", " 'ResponseMetadata': {'RequestId': '3147b3d3-b0df-4a27-9936-efb9442b7ac4',\n", " 'HTTPStatusCode': 200,\n", " 'HTTPHeaders': {'content-type': 'application/x-amz-json-1.1',\n", " 'date': 'Fri, 11 Jan 2019 01:03:49 GMT',\n", " 'x-amzn-requestid': '3147b3d3-b0df-4a27-9936-efb9442b7ac4',\n", " 'content-length': '42',\n", " 'connection': 'keep-alive'},\n", " 'RetryAttempts': 0}}" ] }, "execution_count": 32, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Delete the Predictor: \n", "forecast.delete_predictor(PredictorName=predictorName)" ] }, { "cell_type": "code", "execution_count": 51, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'DatasetName': 'amz_u_streams_v2_ds',\n", " 'ResponseMetadata': {'RequestId': '060d7762-d0d6-45e7-bc38-42816ce0b2e4',\n", " 'HTTPStatusCode': 200,\n", " 'HTTPHeaders': {'content-type': 'application/x-amz-json-1.1',\n", " 'date': 'Fri, 11 Jan 2019 01:43:57 GMT',\n", " 'x-amzn-requestid': '060d7762-d0d6-45e7-bc38-42816ce0b2e4',\n", " 'content-length': '37',\n", " 'connection': 'keep-alive'},\n", " 'RetryAttempts': 0}}" ] }, "execution_count": 51, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Delete Import\n", "forecast.delete_dataset_import(DatasetName=datasetName)" ] }, { "cell_type": "code", "execution_count": 52, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'ResponseMetadata': {'RequestId': '1370ce61-26a3-4ddc-929f-528fa27ded4a',\n", " 'HTTPStatusCode': 200,\n", " 'HTTPHeaders': {'content-type': 'application/x-amz-json-1.1',\n", " 'date': 'Fri, 11 Jan 2019 01:44:02 GMT',\n", " 'x-amzn-requestid': '1370ce61-26a3-4ddc-929f-528fa27ded4a',\n", " 'content-length': '2',\n", " 'connection': 'keep-alive'},\n", " 'RetryAttempts': 0}}" ] }, "execution_count": 52, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Delete Dataset Group\n", "forecast.delete_dataset_group(DatasetGroupName=datasetGroupName)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "conda_python3", "language": "python", "name": "conda_python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.5" } }, "nbformat": 4, "nbformat_minor": 2 }