{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# Feature Store " ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "This notebook demonstrates how to get started with Feature Store, create feature groups, and ingest data into them. These feature groups are stored in your Feature Store.\n", "\n", "Feature groups are resources that contain metadata for all data stored in your Feature Store. A feature group is a logical grouping of features, defined in the feature store to describe records. A feature group’s definition is composed of a list of feature definitions, a record identifier name, and configurations for its online and offline store. \n", "\n", "### Overview\n", "1. Set up\n", "2. Creating a feature group\n", "3. Ingest data into a feature group\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Set up" ] }, { "cell_type": "code", "execution_count": 56, "metadata": {}, "outputs": [], "source": [ "# SageMaker Python SDK version 2.100.0 is required\n", "# boto3 version 1.24.20 is required\n", "import sagemaker\n", "import boto3\n", "import sys\n" ] }, { "cell_type": "code", "execution_count": 57, "metadata": {}, "outputs": [], "source": [ "import pandas as pd\n", "import numpy as np\n", "import io\n", "from sagemaker.session import Session\n", "from sagemaker import get_execution_role\n", "\n", "prefix = \"sagemaker-featurestore-example\"\n", "role = get_execution_role()\n", "\n", "sagemaker_session = sagemaker.Session()\n", "region = sagemaker_session.boto_region_name\n", "s3_bucket_name = sagemaker_session.default_bucket()" ] }, { "cell_type": "code", "execution_count": 58, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "'sagemaker-us-east-1-103233932089'" ] }, "execution_count": 58, "metadata": {}, "output_type": "execute_result" } ], "source": [ "s3_bucket_name" ] }, { "cell_type": "code", "execution_count": 59, "metadata": {}, "outputs": [], "source": [ "sample_data_df = pd.read_csv(\"sample_df.csv\")\n" ] }, { "cell_type": "code", "execution_count": 60, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
ISRCPRED_TYPEMSERMSENRMSEAVG_RMSEMAEAVG_MAESUM_FORECAST_ERRORSAVG_FELEN_DFFIRST_DAY_TRAIN_SET_STREAMSLAST_DAY_TRAIN_SET_STREAMSLINEAR_GRADIENTAVERAGE_DAILY_STREAMSTEST_DATA_MIN_DATE_STREAMSTEST_DATA_MAX_DATE_STREAMSTEST_DATA_LINEAR_GRADIENTMEDIAN_STREAMS
0QMBZ92156553actuals_crossing_predicted149.13664412.2121510.8722970.0872304.1829500.02987811.8640.08474314049620.09848547.91729344460.28571446.0
1USQE10910023actuals_crossing_predicted67.3648568.2076100.2647620.05862629.8424340.213160-18.904-0.13502914015577-0.590909111.18797079871.142857107.0
2QM4TX1960783actuals_crossing_predicted593.51370124.3621370.4350380.17401527.6051890.19718094.4030.67430714084880.03030377.5338357972-1.00000072.0
3USLZJ2190718actuals_crossing_predicted55013.011845234.5485280.2822491.675347411.3134732.937953-463.238-3.308843140244836048.7575763137.39097740433212-118.7142863011.0
4QMDA72177286actuals_crossing_predicted2572.35864250.7184250.4057470.36227428.1287090.200919243.0241.735886140246179-0.507576204.12782017828214.857143196.0
\n", "
" ], "text/plain": [ " ISRC PRED_TYPE MSE RMSE \\\n", "0 QMBZ92156553 actuals_crossing_predicted 149.136644 12.212151 \n", "1 USQE10910023 actuals_crossing_predicted 67.364856 8.207610 \n", "2 QM4TX1960783 actuals_crossing_predicted 593.513701 24.362137 \n", "3 USLZJ2190718 actuals_crossing_predicted 55013.011845 234.548528 \n", "4 QMDA72177286 actuals_crossing_predicted 2572.358642 50.718425 \n", "\n", " NRMSE AVG_RMSE MAE AVG_MAE SUM_FORECAST_ERRORS AVG_FE \\\n", "0 0.872297 0.087230 4.182950 0.029878 11.864 0.084743 \n", "1 0.264762 0.058626 29.842434 0.213160 -18.904 -0.135029 \n", "2 0.435038 0.174015 27.605189 0.197180 94.403 0.674307 \n", "3 0.282249 1.675347 411.313473 2.937953 -463.238 -3.308843 \n", "4 0.405747 0.362274 28.128709 0.200919 243.024 1.735886 \n", "\n", " LEN_DF FIRST_DAY_TRAIN_SET_STREAMS LAST_DAY_TRAIN_SET_STREAMS \\\n", "0 140 49 62 \n", "1 140 155 77 \n", "2 140 84 88 \n", "3 140 2448 3604 \n", "4 140 246 179 \n", "\n", " LINEAR_GRADIENT AVERAGE_DAILY_STREAMS TEST_DATA_MIN_DATE_STREAMS \\\n", "0 0.098485 47.917293 44 \n", "1 -0.590909 111.187970 79 \n", "2 0.030303 77.533835 79 \n", "3 8.757576 3137.390977 4043 \n", "4 -0.507576 204.127820 178 \n", "\n", " TEST_DATA_MAX_DATE_STREAMS TEST_DATA_LINEAR_GRADIENT MEDIAN_STREAMS \n", "0 46 0.285714 46.0 \n", "1 87 1.142857 107.0 \n", "2 72 -1.000000 72.0 \n", "3 3212 -118.714286 3011.0 \n", "4 282 14.857143 196.0 " ] }, "execution_count": 60, "metadata": {}, "output_type": "execute_result" } ], "source": [ "sample_data_df.head()" ] }, { "cell_type": "code", "execution_count": 62, "metadata": {}, "outputs": [], "source": [ "sample_data_df['ISRC'] = sample_data_df['ISRC'].astype(str)\n", "\n", "for col in sample_data_df:\n", " if pd.api.types.is_object_dtype(sample_data_df[col].dtype):\n", " sample_data_df[col] = sample_data_df[col].astype(pd.StringDtype())" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Create a feature group\n", "\n", "We first start by creating feature group names for customer_data and orders_data. Following this, we create two Feature Groups, one for `customer_data` and another for `orders_data`" ] }, { "cell_type": "code", "execution_count": 65, "metadata": {}, "outputs": [], "source": [ "from time import gmtime, strftime, sleep\n", "\n", "sample_feature_group_name = \"test-feature-group-\" + strftime(\"%d-%H-%M-%S\", gmtime())\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Instantiate a FeatureGroup object for customers_data and orders_data. " ] }, { "cell_type": "code", "execution_count": 66, "metadata": {}, "outputs": [], "source": [ "from sagemaker.feature_store.feature_group import FeatureGroup\n", "\n", "sample_feature_group = FeatureGroup(\n", " name=sample_feature_group_name, sagemaker_session=sagemaker_session\n", ")\n" ] }, { "cell_type": "code", "execution_count": 67, "metadata": {}, "outputs": [], "source": [ "import time\n", "\n", "current_time_sec = int(round(time.time()))\n", "\n", "record_identifier_feature_name = \"ISRC\"\n", "sample_data_df[\"EventTime\"] = pd.Series([current_time_sec] * len(sample_data_df), dtype=\"float64\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Append `EventTime` feature to your data frame. This parameter is required, and time stamps each data point." ] }, { "cell_type": "code", "execution_count": 68, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[FeatureDefinition(feature_name='ISRC', feature_type=),\n", " FeatureDefinition(feature_name='PRED_TYPE', feature_type=),\n", " FeatureDefinition(feature_name='MSE', feature_type=),\n", " FeatureDefinition(feature_name='RMSE', feature_type=),\n", " FeatureDefinition(feature_name='NRMSE', feature_type=),\n", " FeatureDefinition(feature_name='AVG_RMSE', feature_type=),\n", " FeatureDefinition(feature_name='MAE', feature_type=),\n", " FeatureDefinition(feature_name='AVG_MAE', feature_type=),\n", " FeatureDefinition(feature_name='SUM_FORECAST_ERRORS', feature_type=),\n", " FeatureDefinition(feature_name='AVG_FE', feature_type=),\n", " FeatureDefinition(feature_name='LEN_DF', feature_type=),\n", " FeatureDefinition(feature_name='FIRST_DAY_TRAIN_SET_STREAMS', feature_type=),\n", " FeatureDefinition(feature_name='LAST_DAY_TRAIN_SET_STREAMS', feature_type=),\n", " FeatureDefinition(feature_name='LINEAR_GRADIENT', feature_type=),\n", " FeatureDefinition(feature_name='AVERAGE_DAILY_STREAMS', feature_type=),\n", " FeatureDefinition(feature_name='TEST_DATA_MIN_DATE_STREAMS', feature_type=),\n", " FeatureDefinition(feature_name='TEST_DATA_MAX_DATE_STREAMS', feature_type=),\n", " FeatureDefinition(feature_name='TEST_DATA_LINEAR_GRADIENT', feature_type=),\n", " FeatureDefinition(feature_name='MEDIAN_STREAMS', feature_type=),\n", " FeatureDefinition(feature_name='EventTime', feature_type=)]" ] }, "execution_count": 68, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# load feature definitions\n", "sample_feature_group.load_feature_definitions(data_frame=sample_data_df)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Below we call create to create two feature groups, customers_feature_group and orders_feature_group respectively" ] }, { "cell_type": "code", "execution_count": 69, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'FeatureGroupArn': 'arn:aws:sagemaker:us-east-1:103233932089:feature-group/test-feature-group-21-11-54-28',\n", " 'ResponseMetadata': {'RequestId': 'abde8d0a-4773-47ff-aa5f-a4c173b3b32a',\n", " 'HTTPStatusCode': 200,\n", " 'HTTPHeaders': {'x-amzn-requestid': 'abde8d0a-4773-47ff-aa5f-a4c173b3b32a',\n", " 'content-type': 'application/x-amz-json-1.1',\n", " 'content-length': '107',\n", " 'date': 'Fri, 21 Jul 2023 11:54:49 GMT'},\n", " 'RetryAttempts': 0}}" ] }, "execution_count": 69, "metadata": {}, "output_type": "execute_result" } ], "source": [ "sample_feature_group.create(\n", " s3_uri=f\"s3://{s3_bucket_name}/{prefix}\",\n", " record_identifier_name=record_identifier_feature_name,\n", " event_time_feature_name=\"EventTime\",\n", " role_arn=role,\n", " enable_online_store=False,\n", ")\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "To confirm that your FeatureGroup has been created we use `DescribeFeatureGroup` and `ListFeatureGroups` APIs to display the created FeatureGroup." ] }, { "cell_type": "code", "execution_count": 70, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'FeatureGroupArn': 'arn:aws:sagemaker:us-east-1:103233932089:feature-group/test-feature-group-21-11-54-28',\n", " 'FeatureGroupName': 'test-feature-group-21-11-54-28',\n", " 'RecordIdentifierFeatureName': 'ISRC',\n", " 'EventTimeFeatureName': 'EventTime',\n", " 'FeatureDefinitions': [{'FeatureName': 'ISRC', 'FeatureType': 'String'},\n", " {'FeatureName': 'PRED_TYPE', 'FeatureType': 'String'},\n", " {'FeatureName': 'MSE', 'FeatureType': 'Fractional'},\n", " {'FeatureName': 'RMSE', 'FeatureType': 'Fractional'},\n", " {'FeatureName': 'NRMSE', 'FeatureType': 'Fractional'},\n", " {'FeatureName': 'AVG_RMSE', 'FeatureType': 'Fractional'},\n", " {'FeatureName': 'MAE', 'FeatureType': 'Fractional'},\n", " {'FeatureName': 'AVG_MAE', 'FeatureType': 'Fractional'},\n", " {'FeatureName': 'SUM_FORECAST_ERRORS', 'FeatureType': 'Fractional'},\n", " {'FeatureName': 'AVG_FE', 'FeatureType': 'Fractional'},\n", " {'FeatureName': 'LEN_DF', 'FeatureType': 'Integral'},\n", " {'FeatureName': 'FIRST_DAY_TRAIN_SET_STREAMS', 'FeatureType': 'Integral'},\n", " {'FeatureName': 'LAST_DAY_TRAIN_SET_STREAMS', 'FeatureType': 'Integral'},\n", " {'FeatureName': 'LINEAR_GRADIENT', 'FeatureType': 'Fractional'},\n", " {'FeatureName': 'AVERAGE_DAILY_STREAMS', 'FeatureType': 'Fractional'},\n", " {'FeatureName': 'TEST_DATA_MIN_DATE_STREAMS', 'FeatureType': 'Integral'},\n", " {'FeatureName': 'TEST_DATA_MAX_DATE_STREAMS', 'FeatureType': 'Integral'},\n", " {'FeatureName': 'TEST_DATA_LINEAR_GRADIENT', 'FeatureType': 'Fractional'},\n", " {'FeatureName': 'MEDIAN_STREAMS', 'FeatureType': 'Fractional'},\n", " {'FeatureName': 'EventTime', 'FeatureType': 'Fractional'}],\n", " 'CreationTime': datetime.datetime(2023, 7, 21, 11, 54, 50, 219000, tzinfo=tzlocal()),\n", " 'OfflineStoreConfig': {'S3StorageConfig': {'S3Uri': 's3://sagemaker-us-east-1-103233932089/sagemaker-featurestore-example',\n", " 'ResolvedOutputS3Uri': 's3://sagemaker-us-east-1-103233932089/sagemaker-featurestore-example/103233932089/sagemaker/us-east-1/offline-store/test-feature-group-21-11-54-28-1689940490/data'},\n", " 'DisableGlueTableCreation': False},\n", " 'RoleArn': 'arn:aws:iam::103233932089:role/dev-accoustic-similarity-role',\n", " 'FeatureGroupStatus': 'Creating',\n", " 'ResponseMetadata': {'RequestId': 'fb85bd5c-f6cb-41b8-9437-65fe7d45e182',\n", " 'HTTPStatusCode': 200,\n", " 'HTTPHeaders': {'x-amzn-requestid': 'fb85bd5c-f6cb-41b8-9437-65fe7d45e182',\n", " 'content-type': 'application/x-amz-json-1.1',\n", " 'content-length': '1898',\n", " 'date': 'Fri, 21 Jul 2023 11:54:56 GMT'},\n", " 'RetryAttempts': 0}}" ] }, "execution_count": 70, "metadata": {}, "output_type": "execute_result" } ], "source": [ "sample_feature_group.describe()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# List Groups " ] }, { "cell_type": "code", "execution_count": 72, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'FeatureGroupSummaries': [{'FeatureGroupName': 'transaction-feature-group-13-08-41-38',\n", " 'FeatureGroupArn': 'arn:aws:sagemaker:us-east-1:103233932089:feature-group/transaction-feature-group-13-08-41-38',\n", " 'CreationTime': datetime.datetime(2023, 6, 13, 8, 42, 40, 870000, tzinfo=tzlocal()),\n", " 'FeatureGroupStatus': 'Created',\n", " 'OfflineStoreStatus': {'Status': 'Active'}},\n", " {'FeatureGroupName': 'transaction-feature-group-12-14-54-54',\n", " 'FeatureGroupArn': 'arn:aws:sagemaker:us-east-1:103233932089:feature-group/transaction-feature-group-12-14-54-54',\n", " 'CreationTime': datetime.datetime(2023, 6, 12, 14, 58, 27, 979000, tzinfo=tzlocal()),\n", " 'FeatureGroupStatus': 'Created'},\n", " {'FeatureGroupName': 'test-feature-group-21-11-54-28',\n", " 'FeatureGroupArn': 'arn:aws:sagemaker:us-east-1:103233932089:feature-group/test-feature-group-21-11-54-28',\n", " 'CreationTime': datetime.datetime(2023, 7, 21, 11, 54, 50, 219000, tzinfo=tzlocal()),\n", " 'FeatureGroupStatus': 'Creating'},\n", " {'FeatureGroupName': 'sample-feature-group-21-11-15-52',\n", " 'FeatureGroupArn': 'arn:aws:sagemaker:us-east-1:103233932089:feature-group/sample-feature-group-21-11-15-52',\n", " 'CreationTime': datetime.datetime(2023, 7, 21, 11, 15, 59, 469000, tzinfo=tzlocal()),\n", " 'FeatureGroupStatus': 'Created'},\n", " {'FeatureGroupName': 'orders-feature-group-19-16-35-14',\n", " 'FeatureGroupArn': 'arn:aws:sagemaker:us-east-1:103233932089:feature-group/orders-feature-group-19-16-35-14',\n", " 'CreationTime': datetime.datetime(2023, 7, 19, 16, 35, 46, 190000, tzinfo=tzlocal()),\n", " 'FeatureGroupStatus': 'Created',\n", " 'OfflineStoreStatus': {'Status': 'Active'}},\n", " {'FeatureGroupName': 'identity-feature-group-13-08-41-38',\n", " 'FeatureGroupArn': 'arn:aws:sagemaker:us-east-1:103233932089:feature-group/identity-feature-group-13-08-41-38',\n", " 'CreationTime': datetime.datetime(2023, 6, 13, 8, 42, 39, 727000, tzinfo=tzlocal()),\n", " 'FeatureGroupStatus': 'Created',\n", " 'OfflineStoreStatus': {'Status': 'Active'}},\n", " {'FeatureGroupName': 'customers-feature-group-19-16-35-14',\n", " 'FeatureGroupArn': 'arn:aws:sagemaker:us-east-1:103233932089:feature-group/customers-feature-group-19-16-35-14',\n", " 'CreationTime': datetime.datetime(2023, 7, 19, 16, 35, 45, 46000, tzinfo=tzlocal()),\n", " 'FeatureGroupStatus': 'Created',\n", " 'OfflineStoreStatus': {'Status': 'Active'}},\n", " {'FeatureGroupName': 'auto-mpg-2023-06-12-13-28-33',\n", " 'FeatureGroupArn': 'arn:aws:sagemaker:us-east-1:103233932089:feature-group/auto-mpg-2023-06-12-13-28-33',\n", " 'CreationTime': datetime.datetime(2023, 6, 12, 13, 40, 54, 714000, tzinfo=tzlocal()),\n", " 'FeatureGroupStatus': 'Created',\n", " 'OfflineStoreStatus': {'Status': 'Active'}}],\n", " 'ResponseMetadata': {'RequestId': '659f7c51-ce5d-49e2-9cef-d9d6ea826f76',\n", " 'HTTPStatusCode': 200,\n", " 'HTTPHeaders': {'x-amzn-requestid': '659f7c51-ce5d-49e2-9cef-d9d6ea826f76',\n", " 'content-type': 'application/x-amz-json-1.1',\n", " 'content-length': '2065',\n", " 'date': 'Fri, 21 Jul 2023 11:55:23 GMT'},\n", " 'RetryAttempts': 0}}" ] }, "execution_count": 72, "metadata": {}, "output_type": "execute_result" } ], "source": [ "sagemaker_session.boto_session.client(\n", " \"sagemaker\", region_name=region\n", ").list_feature_groups() # We use the boto client to list FeatureGroups" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 73, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "FeatureGroup test-feature-group-21-11-54-28 successfully created.\n" ] } ], "source": [ "def check_feature_group_status(feature_group):\n", " status = feature_group.describe().get(\"FeatureGroupStatus\")\n", " while status == \"Creating\":\n", " print(\"Waiting for Feature Group to be Created\")\n", " time.sleep(5)\n", " status = feature_group.describe().get(\"FeatureGroupStatus\")\n", " print(f\"FeatureGroup {feature_group.name} successfully created.\")\n", "\n", "\n", "check_feature_group_status(sample_feature_group)\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "instance_type": "ml.t3.medium", "kernelspec": { "display_name": "conda_amazonei_mxnet_p36", "language": "python", "name": "conda_amazonei_mxnet_p36" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.13" } }, "nbformat": 4, "nbformat_minor": 4 }