import csv import enum import json import math import os import re import boto3 from botocore.exceptions import ClientError from urllib.parse import parse_qs USE_CONSTANT_WEIGHT = False SUBDIRECTORIES_FOR_PREDICTING_ID = ("users", ) ADD_REQUESTS_FILENAME = "add-requests.csv" REPLACE_REQUESTS_FILENAME = "replace-requests.csv" REMOVE_REQUESTS_FILENAME = "remove-requests.csv" HEADER_REQUESTS_FILENAME = "add-http-headers.csv" HEADER_USER_AGENT = "core-load-tester/1.0" sample_file_name = os.environ.get("SAMPLE_FILE", "samples/sample.csv") s3_bucket_requests_extra_config = os.environ.get("S3_BUCKET_REQUESTS_EXTRA_CONFIG") ignored_http_statuses_and_urls_filename = os.environ.get( "IGNORED_HTTP_STATUSES_AND_URLS_FILENAME", os.path.join(os.path.dirname(__file__), "ignored-http-statuses-and_urls.txt"), ) class ContentTypes(str, enum.Enum): form = "application/x-www-form-urlencoded" multipart_form = "multipart/form-data" json = "application/json" def add_testing_task(tasks, method, uri, query_string, body, content_type, weight, stats_name, headers=None): """ When a load test is started, an instance of a User class will be created for each simulated user When these users run they pick tasks that they execute. This method generate tasks for each items in CSV file with testing data. Args: -tasks (dict): all created tasks -method (str): HTTP method GET/POST -uri (str): url of the request -query_string (str): query string of the url -body (str): request body -content_type (str): Content type -weight (str): distributed task weights -stats_name (str): display name for locust statistic -headers (dict): request headers """ params = parse_qs(query_string) headers = headers or {} json_data = None data = None headers.update({"User-Agent": HEADER_USER_AGENT}) if body and content_type and content_type == ContentTypes.json: json_data = json.loads(body) headers.update({"Content-type": content_type}) elif body and content_type and content_type == ContentTypes.form: data = parse_qs(body) headers.update({"Content-type": content_type}) elif body: data = body headers.update({"Content-type": content_type}) def task(self): payload = dict( method=method, url=uri, params=params, data=data, json=json_data, headers=headers, name=stats_name, catch_response=True ) with self.client.request(**payload) as response: pass if USE_CONSTANT_WEIGHT: tasks[task] = 1 else: tasks[task] = int(weight) def read_sample_data() -> list: """Reads sample data from CSV file and returns it as a list of dictionaries.""" sample_data = [] with open(sample_file_name, mode="r") as sample_file: data = list(csv.DictReader(sample_file)) for item in data: if item.get("method").strip().startswith("#"): continue headers = item.get("headers") if headers: item["headers"] = json.loads(headers) sample_data.append(item) return sample_data def count_percentile(percentile, values): """Calculates 95 percentile boundary.""" if not values: return 0 sorted_values = list(sorted(list(values))) index = (percentile / 100) * len(sorted_values) if math.floor(index) == index and index: result = (sorted_values[int(index) - 1] + sorted_values[int(index)]) / 2 else: result = sorted_values[int(math.floor(index))] return result def get_ignored_http_statuses_and_urls(): with open(ignored_http_statuses_and_urls_filename, "r") as ignored_http_statuses_and_urls: return [w.replace(" ", "").split(":") for w in ignored_http_statuses_and_urls.read().split("\n") if w] def add_pattern(sample_data): """ Determines ID for patterns by the following rules: - not a number - haven't 1 digit and 1 letter (exclude api versions) """ dynamic_parts = [] for row in sample_data: uri = row["uri"] parts = [ part for part in uri.split("/") if part ] dynamic_parts += [part for part in parts if check_on_id(part)] for subdir in SUBDIRECTORIES_FOR_PREDICTING_ID: if subdir in uri: index_subdir = parts.index(subdir) index_id = index_subdir + 1 if index_id < len(parts): dynamic_parts.append(parts[index_id]) dynamic_parts = set(dynamic_parts) for row in sample_data: if row.get("pattern"): continue parts = [] for part in row["uri"].split("/"): if part in dynamic_parts: part = "{id}" parts.append(part) pattern = "/".join(parts) row["pattern"] = pattern return sample_data def remove_prefix_duplicate(uri): """Temporary workaround.""" return uri.replace("/vendor-api/vendor-api", "/vendor-api") def check_on_id(item): if item.isdigit(): return True if sum(map(str.isdigit, item)) > 0 and sum(map(str.isalpha, item)) > 0 and not re.match(r"^v\d+$", item): return True return False class S3Resource: def __init__(self, bucket_name=None): self._session = boto3.Session() self._resource = self._session.resource("s3") self._bucket_name = bucket_name or s3_bucket_requests_extra_config def s3_file_data(self, filename): data = [] s3_obj = self._resource.Object(self._bucket_name, filename) try: obj_data = s3_obj.get() except ClientError as e: pass except self._resource.meta.client.exceptions.NoSuchBucket as e: pass else: reader = csv.DictReader(obj_data['Body'].read().decode('utf-8').splitlines()) for row in reader: data.append(row.copy()) return data def add_requests(data, rules): data.extend(rules) return data def replace_requests(data, rules): filtered_data = [] for item in data: replaced = False for rule in rules: if not replaced and rule["method"] == item["method"] and re.search(rule["regex_search_term"], item["uri"]): if rule["search_query_params"] and item["query_string"]: if set(rule["search_query_params"].split("&")) == set(item["query_string"].split("&")): item["query_string"] = rule["replacement_query_params"] else: continue item["uri"] = re.sub(rule["regex_search_term"], rule["replacement"], item["uri"]) replaced = True if replaced: filtered_data.append(item) return filtered_data def removed_requests(data, rules): filtered_data = [] for item in data: ignore = False for rule in rules: if not ignore and rule["method"] == item["method"] and re.search(rule["regex_search_term"], item["uri"]): if rule["search_query_params"] and item["query_string"]: if set(rule["search_query_params"].split("&")) != set(item["query_string"].split("&")): continue ignore = True if not ignore: filtered_data.append(item) return filtered_data def add_header_to_requests(data, rules): for item in data: for rule in rules: if rule["method"] == item["method"] and re.search(rule["regex_search_term"], item["uri"]): item["headers"] = json.loads(rule["header"]) return data def apply_extra_requests_rules(data): s3_resource = S3Resource() add_requests_rules = s3_resource.s3_file_data(filename=ADD_REQUESTS_FILENAME) replace_requests_rules = s3_resource.s3_file_data(filename=REPLACE_REQUESTS_FILENAME) removed_requests_rules = s3_resource.s3_file_data(filename=REMOVE_REQUESTS_FILENAME) header_requests_rules = s3_resource.s3_file_data(filename=HEADER_REQUESTS_FILENAME) if add_requests_rules: data = add_requests(data, add_requests_rules) if replace_requests_rules: data = replace_requests(data, replace_requests_rules) if removed_requests_rules: data = removed_requests(data, removed_requests_rules) if header_requests_rules: data = add_header_to_requests(data, header_requests_rules) return data def get_locust_tasks(): tasks = dict() data = read_sample_data() if s3_bucket_requests_extra_config: data = apply_extra_requests_rules(data) sample_data = add_pattern(data) for row in sample_data: uri = remove_prefix_duplicate(row["uri"]) # for graphql services, it is better to use an alias because all URLs look like /graphql if stats_alias:=row.get("stats_alias"): stats_name = f'{uri} ({stats_alias})' else: stats_name = remove_prefix_duplicate(row["pattern"]) add_testing_task( tasks, row["method"], uri, row["query_string"], row.get("body"), row.get("content_type"), row["count"], stats_name, row.get("headers"), ) return tasks