import csv import os import re import requests from dotenv import load_dotenv # Load environment variables from .env file load_dotenv() def infer_python_php_version(version_number): # Check if the version number follows Python's versioning scheme if version_number.isdigit(): major = version_number[0] minor = version_number[1:] return f"{major}.{minor}" else: # Invalid version format, return as is return version_number def get_ows_image_version(repo_name, token): # Fetch the content of the Dockerfile url = f"https://raw.githubusercontent.com/{repo_name}/master/Dockerfile" headers = { "Authorization": f"token {token}" } response = requests.get(url, headers=headers) if response.status_code != 200: print(f"Failed to fetch Dockerfile for {repo_name}: {response.status_code}") return None, None # Parse the Dockerfile content to find the docker image language and version dockerfile_content = response.text if not dockerfile_content: print(f"Dockerfile not found in {repo_name}") return None, None # Pattern to match language and version lang_vers_pattern = r'php\d{1,2}|node\d{1,2}|python\d{1,2}\.\d{1,2}|python\d{1,3}|python:\d{1,2}\.\d{1,2}' language_pattern = r'php|node|python' version_pattern = r'\d+\.\d+|\d+' lang_vers_match = re.search(lang_vers_pattern, dockerfile_content) lang_vers = lang_vers_match.group() if lang_vers_match else None if not lang_vers: print(f"No language/version found in {repo_name}") return None, None # Find version version_match = re.search(version_pattern, lang_vers) version = version_match.group() if version_match else None # Find language language_match = re.search(language_pattern, lang_vers, re.IGNORECASE) language = language_match.group() if language_match else None if language in ('python', 'php'): version = infer_python_php_version(version) return language, version def get_ows_app_family(repo_name, token): # Fetch the content of the Dockerfile url = f"https://raw.githubusercontent.com/theorchard/terraform-infra/master/prod/{repo_name}/variables.tf" headers = { "Authorization": f"token {token}" } response = requests.get(url, headers=headers) if response.status_code != 200: print(f"Failed to fetch variables.tf for {repo_name}: {response.status_code}") return None # Parse the Dockerfile content to find the Python image version variables_content = response.text if not variables_content: print(f"app family not found for {repo_name}") return None # Define a regular expression pattern to match the default attribute value pattern = r'variable\s+"?application_family"?\s+{\s+default\s+=\s+"([^"\n]+)"' # Search for the pattern in the Terraform file content match = re.search(pattern, variables_content, re.DOTALL) if match: return match.group(1) print(f"Failed to find application_family for {repo_name}.") return None def fetch_repos(organization, token, page=1, per_page=100): url = f"https://api.github.com/orgs/{organization}/repos" headers = { "Authorization": f"token {token}" } params = { "per_page": per_page, "page": page } response = requests.get(url, headers=headers, params=params) if response.status_code != 200: print("Failed to fetch repositories") return [] repos = response.json() return [repo for repo in repos if not repo['archived']] def main(): organization = "theorchard" prefix = {'ows-': [], 'lambda-': [], 'graphql-': []} token = os.getenv("GITHUB_TOKEN") repos = [] page = 1 while True: repo_batch = fetch_repos(organization, token, page=page) if not repo_batch: break for repo in repo_batch: if repo['name'].startswith('lambda-'): prefix['lambda-'].append(repo) if repo['name'].startswith('ows-'): prefix['ows-'].append(repo) page += 1 with open('repository_info.csv', mode='w', newline='') as file: writer = csv.writer(file) writer.writerow(['repo_name', 'repo_link', 'app_family', 'language', 'version']) # Iterate over ows repositories for repo in prefix['ows-']: repo_name = repo['full_name'] repo_link = repo['html_url'] language, version = get_ows_image_version(repo_name, token) app_family = get_ows_app_family(repo['name'], token) writer.writerow([repo['name'], repo_link, app_family, language, version]) if __name__ == "__main__": main()