import git import csv from pprint import pprint from pathlib import Path from utility import name_subs basePath = 'C:/Users/Shanj0hn/github/filtr/' baseGitUrl = 'https://github.com/filtr/' sinceDate = '2020-04-01' beforeDate = '2021-04-01' repos = [ { 'repo': 'ama-website', 'project': 'RTI', 'branch': 'master'}, { 'repo': 'ama-performance-tests', 'project': 'RTI', 'branch': 'master'}, { 'repo': 'ama-backend', 'project': 'RTI', 'branch': 'develop'}, { 'repo': 'rti-amadeus', 'project': 'RTI', 'branch': 'develop'}, { 'repo': 'ama-frontend', 'project': 'RTI', 'branch': 'development'}, { 'repo': 'rti-user-preferences-service', 'project': 'RTI', 'branch': 'develop'}, { 'repo': 'rti-mobile-automation', 'project': 'RTI', 'branch': 'master'}, { 'repo': 'rti-scripts', 'project': 'RTI', 'branch': 'main'}, { 'repo': 'rti-notifications-worker', 'project': 'RTI', 'branch': 'develop'}, { 'repo': 'rti-delphi-data-validation', 'project': 'RTI', 'branch': 'master'}, { 'repo': 'core-images', 'project': 'Core/Atlas', 'branch': 'master'}, { 'repo': 'atlas-um', 'project': 'Core/Atlas', 'branch': 'develop'}, { 'repo': 'atlas-um-integration-example', 'project': 'Core/Atlas', 'branch': 'master'}, { 'repo': 'atlas-auth-proxy', 'project': 'Core/Atlas', 'branch': 'develop'}, #{ 'repo': 'atlas-ui-components', 'project': 'Core/Atlas', 'branch': 'master'}, { 'repo': 'mct-frontend', 'project': 'Decibel', 'branch': 'develop'}, { 'repo': 'marketing-data-entry', 'project': 'Decibel', 'branch': 'develop'}, { 'repo': 'decibel-pg2kinesis', 'project': 'Decibel', 'branch': 'master'}, { 'repo': 'frontend-shared-icons', 'project': 'Decibel', 'branch': 'master'}, { 'repo': 'frontend-shared-theme', 'project': 'Decibel', 'branch': 'master'}, # Apollo Go { 'repo': 'apollo-go', 'project': 'Apollo Go', 'branch': 'develop'}, { 'repo': 'apollo-go-mobile-automation', 'project': 'Apollo Go', 'branch': 'master'}, { 'repo': 'retrieve-and-save-mobile-versions-lambda', 'project': 'Apollo Go', 'branch': 'develop'}, { 'repo': 'send-push-messages-lambda', 'project': 'Apollo Go', 'branch': 'develop'}, { 'repo': 'send-push-notifications-lambda', 'project': 'Apollo Go', 'branch': 'master'}, # Apollo Portal { 'repo': 'filtr-admin', 'project': 'Apollo Portal', 'branch': 'develop'}, { 'repo': 'filtr-api', 'project': 'Apollo Portal', 'branch': 'develop'}, { 'repo': 'filtr-privateapi', 'project': 'Apollo Portal', 'branch': 'develop'}, { 'repo': 'filtr-admin-accounts', 'project': 'Apollo Portal', 'branch': 'master'}, { 'repo': 'apollo-db-migrations', 'project': 'Apollo Portal', 'branch': 'master'}, { 'repo': 'apollo-main-db-models-pkg', 'project': 'Apollo Portal', 'branch': 'master'}, { 'repo': 'apollo-new-music-friday-tracks-job', 'project': 'Apollo Portal', 'branch': 'develop'}, { 'repo': 'apollo-ui-test-automation', 'project': 'Apollo Portal', 'branch': 'master'}, { 'repo': 'apollo-clear-notifications-lambda', 'project': 'Apollo Portal', 'branch': 'master'}, { 'repo': 'apollo-gate-api', 'project': 'Apollo Portal', 'branch': 'master'}, { 'repo': 'apollo-api', 'project': 'Apollo Portal', 'branch': 'master'}, { 'repo': 'apollo-load-testing-framework', 'project': 'Apollo Portal', 'branch': 'master'}, { 'repo': 'apollo-notifications-pkg', 'project': 'Apollo Portal', 'branch': 'master'}, { 'repo': 'apollo-user-data', 'project': 'Apollo Portal', 'branch': 'develop'}, { 'repo': 'apollo-push-client-pkg', 'project': 'Apollo Portal', 'branch': 'master'}, { 'repo': 'apollo-auth-api', 'project': 'Apollo Portal', 'branch': 'master'}, { 'repo': 'apollo-personalized-playlists-update-job', 'project': 'Apollo Portal', 'branch': 'develop'}, { 'repo': 'apollo-auth0-unblock-bruteforce-users-lambda', 'project': 'Apollo Portal', 'branch': 'master'}, { 'repo': 'apollo-download-spotify-charts-lambda', 'project': 'Apollo Portal', 'branch': 'master'}, { 'repo': 'apollo-read-push-messages-job', 'project': 'Apollo Portal', 'branch': 'develop'}, { 'repo': 'apollo-read-push-messages-job', 'project': 'Apollo Portal', 'branch': 'develop'}, { 'repo': 'apollo-vendors-charts-availablity-job', 'project': 'Apollo Portal', 'branch': 'master'}, { 'repo': 'apollo-admin-panel', 'project': 'Apollo Portal', 'branch': 'master'}, { 'repo': 'apollo-api-healthcheck-lambda', 'project': 'Apollo Portal', 'branch': 'develop'}, { 'repo': 'apollo-save-push-receipts-lambda', 'project': 'Apollo Portal', 'branch': 'develop'}, { 'repo': 'go-apollo', 'project': 'Apollo Portal', 'branch': 'develop'}, { 'repo': 'apollo-traefik', 'project': 'Apollo Portal', 'branch': 'master'}, { 'repo': 'apollo-global-track-priorities-tracks-job', 'project': 'Apollo Portal', 'branch': 'develop'}, { 'repo': 'apollo-update-hot-hits-history-job', 'project': 'Apollo Portal', 'branch': 'master'}, { 'repo': 'apollo-update-track-playlist-history-job', 'project': 'Apollo Portal', 'branch': 'master'}, { 'repo': 'apollo-copy-track-playlist-history-last-week-job', 'project': 'Apollo Portal', 'branch': 'develop'}, { 'repo': 'apollo-update-streams-cache-job', 'project': 'Apollo Portal', 'branch': 'develop'}, { 'repo': 'apollo-flask-sqlalchemy-replication-pkg', 'project': 'Apollo Portal', 'branch': 'develop'}, { 'repo': 'apollo-dsp-smart-cache-job', 'project': 'Apollo Portal', 'branch': 'develop'}, { 'repo': 'vendor-api', 'project': 'Apollo Portal', 'branch': 'develop'}, { 'repo': 'update-market-vendors-lambda', 'project': 'Apollo Portal', 'branch': 'master'}, { 'repo': 'vendor-image-caching-lambda', 'project': 'Apollo Portal', 'branch': 'master'}, { 'repo': 'copy-auth0-logs-to-cloudwatch-lambda', 'project': 'Apollo Portal', 'branch': 'develop'}, { 'repo': 'dsp-api', 'project': 'Apollo Portal', 'branch': 'master'}, { 'repo': 'notifications-api', 'project': 'Apollo Portal', 'branch': 'master'}, { 'repo': 'spotify-charts-entries-notifications-job', 'project': 'Apollo Portal', 'branch': 'develop'}, { 'repo': 'spotify-charts-exits-notifications-job', 'project': 'Apollo Portal', 'branch': 'develop'}, { 'repo': 'spotify-charts-moves-notifications-job', 'project': 'Apollo Portal', 'branch': 'develop'}, { 'repo': 'spotify-playlists-additions-notifications-job', 'project': 'Apollo Portal', 'branch': 'develop'}, { 'repo': 'spotify-playlists-update-notifications-job', 'project': 'Apollo Portal', 'branch': 'develop'}, { 'repo': 'track-visit-events-handling-lambda', 'project': 'Apollo Portal', 'branch': 'master'}, { 'repo': 'ui-automation-framework-pkg', 'project': 'Apollo Portal', 'branch': 'master'}, { 'repo': 'update-market-vendors-lambda', 'project': 'Apollo Portal', 'branch': 'master'}, # Delphi { 'repo': 'delphi-ad-sources', 'project': 'Delphi', 'branch': 'develop'}, { 'repo': 'delphi-ad-sources-snowflake', 'project': 'Delphi', 'branch': 'release'}, { 'repo': 'delphi-ae-reporting-snowflake', 'project': 'Delphi', 'branch': 'develop'}, { 'repo': 'delphi-api', 'project': 'Delphi', 'branch': 'master'}, { 'repo': 'delphi-api-cache', 'project': 'Delphi', 'branch': 'master'}, { 'repo': 'delphi-api-elasticsearch', 'project': 'Delphi', 'branch': 'master'}, { 'repo': 'delphi-api-java', 'project': 'Delphi', 'branch': 'master'}, { 'repo': 'delphi-api-nginx', 'project': 'Delphi', 'branch': 'master'}, { 'repo': 'delphi-api-qa-automation', 'project': 'Delphi', 'branch': 'master'}, { 'repo': 'delphi-apple-music-charts', 'project': 'Delphi', 'branch': 'develop'}, { 'repo': 'delphi-application-enablement-sf-orchestration', 'project': 'Delphi', 'branch': 'release'}, { 'repo': 'delphi-application-enablement-snowflake', 'project': 'Delphi', 'branch': 'develop'}, { 'repo': 'delphi-bigtable-reader', 'project': 'Delphi', 'branch': 'master'}, { 'repo': 'delphi-chartmetric', 'project': 'Delphi', 'branch': 'master'}, { 'repo': 'delphi-chartmetric-db-schema', 'project': 'Delphi', 'branch': 'master'}, { 'repo': 'delphi-chartmetric-share', 'project': 'Delphi', 'branch': 'DSMA-24'}, { 'repo': 'delphi-charts-snowflake', 'project': 'Delphi', 'branch': 'develop'}, { 'repo': 'delphi-config', 'project': 'Delphi', 'branch': 'master'}, { 'repo': 'delphi-daily-data-alert', 'project': 'Delphi', 'branch': 'master'}, { 'repo': 'delphi-data-health', 'project': 'Delphi', 'branch': 'develop'}, { 'repo': 'delphi-data-health-configs', 'project': 'Delphi', 'branch': 'master'}, { 'repo': 'delphi-etl-apps', 'project': 'Delphi', 'branch': 'master'}, { 'repo': 'delphi-etl-apps-proto', 'project': 'Delphi', 'branch': 'master'}, { 'repo': 'delphi-etl-commons', 'project': 'Delphi', 'branch': 'master'}, { 'repo': 'delphi-etl-conversion', 'project': 'Delphi', 'branch': 'master'}, { 'repo': 'delphi-etl-mapping', 'project': 'Delphi', 'branch': 'develop'}, { 'repo': 'delphi-etl-orchestration', 'project': 'Delphi', 'branch': 'master'}, { 'repo': 'delphi-etl-qa-automation', 'project': 'Delphi', 'branch': 'dev'}, { 'repo': 'delphi-exploration-area', 'project': 'Delphi', 'branch': 'master'}, { 'repo': 'delphi-exploration-etl', 'project': 'Delphi', 'branch': 'develop'}, { 'repo': 'delphi-exploration-snowflake', 'project': 'Delphi', 'branch': 'master'}, { 'repo': 'delphi-gdpr', 'project': 'Delphi', 'branch': 'master'}, { 'repo': 'delphi-kong', 'project': 'Delphi', 'branch': 'master'}, { 'repo': 'delphi-lma', 'project': 'Delphi', 'branch': 'master'}, { 'repo': 'delphi-marketing-snowflake', 'project': 'Delphi', 'branch': 'release'}, { 'repo': 'delphi-master-catalog', 'project': 'Delphi', 'branch': 'develop'}, { 'repo': 'delphi-master-data', 'project': 'Delphi', 'branch': 'develop'}, { 'repo': 'delphi-notifications', 'project': 'Delphi', 'branch': 'master'}, { 'repo': 'delphi-public-data', 'project': 'Delphi', 'branch': 'master'}, { 'repo': 'delphi-qa-automation', 'project': 'Delphi', 'branch': 'master'}, { 'repo': 'delphi-simple-etl', 'project': 'Delphi', 'branch': 'develop'}, { 'repo': 'delphi-slz', 'project': 'Delphi', 'branch': 'master'}, { 'repo': 'delphi-slz-admin', 'project': 'Delphi', 'branch': 'release'}, { 'repo': 'delphi-slz-api', 'project': 'Delphi', 'branch': 'master'}, { 'repo': 'delphi-slz-db-schema', 'project': 'Delphi', 'branch': 'master'}, { 'repo': 'delphi-slz-qa-automation', 'project': 'Delphi', 'branch': 'dev'}, { 'repo': 'delphi-spark-poc', 'project': 'Delphi', 'branch': 'master'}, { 'repo': 'delphi-spotify-charts', 'project': 'Delphi', 'branch': 'master'}, { 'repo': 'delphi-tiktok-chartmetrics-sync', 'project': 'Delphi', 'branch': 'develop'}, { 'repo': 'delphi-util', 'project': 'Delphi', 'branch': 'release'}, { 'repo': 'docs', 'project': 'Delphi', 'branch': 'master'}, ] users = [] indexedResults = {} outputFile = f"{basePath}gitreports/gitstat.csv" for r in repos: gitFolder = f"{basePath}{r['repo']}" # Clone if not clonned if not Path(gitFolder).is_dir(): git.Repo.clone_from(f"{baseGitUrl}{r['repo']}.git", gitFolder, progress=None) # Pull latest changes g = git.Git(f"{basePath}{r['repo']}") g.checkout(r['branch']) g.pull() # Get user id list print(f"\nRepo: {r['repo']}, branch: {r['branch']}") """ userSummaries = g.shortlog(numbered=True, summary=True, no_merges=True, since=sinceDate, before=beforeDate) if userSummaries != '': for line in userSummaries.split('\n'): line = line.strip() user = line.split('\t')[1].replace("'","") #user = name_subs(user) users.append(user) """ gitlog = g.log(numstat=True, no_merges=True, no_color=True, since=sinceDate, before=beforeDate, pretty=f"format:'%an'").split("\n\n")[1:] for record in gitlog: lines = record.split('\n') author = lines[0].replace("'","") author = name_subs(author) for line in lines[1:]: values = line.split('\t') if len(values) == 3 : # composite index of author and filename index = r['repo'] + author + values[2] if values[0] == '-': indexedResults[index] = { 'Committer': author, 'Filename': values[2], 'Lines_added': 'binary', 'Lines_removed': 'binary', 'Group': r['project'], 'Project': r['repo'], } else: indexedResults[index] = { 'Committer': author, 'Filename': values[2], } if 'Lines_added' in indexedResults[index]: indexedResults[index]['Lines_added'] += int(values[0]) else: indexedResults[index]['Lines_added'] = int(values[0]) if 'Lines_removed' in indexedResults[index]: indexedResults[index]['Lines_removed'] += int(values[1]) else: indexedResults[index]['Lines_removed'] = int(values[1]) indexedResults[index]['Group'] = r['project'] indexedResults[index]['Project'] = r['repo'] else: print("No info, skipping repo") result = list(indexedResults.values()) keys = result[0].keys() with open(outputFile, 'w', newline='') as of: dict_writer = csv.DictWriter(of, keys) dict_writer.writeheader() dict_writer.writerows(result)