From 6a58abb19ab7d6ca50094e03f0046c649d0b1ae4 Mon Sep 17 00:00:00 2001 From: Daniel Milnes Date: Sat, 16 Nov 2019 22:23:56 +0000 Subject: [PATCH] Add CSV Support --- README.md | 8 +++--- hstsparser.py | 69 ++++++++++++++++++++++++++++++++++++++++----------- 2 files changed, 60 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index d015dc6..5e21539 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # HSTS Parser -[![Build Status](https://thebeanogamer.visualstudio.com/HSTSparser/_apis/build/status/HSTSparser?branchName=master)](https://thebeanogamer.visualstudio.com/HSTSparser/_build/latest?definitionId=2&branchName=master) ![Publish Status](https://thebeanogamer.vsrm.visualstudio.com/_apis/public/Release/badge/f24623e9-719d-4c7f-b194-3be7917a22bf/1/1) ![Licence](https://img.shields.io/github/license/thebeanogamer/hstsparser) ![Python 3.7.x](https://img.shields.io/badge/python-3.7.x-yellow.svg) +[![Build Status](https://thebeanogamer.visualstudio.com/HSTSparser/_apis/build/status/HSTSparser?branchName=master)](https://thebeanogamer.visualstudio.com/HSTSparser/_build/latest?definitionId=2&branchName=master) ![Publish Status](https://thebeanogamer.vsrm.visualstudio.com/_apis/public/Release/badge/f24623e9-719d-4c7f-b194-3be7917a22bf/1/1) ![Licence](https://img.shields.io/github/license/thebeanogamer/hstsparser) ![Python 3.7.x](https://img.shields.io/badge/python-3.7.x-yellow.svg) HSTS Parser is a simple tool to parse Firefox and Chrome's HSTS databases into actually helpful forensic artifacts! You can read more about the research behind this tool and potential uses for it over on [my blog](https://blog.daniel-milnes.uk/hsts-for-forensics-you-can-run-but-you-cant)! @@ -30,11 +30,13 @@ positional arguments: optional arguments: -h, --help show this help message and exit -w WORDLIST The path to the database to be processed + --csv CSV Output to a CSV file --firefox Process a Firefox database --chrome Process a Chrome database ``` ### Examples + #### Firefox ```shell @@ -57,11 +59,11 @@ python3 hstsparser.py -w wordlist.txt --chrome TransportSecurity ### Firefox -![](https://blog.daniel-milnes.uk/content/images/2019/11/image-3.png) +![Screenshot of Firefox Processing](https://blog.daniel-milnes.uk/content/images/2019/11/image-3.png) ### Chrome with Wordlist -![](https://blog.daniel-milnes.uk/content/images/2019/11/image-4.png) +![Screenshot of Chrome Processing with a wordlist](https://blog.daniel-milnes.uk/content/images/2019/11/image-4.png) ## Links diff --git a/hstsparser.py b/hstsparser.py index c52d536..074bdc0 100644 --- a/hstsparser.py +++ b/hstsparser.py @@ -1,3 +1,4 @@ +import csv import datetime import json import os @@ -9,7 +10,7 @@ from prettytable import PrettyTable -def convert(domain): +def convert_domain(domain): output = [chr(0)] idx = 0 for char in reversed(domain): @@ -23,7 +24,7 @@ def convert(domain): return b64encode(sha256("".join(reversed(output)).encode("utf-8")).digest()) -def printDB(database, field_names): +def print_db(database, field_names): table = PrettyTable() table.field_names = field_names for i in database: @@ -43,6 +44,35 @@ def is_valid_file(parser, arg): return open(arg, "r") +def file_already_exists(parser, arg): + if os.path.exists(arg): + parser.error(f"The file {arg} already exists!") + else: + return open(arg, "w", newline="") + + +def print_if_no_args(database, field_names): + if not args.csv_file: + print_db(database, field_names) + else: + file_write(database, field_names) + + +def file_write(database, field_names): + if args.csv_file: + with args.csv_file as csvfile: + csvfile = csv.writer(csvfile) + csvfile.writerow(field_names) + for i in database: + csvfile.writerow(i) + + +def date_round(date): + return date - datetime.timedelta( + minutes=date.minute % 10, seconds=date.second, microseconds=date.microsecond + ) + + parser = ArgumentParser(description="Process HSTS databases") parser.add_argument( dest="database_file", @@ -57,6 +87,13 @@ def is_valid_file(parser, arg): metavar="WORDLIST", type=lambda x: is_valid_file(parser, x), ) +parser.add_argument( + "--csv", + dest="csv_file", + help="Output to a CSV file", + metavar="CSV", + type=lambda x: file_already_exists(parser, x), +) group = parser.add_mutually_exclusive_group(required=True) group.add_argument("--firefox", action="store_true", help="Process a Firefox database") group.add_argument("--chrome", action="store_true", help="Process a Chrome database") @@ -80,8 +117,10 @@ def is_valid_file(parser, arg): record.append("Yes") else: record.append("No") + if args.csv_file: + record[3] = date_round(record[3]) database.append(record) - printDB( + print_if_no_args( database, ["URL", "Visits", "Last Accessed", "Expiry", "Type", "Include Subdomains"], ) @@ -95,26 +134,28 @@ def is_valid_file(parser, arg): subdomains = "Yes" else: subdomains = "No" - database.append( - [ - i, - datetime.datetime.fromtimestamp(current["expiry"]), - subdomains, - datetime.datetime.fromtimestamp(current["sts_observed"]), - ] - ) + record = [ + i, + datetime.datetime.fromtimestamp(current["expiry"]), + subdomains, + datetime.datetime.fromtimestamp(current["sts_observed"]), + ] + if args.csv_file: + record[1] = date_round(record[1]) + record[3] = date_round(record[3]) + database.append(record) if args.wordlist_file: wordlist = args.wordlist_file.read().splitlines() rainbow = [] for i in wordlist: - rainbow.append(convert(i)) + rainbow.append(convert_domain(i)) for i in database: for j in range(0, len(rainbow)): if i[0] == rainbow[j].decode("utf-8"): i.append(wordlist[j]) if len(i) == 4: i.append("") - printDB( + print_if_no_args( database, [ "Base64 URL Hash", @@ -125,7 +166,7 @@ def is_valid_file(parser, arg): ], ) else: - printDB( + print_if_no_args( database, ["Base64 URL Hash", "Expiry", "Include Subdomains", "Last Observed"], )