log4shell/tools/log4shell_softwarelist/softwarelist_parser.py at main · b-keller/log4shell · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
import csv as py_csv
import sys
import json as py_json
import pathlib
import tempfile

from typing import List

import click
import mistune
import requests
import unicodedata

from bs4 import BeautifulSoup
from bs4.element import Tag

HEADERS = [
    'Supplier',
    'Product',
    'Version',
    'Status',
    'Notes',
    'Links'
]

def download_softwarelist() -> str:
    """ Download softwarelist to parse

    :return: Markdown content of software list
    """
    url = "https://raw.githubusercontent.com/NCSC-NL/log4shell/main/software/README.md"
    response = requests.get(url)
    return response.content

def parse_links(links: List[Tag] = None) -> dict:
    """ Get all links info from Links column

    :return: Dictionary with `{link_text: link_href}`
    """
    return {link.text: link.get('href') for link in links}

def parse_record(record: List[Tag] = None) -> dict:
    """ Parse single tr record in Software list

    :return: Dictionary of parsed cell from record
    """

    result = dict()

    for index, header in enumerate(HEADERS):
        # Parse links differently
        if index == 5:
            if len(record) == 5:
                result[header] = {}
            else:
                result[header] = parse_links(record[index].find_all('a'))

        else:
            # Ensure unicode in text is properly parsed
            result[header] = unicodedata.normalize("NFKD", record[index].text)

    return result

@click.group()
@click.option('--path', default='../../software/README.md', help='Path to software list README.md', type=click.File())
@click.pass_context
def main(ctx, path):

    # Parse Markdown to HTML and get soup
    content = path.read()
    html = mistune.html(content)
    soup = BeautifulSoup(html, 'html.parser')

    # Look for all tr columns with td fields, after first h3 header
    records = list()
    first_h3 = soup.find('h3')

    for x in first_h3.next_elements:
        if isinstance(x, Tag) and x.name == 'tr':
            tds = x.find_all('td')

            if tds:  # ensure empty tr are ignored
                records.append(parse_record(tds))

    ctx.obj['records'] = records


@main.command()
@click.argument('output', default='-', type=click.File('w+'))
@click.pass_context
def json(ctx, output):
    py_json.dump(ctx.obj['records'], output)

@main.command()
@click.argument('output', default='-', type=click.File('w+'))
@click.pass_context
def csv(ctx, output):
    writer = py_csv.DictWriter(output, HEADERS)

    writer.writeheader()

    # cleanup links
    for record in ctx.obj['records']:
        record['Links'] = list(record['Links'].values())
        writer.writerow(record)


if __name__ == '__main__':
    main(obj={})