References

A humble, and fast, security-oriented HTTP headers analyzer.

ExportStates

Bases: NamedTuple

Formatting states for HTML/PDF exports, related to -o all option.

Source code in humble.py

class ExportStates(NamedTuple):
    """Formatting states for HTML/PDF exports, related to `-o all` option."""

    response: bool
    enabled: bool
    browser: bool

SSLContextAdapter

Bases: HTTPAdapter

Custom SSL adapter.

Disables SSL validation for unrestricted URL analysis.

Note

The following checks are disabled to allow the analysis of URLs in environments with self-signed certificates, outdated software, or development configurations:

• Certificate Verification
• Hostname Verification
• Certificate Requirement

Source code in humble.py

class SSLContextAdapter(requests.adapters.HTTPAdapter):
    """Custom SSL adapter.

    Disables SSL validation for unrestricted URL analysis.

    ??? note
        The following checks are disabled to allow the analysis of URLs
        in environments with self-signed certificates, outdated software,
        or development configurations:

        - Certificate Verification
        - Hostname Verification
        - Certificate Requirement
    """

    def init_poolmanager(self, *args, **kwargs):
        """Initialize the pool manager.

        With an unverified SSL context and restricted ciphers.

        """
        # nosemgrep: unverified-ssl-context
        context = ssl._create_unverified_context()
        context.check_hostname = False
        context.verify_mode = ssl.CERT_NONE
        context.set_ciphers(FORCED_CIPHERS) # nosemgrep: no-set-ciphers
        kwargs["ssl_context"] = context
        return super().init_poolmanager(*args, **kwargs)

init_poolmanager(*args, **kwargs)

Initialize the pool manager.

With an unverified SSL context and restricted ciphers.

Source code in humble.py

def init_poolmanager(self, *args, **kwargs):
    """Initialize the pool manager.

    With an unverified SSL context and restricted ciphers.

    """
    # nosemgrep: unverified-ssl-context
    context = ssl._create_unverified_context()
    context.check_hostname = False
    context.verify_mode = ssl.CERT_NONE
    context.set_ciphers(FORCED_CIPHERS) # nosemgrep: no-set-ciphers
    kwargs["ssl_context"] = context
    return super().init_poolmanager(*args, **kwargs)

add_xml_item(line, section)

Add a new item to the section.

Related to -o xml option.

Source code in humble.py

def add_xml_item(line, section):
    """Add a new item to the section.

    Related to `-o xml` option.
    """
    item = ET.SubElement(section, "item")
    if ": " in line and all(sub not in line for sub in XML_STRING):
        key, value = line.split(": ", 1)
        item.set("name", key.strip())
        item.text = value.strip()
    else:
        item.text = line

adjust_old_analysis(url_ln)

Adjust analysis entries in analysis history file, analysis_h.txt.

Note

Applied to those made before 2024-11-28 when the enabled security headers total was not yet recorded; ensures that old entries in that file remain compatible with the current analysis format.

Source code in humble.py

def adjust_old_analysis(url_ln):
    """Adjust analysis entries in analysis history file, `analysis_h.txt`.

    ??? note
        Applied to those made before 2024-11-28 when the
        *enabled security headers* total was not yet recorded; ensures that old
        entries in that file remain compatible with the current analysis
        format.
    """
    updated_lines = []
    for i in url_ln:
        fields = i.strip().split(";")
        if len(fields) == LENGTH_BOUNDS[1]:
            fields = [field.strip() for field in fields]
            fields.insert(2, "0")
            updated_lines.append(" ; ".join(fields) + "\n")
        else:
            updated_lines.append(i)
    return updated_lines

analyze_input_file(input_file)

Analyze HTTP response headers from a raw text dump or a HAR file.

See curl's --dump-header and the HAR specification or its historical draft.

Related to -if option.

Source code in humble.py

def analyze_input_file(input_file):
    """Analyze HTTP response headers from a raw text dump or a HAR file.

    See curl's [`--dump-header`](https://curl.se/docs/manpage.html#-D) and
    the [`HAR specification`](https://w3c.github.io/web-performance/specs/HAR/Overview.html)
    or its historical [`draft`](http://www.softwareishard.com/blog/har-12-spec/).

    Related to `-if` option.
    """
    file_path = Path(input_file)
    if not file_path.exists():
        print_error_detail("[args_inputnotfound]")
        return {}, False, 0
    input_headers = {}
    status_code = 0
    try:
        with file_path.open(encoding="utf8") as f:
            first_char = f.read(1).strip()
        if first_char == "{":
            input_headers, status_code = parse_har_file(file_path)
            if not input_headers:
                print_error_detail("[args_harlines]")
            return input_headers, False, status_code
        with file_path.open(encoding="utf8") as input_source:
            input_headers, status_code = parse_input_file(
                input_headers, input_source, status_code,
            )
    except UnicodeDecodeError:
        print_error_detail("[args_inputunicode]")
    return input_headers, False, status_code

apply_pdf_color(colon_idx, hcolor, line, vcolor)

Add the specific HTML tag to indicate the corresponding color.

Sanitizes header values using HTML escaping to ensure that special characters (e.g., < or > in headers) are rendered correctly and do not interfere with the PDF structure.

Related to -o pdf option.

Source code in humble.py

def apply_pdf_color(colon_idx, hcolor, line, vcolor):
    """Add the specific HTML tag to indicate the corresponding color.

    Sanitizes header values using HTML escaping to ensure that special
    characters (e.g., `<` or `>` in headers) are rendered correctly and do not
    interfere with the PDF structure.

    Related to `-o pdf` option.
    """
    if colon_idx == -1:
        return f'{HTML_TAGS[16]}{hcolor}">{escape(line)}{HTML_TAGS[17]}'
    header_part = escape(line[:colon_idx + 2])
    value_part = escape(line[colon_idx + 2:])
    return (
        f'{HTML_TAGS[16]}{hcolor}">{header_part}{HTML_TAGS[18]}'
        f'{HTML_TAGS[19]}{vcolor}">{value_part}{HTML_TAGS[17]}'
    )

build_cicd_info(info_lines)

Build the CI/CD info dict from analysis metadata lines.

Related to -cicd option.

Source code in humble.py

def build_cicd_info(info_lines):
    """Build the CI/CD info dict from analysis metadata lines.

    Related to `-cicd` option.
    """
    file_label = get_detail("[cicd_file]", replace=True)
    return {
        key.strip(): value.strip()
        for line in info_lines
        if ":" in line
        for key, value in [line.split(":", 1)]
        if key.strip() != file_label
    }

build_cicd_totals(tmp_filename, info_lines, totals, labels, threshold=None)

Build the CI/CD totals from analysis metadata and section totals.

If threshold is provided, appends a 'Security Gate' field with pass/fail status; related to -cicd GRADE option.

Source code in humble.py

def build_cicd_totals(tmp_filename, info_lines, totals, labels, threshold=None):
    """Build the CI/CD totals from analysis metadata and section totals.

    If `threshold` is provided, appends a 'Security Gate' field with pass/fail
    status; related to `-cicd GRADE` option.
    """
    _, _, info_label = labels
    return {
        info_label: build_cicd_info(info_lines),
        **totals,
        get_detail("[cicd_detailed]", replace=True): {
            get_detail("[cicd_path]",
                       replace=True): str(Path(tmp_filename).resolve()),
        },
        **(
            {get_detail("[cicd_gate]", replace=True): {
                get_detail("[cicd_gate_grade]", replace=True): threshold,
            }}
            if threshold is not None else {}
        ),
    }

build_html_writers()

Build the ordered rules that format the main sections.

Each writer takes (html_final, ln_rstrip), writes the line if it matches and returns whether it did; the values they depend on are resolved once here, instead of once per line.

Related to -o html option.

Source code in humble.py

def build_html_writers():
    """Build the ordered rules that format the main sections.

    Each writer takes `(html_final, ln_rstrip)`, writes the line if it
    matches and returns whether it did; the values they depend on are
    resolved once here, instead of once per line.

    Related to `-o html` option.
    """
    ok_string = get_detail(DIR_MSG[2]).rstrip()
    ko_strings = [get_detail(f"[{i}]").rstrip() for i in ("no_sec_headers",
                                                          "no_enb_headers")]
    lang_slice = SLICE_INT[6] if args.lang else SLICE_INT[7]
    html_writers = (
        partial(format_html_warnings, ko_strings=ko_strings,
                ok_string=ok_string),
        partial(format_html_references, lang_slice=lang_slice),
        format_html_compatibility,
    )
    html_rest = partial(format_html_rest, l_empty=l_empty,
                        l_total=sorted(set(l_miss + l_ins)),
                        fng_sorted=sorted(l_fng),
                        header_prefixes=tuple((header, f"{header}: ") for
                                              header in headers))
    return html_writers, html_rest

build_tmp_file(export_date, file_ext, lang, humble_str, url)

Build the default filename for the temporary export file.

Formats a unique name from the URL scheme, domain, port, and timestamp; related to -o option.

Tip

tldextract is lazy-loaded to avoid unnecessary overhead when the analysis is not exported.

Source code in humble.py

def build_tmp_file(export_date, file_ext, lang, humble_str, url):
    """Build the default filename for the temporary export file.

    Formats a unique name from the URL scheme, domain, port, and timestamp;
    related to `-o` option.

    ??? tip
        `tldextract` is lazy-loaded to avoid unnecessary overhead when the
        analysis is not exported.
    """
    import tldextract
    url_str = tldextract.extract(URL)
    url_sub = f"_{url_str.subdomain}." if url_str.subdomain else "_"
    url_prt = f"_{url.port}_" if url.port else "_"
    return (
        f"{humble_str}_{url.scheme}"
        f"{url_sub}{url_str.domain}.{url_str.suffix}"
        f"{url_prt}{export_date}{lang}{file_ext}"
    )

calculate_highlights(url_ln, field_index, func)

Extract the specific date from an analysis.

Based on the required highlight metric; related to -a option.

Source code in humble.py

def calculate_highlights(url_ln, field_index, func):
    """Extract the specific date from an analysis.

    Based on the required highlight metric; related to `-a` option.
    """
    values = [int(line.split(";")[field_index].strip()) for line in url_ln]
    target_value = func(values)
    target_line = next(line for line in url_ln
                       if int(line.split(";")[field_index].strip()) ==
                       target_value)
    return target_line.split(";")[0].strip()

calculate_trends(values)

Compute trends based on several analyses of the same URL.

Info

Trends are related to the totals of missing headers, fingerprints, deprecated/insecure, empty, and total warnings for a given URL: the analysis history file, analysis_h.txt, must contain at least five analyses of the same URL in order to calculate reliable trends, and only the five most recent analyses of that URL are taken into account.

Trend values:

• Stable: all five totals are identical
• Improving: totals consistently decrease
• Worsening: totals consistently increase
• Fluctuating: No clear trend is detected; totals alternate

Source code in humble.py

def calculate_trends(values):
    """Compute trends based on several analyses of the same URL.

    ??? info
        Trends are related to the totals of missing headers, fingerprints,
        deprecated/insecure, empty, and total warnings for a given URL: the
        analysis history file, analysis_h.txt, must contain at least **five**
        analyses of the same URL in order to calculate reliable trends, and
        **only** the five most recent analyses of that URL are taken into
        account.<br>
        <br>
        Trend values:<br>

        - `Stable`: all five totals are identical
        - `Improving`: totals consistently decrease
        - `Worsening`: totals consistently increase
        - `Fluctuating`: No clear trend is detected; totals alternate
    """
    if len(values) < LENGTH_BOUNDS[0]:
        return print_detail_l("[t_insufficient]", analytics=True)
    trends_list = values[-5:]
    if all(x == trends_list[0] for x in trends_list):
        return print_detail_l("[t_stable]", analytics=True)
    inc_trend = sum(trends_list[i] > trends_list[i - 1] for i in range(1, 5))
    dec_trend = sum(trends_list[i] < trends_list[i - 1] for i in range(1, 5))
    if dec_trend > inc_trend:
        return print_detail_l("[t_improving]", analytics=True)
    if inc_trend > dec_trend:
        return print_detail_l("[t_worsening]", analytics=True)
    return print_detail_l("[t_fluctuating]", analytics=True)

check_analysis(filepath)

Check if analysis history file, analysis_h.txt, exists.

Source code in humble.py

def check_analysis(filepath):
    """Check if analysis history file, `analysis_h.txt`, exists."""
    if not Path(filepath).exists():
        detail = "[no_analysis]" if URL else "[no_global_analysis]"
        print_error_detail(detail)

check_cicd(analysis_grade, threshold_grade)

Check if the analysis grade fails the minimum required for CI/CD.

Both grades must belong to GRADE_ORDER, which is guaranteed by validate_cicd_grade; equal grades pass.

Source code in humble.py

def check_cicd(analysis_grade, threshold_grade):
    """Check if the analysis grade fails the minimum required for CI/CD.

    Both grades must belong to `GRADE_ORDER`, which is guaranteed by
    `validate_cicd_grade`; equal grades pass.
    """
    return GRADE_ORDER.index(analysis_grade) < GRADE_ORDER.index(
        threshold_grade)

check_export_scope()

Determine the scope of the export process based on the -o option.

If the value of that option is not all (e.g., csv), it proceeds to export the analysis in that format; otherwise, it exports it to all supported formats and terminate execution.

Source code in humble.py

def check_export_scope():
    """Determine the scope of the export process based on the `-o` option.

    If the value of that option is not `all` (e.g., `csv`), it proceeds to
    export the analysis in that format; otherwise, it exports it to all
    supported formats and terminate execution.
    """
    if args.output != "all":
        check_output_format(final_filename, reliable, tmp_filename)
    else:
        export_all_formats(final_filename, tmp_filename)

check_frame_options(args, headers_l, l_miss, m_cnt, skip_headers)

Determine whether to report a missing X-Frame-Options header.

Info

X-Frame-Options won't be reported as missing if the frame-ancestors directive of Content-Security-Policy is present: the latter is the comprehensive, modern, and preferred way.

Source code in humble.py

def check_frame_options(args, headers_l, l_miss, m_cnt, skip_headers):
    """Determine whether to report a missing `X-Frame-Options` header.

    ??? info
        `X-Frame-Options` won't be reported as missing if the
        `frame-ancestors` directive of `Content-Security-Policy` is
        present: the latter is the [comprehensive](https://developer.
        mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/
        X-Frame-Options){:target="_blank"}, [modern](https://developer.mozilla.org/en-US/
        docs/Web/HTTP/Reference/Headers/Content-Security-Policy/
        frame-ancestors){:target="_blank"}, and preferred way.
    """
    xfo_needed = ("x-frame-options" not in skip_headers) and \
        ("x-frame-options" not in headers_l)
    fa_needed = "frame-ancestors" not in \
        headers_l.get("content-security-policy", "")
    if xfo_needed and fa_needed:
        l_miss.append("X-Frame-Options")
        m_cnt += 1
        print_header("X-Frame-Options")
        if not args.brief:
            print_detail("[mxfo]", 2)
    return m_cnt

check_input_traversal(user_input)

Check user input for path traversal patterns.

Exit is one is found; related to -of and -op options.

Source code in humble.py

def check_input_traversal(user_input):
    """Check user input for path traversal patterns.

    Exit is one is found; related to `-of` and `-op` options.
    """
    input_traversal_ptrn = re.compile(RE_PATTERN[2])
    if input_traversal_ptrn.search(user_input):
        print(f"\n{get_detail('[args_input_traversal]', replace=True)}\
: ('{user_input}')")
        sys.exit(1)

check_missing_headers(m_cnt, l_miss, l_detail, merged_set, xfo_skipped)

Print the missing security-related HTTP response headers.

Based on those I consider essential.

Note

strict=False is used because l_detail contains [mxfo] but l_miss does not yet contain X-Frame-Options: this header is checked for in check_frame_options function.

Note

The highlighted headers are defined in the EXP_HEADERS tuple and correspond to some of those indicated in the MDN list of HTTP headers.

Source code in humble.py

def check_missing_headers(m_cnt, l_miss, l_detail, merged_set, xfo_skipped):
    """Print the missing security-related HTTP response headers.

    Based on those I consider essential.

    ??? note
        `strict=False` is used because `l_detail` contains `[mxfo]` but `l_miss`
        does not yet contain `X-Frame-Options`: this header is checked for in
        `check_frame_options` function.

    ??? note
        The highlighted [headers](https://developer.mozilla.org/en-US/docs/
        MDN/Writing_guidelines/Experimental_deprecated_obsolete){:target="_blank"}
        are defined in the `EXP_HEADERS` tuple and correspond to some of those
        indicated in the MDN [list](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers){:target="_blank"}
        of HTTP headers.
    """
    for header, detail in zip(l_miss, l_detail, strict=False):
        lower_header = header.lower()
        if lower_header not in merged_set and not xfo_skipped:
            print_header(
                f"{get_detail('[exp_header]', replace=True)}{header}"
                if lower_header in EXP_HEADERS else header)
            if not args.brief:
                print_detail(detail, 2)
            m_cnt += 1
    return m_cnt

check_output_format(final_filename, reliable, tmp_filename)

Dispatch the export logic for the selected output format.

Maps each supported format (text, CSV, JSON, XLSX, XML, HTML and PDF) to its corresponding function. For text output, handles CI/CD totals and OWASP compliance checks; for JSON, toggles between brief and detailed reports.

Related to -o option.

Source code in humble.py

def check_output_format(final_filename, reliable, tmp_filename):
    """Dispatch the export logic for the selected output format.

    Maps each supported format (text, CSV, JSON, XLSX, XML, HTML and PDF) to its
    corresponding function. For text output, handles CI/CD totals and OWASP
    compliance checks; for JSON, toggles between brief and detailed reports.

    Related to `-o` option.
    """
    match args.output:
        case "txt":
            if args.cicd:
                print_cicd_totals(tmp_filename, args.cicd if
                                  isinstance(args.cicd, str) else None)
            print_export_path(tmp_filename, reliable)
            if "-c" in sys.argv:
                check_owasp_compliance(tmp_filename)
        case "csv":
            generate_csv(final_filename, tmp_filename)
        case "json":
            (generate_json(final_filename, tmp_filename) if args.brief else
             generate_json_detailed(final_filename, tmp_filename))
        case "xlsx":
            generate_csv(final_filename, tmp_filename, to_xlsx=True)
        case "xml":
            generate_xml(final_filename, tmp_filename)
        case "html":
            export_html_file(final_filename, tmp_filename)
        case "pdf":
            export_pdf_file(tmp_filename)

check_output_path(args)

Validate the provided path when exporting an analysis.

Exit in case of error; related to -op option.

Source code in humble.py

def check_output_path(args):
    """Validate the provided path when exporting an analysis.

    Exit in case of error; related to `-op` option.
    """
    check_input_traversal(args.output_path)
    output_path = Path(args.output_path).resolve()
    if args.output is None:
        print_error_detail("[args_nooutputfmt]")
    elif output_path.exists():
        validate_path(output_path)
    else:
        msg = get_detail("[args_noexportpath]", replace=True)
        print(f"\n {msg} ('{output_path}')")
        sys.exit(1)

check_owasp_compliance(tmp_filename)

OWASP Secure Headers Project best practices checks.

Related to -c option.

Source code in humble.py

def check_owasp_compliance(tmp_filename):
    """`OWASP Secure Headers Project` best practices checks.

    Related to `-c` option.
    """
    Path(tmp_filename).unlink()
    header_list = []
    header_dict = {}
    with PATHS["owasp_compliance"].open("r", encoding="utf8") as owasp_file:
        for line in islice(owasp_file, SLICE_INT[8], None):
            header_name, header_val = line.split(": ", 1)
            header_list.append(header_name.lower())
            header_dict[header_name] = header_val.rstrip()
    print_owasp_findings(header_dict, header_list)

check_proxy_url(proxy_host, proxy_port, timeout, failed_proxy)

Check if the proxy server is reachable, related to -p option.

Source code in humble.py

def check_proxy_url(proxy_host, proxy_port, timeout, failed_proxy):
    """Check if the proxy server is reachable, related to `-p` option."""
    try:
        with create_connection((proxy_host, proxy_port), timeout=timeout):
            pass
    except OSError:
        failed_proxy.set()

check_python_version()

Verify that the host's Python version meets the minimum requirements.

Prints an error and exit if the version is below the threshold defined in PYTHON_REQUIRED_VERSION.

Source code in humble.py

def check_python_version():
    """Verify that the host's Python version meets the minimum requirements.

    Prints an error and exit if the version is below the threshold defined in
    `PYTHON_REQUIRED_VERSION`.
    """
    if sys.version_info < PYTHON_REQUIRED_VERSION:
        host_python = f"{sys.version_info.major}.{sys.version_info.minor}"
        host_msg = get_detail("[python_host_version]", replace=True)
        print(f"\n{host_msg} {host_python}.")
        print_detail("[python_required_version]", 3)
        sys.exit(1)

check_russian_scope()

Validate if the target domain is within the Russian scope and exit if so.

Note

You can read my reasons here.

Source code in humble.py

def check_russian_scope():
    """Validate if the target domain is within the Russian scope and exit if so.

    ??? note
        You can read my reasons [here](https://github.com/rfc-st/humble/blob/master/CODE_OF_CONDUCT.md#update-20220326){:target="_blank"}.
    """
    domain = urlparse(URL).netloc.split(":")[0]
    try:
        sff = domain.encode("ascii").decode("idna")
    except UnicodeError:
        sff = domain
    if sff.split(".")[-1].upper() in {"RU", "РФ"}:
        print_detail("[ru_check]", 3)
        sys.exit(1)

check_skip_file()

Exclude the headers defined in the humble.skip file from the analysis.

Returns a list of clean header name strings.

Source code in humble.py

def check_skip_file():
    """Exclude the headers defined in the `humble.skip` file from the analysis.

    Returns a list of clean header name strings.
    """
    file_skipped = []
    skip_file = Path("humble.skip")
    if skip_file.exists():
        with suppress(FileNotFoundError, PermissionError), \
             skip_file.open("r", encoding="utf-8") as humble_skip_file:
            file_skipped = [
                line.strip() for line in humble_skip_file
                if line.strip() and not line.strip().startswith("#")
            ]
    return file_skipped

check_unsafe_cookies()

Set-Cookie header analysis.

Source code in humble.py

def check_unsafe_cookies():  # sourcery skip: use-named-expression
    """`Set-Cookie` header analysis."""
    unsafe_cks = [ck.split("=", 1)[0].strip() for ck in
                  re.split(RE_PATTERN[14], stc_header) if
                  any(val not in ck.lower() for val in t_cookie_sec)]
    if unsafe_cks:
        print_detail_r("[iset_h]", is_red=True)
        if not args.brief:
            print_unsafe_cookies(unsafe_cks)
        i_cnt[0] += 1

check_updates(local_version)

Check for updated versions of humble on GitHub.

After that exit; related to -v option.

Source code in humble.py

def check_updates(local_version):
    """Check for updated versions of `humble` on GitHub.

    After that exit; related to `-v` option.
    """
    try:
        github_response = requests.get(URL_LIST[3], timeout=REQ_TIMEOUT)
        github_response.raise_for_status()
        github_repo = github_response.text
        github_date = re.search(RE_PATTERN[4], github_repo).group()
        github_version = date.fromisoformat(github_date)
        days_diff = (github_version - local_version).days
        check_updates_diff(days_diff, github_version, local_version)
    except (requests.exceptions.RequestException, AttributeError, ValueError):
        print_error_detail("[update_error]")
    sys.exit(0)

check_updates_diff(days_diff, github_version, local_version)

humble update logic.

Check whether the local version is more than a month older than the latest on GitHub, related to -v option.

Source code in humble.py

def check_updates_diff(days_diff, github_version, local_version):
    """`humble` update logic.

    Check whether the local version is more than a month older than the latest
    on GitHub, related to `-v` option.
    """
    print(f" \n{STYLE[0]}{get_detail('[humble_latest]', replace=True)} \
{github_version} \n {get_detail('[humble_local]', replace=True)} \
{local_version}{STYLE[4]}")
    if days_diff > DAYS_DIFF:
        print(f"\n{get_detail('[humble_not_recent]')}\n\
{get_detail('[github_humble]', replace=True)}\n")
    else:
        print_detail("[humble_recent]", 8)

choose_xlsx_format(bold_fmt, cell_fmt, cell_value, col_index, hidden_fmt, row_index, prev_section)

Choose the cell format for an XLSX export based on position and content.

Tracks section changes across rows to apply bold, hidden, or standard formatting; related to -o xlsx option.

Source code in humble.py

def choose_xlsx_format(bold_fmt, cell_fmt, cell_value, col_index, hidden_fmt,
                       row_index, prev_section):
    """Choose the cell format for an XLSX export based on position and content.

    Tracks section changes across rows to apply bold, hidden, or standard
    formatting; related to `-o xlsx` option.
    """
    if row_index == 0 and col_index in (0, 1):
        return bold_fmt, prev_section
    if col_index == 0 and row_index > 0:
        return (hidden_fmt, prev_section) if cell_value == prev_section \
            else (cell_fmt, cell_value)
    return cell_fmt, prev_section

clean_html_final(final_filename)

Remove content related to preformatted text.

Related to -o html option.

Source code in humble.py

def clean_html_final(final_filename):
    """Remove content related to preformatted text.

    Related to `-o html` option.
    """
    html_path = Path(final_filename)
    html_content = re.sub(RE_PATTERN[22], "", html_path.read_text(
        encoding="utf8")).replace(HTML_PRE_ARTIFACT, "")
    html_path.write_text(html_content, encoding="utf8")

color_pdf_line(line, hcolor, vcolor, chunks, i, pdf)

Locate lines to which a specific color should be applied.

Relate to -o pdf option.

Source code in humble.py

def color_pdf_line(line, hcolor, vcolor, chunks, i, pdf):
    """Locate lines to which a specific color should be applied.

    Relate to `-o pdf` option.
    """
    colon_idx = line.find(": ")
    ln_final = apply_pdf_color(colon_idx, hcolor, line, vcolor)
    pdf.write_html(ln_final)
    condition = chunks and len(chunks) == LENGTH_BOUNDS[5] and i == 0
    return hcolor if condition else None

compare_analysis_results(analysis_totals, current_counts)

Print the differences in totals between analyses of the same URL.

Source code in humble.py

def compare_analysis_results(analysis_totals, current_counts):
    """Print the differences in totals between analyses of the same URL."""
    status_map = {"First": "[first_analysis]",
                  "Not available": "[notaval_analysis]"}
    if analysis_totals[0] in status_map:
        return [get_detail(status_map[analysis_totals[0]], replace=True)] * 6
    return [
        get_detail("[no_changes]", replace=True) if (d - int(c)) == 0
        else f"{d - int(c):+d}" for d, c in zip(current_counts,
                                                analysis_totals, strict=True)
    ]

csp_analyze_content(csp_header)

Content-Security-Policy header analysis.

Source code in humble.py

def csp_analyze_content(csp_header):
    """`Content-Security-Policy` header analysis."""
    csp_deprecated = set()
    csp_dirs_vals = [directive.strip() for directive in csp_header.split(";") if
                     directive.strip()]
    csp_dirs = {directive.split()[0] for directive in csp_dirs_vals}
    for csp_dir in csp_dirs_vals:
        csp_deprecated |= ({value for value in t_csp_dep if value in csp_dir})
    if csp_deprecated:
        csp_print_deprecated(csp_deprecated)
    if "'strict-dynamic'" in csp_header:
        csp_check_ignored(csp_header)
    csp_check_missing(csp_dirs)
    csp_check_additional(csp_dirs_vals)

csp_base64_nonce(nonce, nonce_refs, i_cnt)

Content-Security-Policy header checks related to Base64 nonces.

Source code in humble.py

def csp_base64_nonce(nonce, nonce_refs, i_cnt):
    """`Content-Security-Policy` header checks related to Base64 nonces."""
    try:
        return csp_print_nonce(nonce, nonce_refs, i_cnt) if \
            len(b64decode(nonce, validate=True)) < LENGTH_BOUNDS[2] else False # nosec
    except binascii.Error:
        return csp_print_nonce(nonce, nonce_refs, i_cnt)

csp_check_additional(csp_dirs_vals)

Content-Security-Policy header check.

Related to broad and insecure values.

Source code in humble.py

def csp_check_additional(csp_dirs_vals):
    """`Content-Security-Policy` header check.

    Related to broad and insecure values.
    """
    checks = [(t_csp_broad, csp_check_broad),
              (t_csp_insecs, csp_check_insecure)]
    for match, csp_func in checks:
        if any(val in directive for directive in csp_dirs_vals for val in
               match):
            csp_func(csp_dirs_vals)
    csp_check_eval(csp_dirs_vals)
    csp_check_inline(csp_dirs_vals)

csp_check_broad(csp_dirs_vals)

Content-Security-Policy header check related to broad values.

Source code in humble.py

def csp_check_broad(csp_dirs_vals):
    """`Content-Security-Policy` header check related to broad values."""
    csp_broad_v = sorted({value for dir_vals in csp_dirs_vals if
                          dir_vals.strip() for value in dir_vals.split()[1:]
                          if f" {value} " in t_csp_broad})
    if not csp_broad_v:
        return
    csp_broad_dirs = {dir_vals.split()[0] for dir_vals in csp_dirs_vals
                      if any(f" {broad_val} " in t_csp_broad for broad_val in
                             dir_vals.split()[1:])}
    csp_print_broad(csp_broad_dirs, csp_broad_v, i_cnt)

csp_check_eval(csp_dirs_vals)

Content-Security-Policy header check.

Related to unsafe-eval and wasm-unsafe-eval keywords.

Source code in humble.py

def csp_check_eval(csp_dirs_vals):
    """`Content-Security-Policy` header check.

    Related to `unsafe-eval` and `wasm-unsafe-eval` keywords.
    """
    csp_unsafe_dirs = [
        dir_vals.split()[0] if " " in dir_vals else dir_vals
        for dir_vals in csp_dirs_vals
        if "unsafe-eval" in dir_vals and "wasm-unsafe-eval" not in dir_vals]
    if csp_unsafe_dirs:
        csp_print_unsafe(csp_unsafe_dirs, "[icspe_h]", "[icspev]", 5, i_cnt)

csp_check_hashes(csp_h)

Content-Security-Policy header checks related to hashes.

Source code in humble.py

def csp_check_hashes(csp_h):
    """`Content-Security-Policy` header checks related to hashes."""
    csp_unquoted_hashes(csp_h)
    invalid_algos = set()
    csp_hashes = re.findall(RE_PATTERN[17], csp_h)
    for algo, b64hash in csp_hashes:
        try:
            decoded = b64decode(b64hash, validate=True) # nosec
            if len(decoded) != HASH_CHARS[algo]:
                invalid_algos.add(algo)
        except binascii.Error:
            invalid_algos.add(algo)
    if invalid_algos:
        print_detail_r("[icshash_h]", is_red=True)
        i_cnt[0] += 1
        if not args.brief:
            print(get_detail("[icshash_f]", replace=True))
            print_detail("[icshashr_f]", num_lines=2)

csp_check_ignored(csp_header)

Content-Security-Policy header check.

Related to strict-dynamic keyword.

Source code in humble.py

def csp_check_ignored(csp_header):
    """`Content-Security-Policy` header check.

    Related to `strict-dynamic` keyword.
    """
    hash_p = bool(re.search(RE_PATTERN[17], csp_header))
    nonce_p = bool(re.search(RE_PATTERN[6], csp_header))
    if not (hash_p or nonce_p):
        i_cnt[0] += 1
        if args.brief:
            print_detail_r("[icsig_d]", is_red=True)
        else:
            print_detail_r("[icsig_d]", is_red=True)
            print_detail("[icsig]", num_lines=2)
    return False

csp_check_inline(csp_dirs_vals)

Content-Security-Policy header check.

Related to unsafe-inline keyword.

Source code in humble.py

def csp_check_inline(csp_dirs_vals):
    """`Content-Security-Policy` header check.

    Related to `unsafe-inline` keyword.
    """
    csp_unsafe_dirs = [
        dir_vals.split()[0] if " " in dir_vals else dir_vals
        for dir_vals in csp_dirs_vals if "unsafe-inline" in dir_vals]
    if csp_unsafe_dirs:
        csp_print_unsafe(csp_unsafe_dirs, "[icsp_h]", "[icsp]", 5, i_cnt)

csp_check_insecure(csp_dirs_vals)

Content-Security-Policy header check related to insecure values.

Source code in humble.py

def csp_check_insecure(csp_dirs_vals):
    """`Content-Security-Policy` header check related to insecure values."""
    csp_insec_v = sorted({value for value in t_csp_insecs if
                          any(value in directive for directive in
                              csp_dirs_vals)})
    csp_insec_dirs = {dir_vals.split()[0] for dir_vals in csp_dirs_vals
                      if any(unsafe_val in dir_vals for unsafe_val in
                             t_csp_insecs)}
    csp_print_insecure(csp_insec_v, csp_insec_dirs, i_cnt)

csp_check_ip(csp_h)

Content-Security-Policy header check related to IP address values.

Source code in humble.py

def csp_check_ip(csp_h):
    """`Content-Security-Policy` header check related to IP address values."""
    localhost_ip = ip_address(t_csp_checks[4])
    ip_matches = re.findall(RE_PATTERN[1], csp_h)
    for match in ip_matches:
        with suppress(ValueError):
            ip_match = ip_address(match)
            if ip_match != localhost_ip:
                print_details("[icsipa_h]", "[icsipa]", "m", i_cnt)
                break

csp_check_missing(csp_dirs)

Content-Security-Policy header check.

Related to missing directives.

Source code in humble.py

def csp_check_missing(csp_dirs):
    """`Content-Security-Policy` header check.

    Related to missing directives.
    """
    csp_refs = [("[icspmb_h]", "[icspmb]"), ("[icspmc_h]", "[icspmc]"),
                ("[icspmcn_h]", "[icspmcn]"), ("[icspmfo_h]", "[icspmfo]"),
                ("[icspmf_h]", "[icspmf]"), ("[icspmfa_h]", "[icspmfa]"),
                ("[icspmi_h]", "[icspmi]"), ("[icspmo_h]", "[icspmo]"),
                ("[icspmr_h]", "[icspmr]"), ("[icspms_h]", "[icspms]"),
                ("[icspmst_h]", "[icspmst]"), ("[icspmstt_h]", "[icspmstt]"),
                ("[icspmsw_h]", "[icspmsw]")]
    for directive, (csp_ref_brief, csp_ref) in zip(t_csp_miss, csp_refs,
                                                   strict=True):
        if directive not in csp_dirs:
            csp_print_missing(csp_ref, csp_ref_brief)

csp_check_nonces(csp_h)

Content-Security-Policy header checks.

Related to hexadecimal and Base64 nonces.

Source code in humble.py

def csp_check_nonces(csp_h):
    """`Content-Security-Policy` header checks.

    Related to hexadecimal and Base64 nonces.
    """
    if re.search(RE_PATTERN[23], csp_h):
        print_details("[icsncei_h]", "[icsncei]", "d", i_cnt)
    nonce_refs = ("[icsnces_h]", "[icsncesn]", "[icsnces]")
    for nonce in re.findall(RE_PATTERN[6], csp_h):
        if (re.match(RE_PATTERN[12], nonce) and
            csp_hex_nonce(nonce, nonce_refs, i_cnt)) or \
           (re.match(RE_PATTERN[13], nonce) and
           csp_base64_nonce(nonce, nonce_refs, i_cnt)):
            return

csp_check_unknown(csp_h)

Content-Security-Policy header check related to unknown directives.

Source code in humble.py

def csp_check_unknown(csp_h):
    """`Content-Security-Policy` header check related to unknown directives."""
    unknown_dir = []
    csp_dirs = [d.strip() for d in csp_h.split(";") if d.strip()]
    for directive in csp_dirs:
        if match := re.match(RE_PATTERN[19], directive):
            dir_name = match[1]
            if dir_name not in t_csp_dirs + t_csp_dep:
                unknown_dir.append(dir_name)
    if unknown_dir:
        csp_print_unknown(unknown_dir)

csp_hex_nonce(nonce, nonce_refs, i_cnt)

Content-Security-Policy header checks.

Related to hexadecimal nonces.

Source code in humble.py

def csp_hex_nonce(nonce, nonce_refs, i_cnt):
    """`Content-Security-Policy` header checks.

    Related to hexadecimal nonces.
    """
    return csp_print_nonce(nonce, nonce_refs, i_cnt) \
        if len(nonce) < LENGTH_BOUNDS[3] else False

csp_print_broad(csp_broad_dirs, csp_broad_v, i_cnt)

Print the broad value in the Content-Security-Policy header.

Source code in humble.py

def csp_print_broad(csp_broad_dirs, csp_broad_v, i_cnt):
    """Print the broad value in the `Content-Security-Policy` header."""
    print_detail_r("[icsw_h]", is_red=True)
    if not args.brief:
        print_detail_l(DIR_MSG[0] if len(csp_broad_dirs) > 1 else DIR_MSG[1])
        print(" " + ", ".join(f"'{directive}'" for directive in
                              sorted(csp_broad_dirs)) + ".")
        print_detail_l("[icsw]")
        print(", ".join(f"'{value}'" for value in csp_broad_v))
        print_detail("[icsw_b]", num_lines=1)
    i_cnt[0] += 1

csp_print_deprecated(csp_deprecated)

Content-Security-Policy header check.

Print deprecated directives.

Source code in humble.py

def csp_print_deprecated(csp_deprecated):
    """`Content-Security-Policy` header check.

    Print deprecated directives.
    """
    i_cnt[0] += 1
    print_detail_r("[icsi_d]", is_red=True) if args.brief else \
        csp_print_details(csp_deprecated, "[icsi_d]", "[icsi_d_s]",
                          "[icsi_d_r]")

csp_print_details(csp_values, csp_title, csp_desc, csp_refs)

Content-Security-Policy` header related.

Group the deprecated directives.

Source code in humble.py

def csp_print_details(csp_values, csp_title, csp_desc, csp_refs):
    """Content-Security-Policy` header related.

    Group the deprecated directives.
    """
    csp_values = ", ".join(f"'{value}'" for value in sorted(csp_values))
    print_detail_r(csp_title, is_red=True)
    print_detail_l(csp_desc)
    print(csp_values)
    print_detail(csp_refs, num_lines=3)

csp_print_insecure(csp_insec_v, csp_insec_dirs, i_cnt)

Print the insecure value in the Content-Security-Policy header.

Source code in humble.py

def csp_print_insecure(csp_insec_v, csp_insec_dirs, i_cnt):
    """Print the insecure value in the `Content-Security-Policy` header."""
    print_detail_r("[icsh_h]", is_red=True)
    if not args.brief:
        csp_values = ", ".join(f"'{value}'" for value in csp_insec_v)
        print_detail_l(DIR_MSG[0] if len(csp_insec_dirs) > 1 else DIR_MSG[1])
        print(" " + ", ".join(f"'{directive}'" for directive in
                              sorted(csp_insec_dirs)) + ".")
        print_detail_l("[icsh]")
        print(csp_values)
        print_detail("[icsh_b]", num_lines=2)
    i_cnt[0] += 1

csp_print_missing(csp_ref, csp_ref_brief)

Print the missing directive in the Content-Security-Policy header.

Source code in humble.py

def csp_print_missing(csp_ref, csp_ref_brief):
    """Print the missing directive in the `Content-Security-Policy` header."""
    if args.brief:
        i_cnt[0] += 1
        print_detail_r(csp_ref_brief, is_red=True)
    elif csp_ref == "[icspmfa]":
        i_cnt[0] += 1
        print_detail_r(csp_ref_brief, is_red=True)
        print_detail(csp_ref, num_lines=4)
    else:
        print_details(csp_ref_brief, csp_ref, "d", i_cnt)

csp_print_nonce(nonce, nonce_refs, i_cnt)

Content-Security-Policy header checks.

Print insecure Base64 and hexadecimal nonces

Source code in humble.py

def csp_print_nonce(nonce, nonce_refs, i_cnt):
    """`Content-Security-Policy` header checks.

    Print insecure Base64 and hexadecimal nonces
    """
    print_detail_r(nonce_refs[0], is_red=True)
    if not args.brief:
        print_detail_l(nonce_refs[1])
        print(f"'{nonce}'.")
        print_detail(nonce_refs[2], num_lines=2)
    i_cnt[0] += 1
    return True

csp_print_unknown(unknown_dir)

Print unknown directives in the Content-Security-Policy header.

Source code in humble.py

def csp_print_unknown(unknown_dir):
    """Print unknown directives in the `Content-Security-Policy` header."""
    # sourcery skip: use-fstring-for-concatenation
    print_detail_r("[icspiu_h]", is_red=True)
    if not args.brief:
        print_detail_l(DIR_MSG[0] if len(unknown_dir) > 1 else DIR_MSG[1])
        print(" " + ", ".join(f"'{directive}'" for directive in
                              sorted(unknown_dir)) + ".")
        print_detail("[icspiu]", num_lines=3)
    i_cnt[0] += 1

csp_print_unsafe(csp_unsafe_dirs, detail_t, detail_d, lines_n, i_cnt)

Content-Security-Policy header check.

Print the occurrences of unsafe-eval and unsafe-inline keywords.

Source code in humble.py

def csp_print_unsafe(csp_unsafe_dirs, detail_t, detail_d, lines_n, i_cnt):
    """`Content-Security-Policy` header check.

    Print the occurrences of `unsafe-eval` and `unsafe-inline` keywords.
    """
    print_detail_r(detail_t, is_red=True)
    if not args.brief:
        print_detail_l(DIR_MSG[0] if len(csp_unsafe_dirs) > 1 else DIR_MSG[1])
        print(" " + ", ".join(f"'{directive}'" for directive in
                              sorted(set(csp_unsafe_dirs))) + ".")
        print_detail(detail_d, num_lines=lines_n)
    i_cnt[0] += 1

csp_unquoted_hashes(csp_h)

Content-Security-Policy header check related to unquoted hashes.

Source code in humble.py

def csp_unquoted_hashes(csp_h):
    """`Content-Security-Policy` header check related to unquoted hashes."""
    if re.search(RE_PATTERN[18], csp_h):
        print_detail_r("[icshash_h]", is_red=True)
        i_cnt[0] += 1
        if not args.brief:
            print(get_detail("[icshash_f]", replace=True))
            print_detail("[icshashr_f]", num_lines=2)

custom_help_formatter(prog)

Format help output with an increased character limit per line.

Source code in humble.py

def custom_help_formatter(prog):
    """Format help output with an increased character limit per line."""
    return RawDescriptionHelpFormatter(prog, max_help_position=43)

decrease_html_spacing(tmp_filename)

Decrease the spacing between sections.

Related to -o html option.

Source code in humble.py

def decrease_html_spacing(tmp_filename):
    """Decrease the spacing between sections.

    Related to `-o html` option.
    """
    initial_ln, prev_blank_ln = False, False
    cleaned_ln = []
    with Path(tmp_filename).open(encoding="utf8") as html_source:
        for line in html_source:
            if not initial_ln and INFO_SECTION in line:
                initial_ln = True
            if initial_ln and not line.strip() and prev_blank_ln:
                continue
            prev_blank_ln = initial_ln and not line.strip()
            cleaned_ln.append(line)
    with Path(tmp_filename).open("w", encoding="utf8") as html_output:
        html_output.writelines(cleaned_ln)

delete_lines(*, reliable=True)

Clear console lines to standardize the final analysis output format.

Removes previously printed lines to ensure that final messages (success or error) are consistently padded with a blank line before and after.

Source code in humble.py

def delete_lines(*, reliable=True):
    """Clear console lines to standardize the final analysis output format.

    Removes previously printed lines to ensure that final messages (success
    or error) are consistently padded with a blank line before and after.
    """
    if not reliable:
        sys.stdout.write(DELETED_LINES)
    sys.stdout.write(DELETED_LINES)

escape_html_value(ln)

Escape a label-delimited value while preserving humble's own tags.

Only the content inside the single quotes following a localized value label (Value:/Valor:) is escaped, with quote=False so benign output is unchanged; lines without such a label are returned unaltered. Related to -o html option.

Source code in humble.py

def escape_html_value(ln):
    """Escape a label-delimited value while preserving humble's own tags.

    Only the content inside the single quotes following a localized value
    label (`Value:`/`Valor:`) is escaped, with `quote=False` so benign
    output is unchanged; lines without such a label are returned unaltered.
    Related to `-o html` option.
    """
    for label in VALUE_LABELS:
        head, sep, tail = ln.partition(f"{label}'")
        if sep and tail.endswith("'\n"):
            value = escape(tail[:-2], quote=False)
            return f"{head}{sep}{value}'\n"
        if sep and tail.endswith("'"):
            value = escape(tail[:-1], quote=False)
            return f"{head}{sep}{value}'"
    return ln

export_all_formats(final_filename, tmp_filename)

Export the analysis to all supported formats.

This function sequentially invokes the functions for CSV, XLSX, JSON, XML, HTML, PDF, and TXT exports. It passes the export_all flag where applicable to ensure consistent processing and terminate execution after displaying the final export path.

Related to -o all option.

Source code in humble.py

def export_all_formats(final_filename, tmp_filename):
    """Export the analysis to all supported formats.

    This function sequentially invokes the functions for CSV, XLSX, JSON,
    XML, HTML, PDF, and TXT exports. It passes the `export_all` flag where
    applicable to ensure consistent processing and terminate execution after
    displaying the final export path.

    Related to `-o all` option.
    """
    generate_csv(final_filename, tmp_filename, export_all=True)
    if args.brief:
        generate_json(final_filename, tmp_filename, export_all=True)
    else:
        generate_json_detailed(final_filename, tmp_filename, export_all=True)
    generate_xml(final_filename, tmp_filename, export_all=True)
    generate_csv(final_filename, tmp_filename, to_xlsx=True, export_all=True)
    normalize_htmlpdf_all_export("html", tmp_filename, final_filename)
    normalize_htmlpdf_all_export("pdf", tmp_filename)
    normalize_txt_all_export(tmp_filename)
    print_export_path(final_filename, reliable, export_all=True)
    sys.exit(0)

export_html_file(final_filename, tmp_filename, *, export_all=False)

HTML export of the analysis, related to -o html option.

Source code in humble.py

def export_html_file(final_filename, tmp_filename, *, export_all=False):
    """HTML export of the analysis, related to `-o html` option."""
    generate_html()
    decrease_html_spacing(tmp_filename)
    html_writers, html_rest = build_html_writers()
    inside_section = False
    with (
        Path(tmp_filename).open(encoding="utf8") as html_source,
        Path(final_filename).open("a", encoding="utf8") as html_final,
    ):
        for ln in html_source:
            inside_section = write_html_line(html_final, ln, html_writers,
                                             html_rest, inside_section)
        if inside_section:
            html_final.write(HTML_TAGS[12])
        html_final.write(HTML_TAGS[13])
    clean_html_final(final_filename)
    finalize_export(final_filename, tmp_filename, "html", export_all)

export_pdf_file(tmp_filename, *, export_all=False)

PDF export of the analysis, related to -o pdf option.

Tip

fpdf2 is lazy-loaded to avoid unnecessary overhead when PDF export is not used.

Source code in humble.py

def export_pdf_file(tmp_filename, *, export_all=False):
    """PDF export of the analysis, related to `-o pdf` option.

    ??? tip
        `fpdf2` is lazy-loaded to avoid unnecessary overhead when PDF export is
        not used.
    """
    from fpdf import FPDF, YPos

    class PDF(FPDF):

        def header(self):
            self.set_font("Courier", "B", 9)
            self.set_y(10)
            self.set_text_color(0, 0, 0)
            self.cell(0, 5, get_detail("[humble_desc]"), new_x="CENTER",
                      new_y="NEXT", align="C")
            self.ln(1)
            self.cell(0, 5, BANNER_VERSION, align="C")
            self.ln(9 if self.page_no() == 1 else 13)

        def footer(self):
            self.set_y(-15)
            self.set_font("Helvetica", "I", 8)
            self.set_text_color(0, 0, 0)
            self.cell(0, 10, f"{print_detail_s('[pdf_footer]')} \
{self.page_no()}{get_detail('[pdf_footer2]')} {{nb}}", align="C")

    pdf = PDF()
    initialize_pdf(pdf, tmp_filename, YPos, export_all=export_all)

extract_date_metrics(url_ln)

Extract totals, by month and year, from the analyses performed on a URL.

Related to -a option.

Source code in humble.py

def extract_date_metrics(url_ln):
    """Extract totals, by month and year, from the analyses performed on a URL.

    Related to `-a` option.
    """
    year_cnt, year_wng = defaultdict(int), defaultdict(int)
    for line in url_ln:
        year = int(line[:SLICE_INT[11]])
        year_cnt[year] += 1
        year_wng[year] += int(line.rsplit(" ; ", 1)[-1])
    years_str = generate_date_groups(year_cnt, url_ln)
    avg_wng_y = sum(year_wng.values()) // len(year_wng)
    return years_str, avg_wng_y, year_wng

extract_global_metrics(all_analysis)

Compute metrics to print statistics across all URL analyses.

Related to -a option.

Source code in humble.py

def extract_global_metrics(all_analysis):
    """Compute metrics to print statistics across all URL analyses.

    Related to `-a` option.
    """
    url_ln = list(all_analysis)
    if not url_ln:
        print_error_detail("[no_global_analysis]")
    adj_url_ln = adjust_old_analysis(url_ln)
    total_a = len(adj_url_ln)
    first_m = get_global_first_metrics(adj_url_ln)
    second_m = [get_second_metrics(adj_url_ln, i, total_a) for i
                in range(2, 7)]
    third_m = get_third_metrics(adj_url_ln)
    additional_m = get_additional_metrics(adj_url_ln)
    analytics_l = get_analytics_length(SECTION_V[12:26])
    analytics_s = get_analytics_length(SECTION_V[:5])
    analytics_w = get_analytics_length(SECTION_V[5:12])
    return print_global_metrics(analytics_l, analytics_s, analytics_w, total_a,
                                first_m, second_m, third_m, additional_m)

extract_har_content(har_data)

Extract response headers and status code from a HAR file.

A HAR file may be valid JSON yet not follow the expected structure, so any unexpected type yields empty headers, reported by the caller via [args_harlines].

Source code in humble.py

def extract_har_content(har_data):
    """Extract response headers and status code from a HAR file.

    A HAR file may be valid JSON yet not follow the expected structure, so
    any unexpected type yields empty headers, reported by the caller via
    `[args_harlines]`.
    """
    try:
        response = har_data["log"]["entries"][0]["response"]
        status_code = int(response.get("status", 0))
        input_headers = {header["name"].title():
                         (header.get("value") or "").strip()
                         for header in response["headers"]
                         if header.get("name")}
    except (AttributeError, IndexError, KeyError, TypeError, ValueError):
        return {}, 0
    return input_headers, status_code

fetch_cicd_grade(totals)

Fetch the analysis grade from the totals.

Related to -cicd GRADE option.

Source code in humble.py

def fetch_cicd_grade(totals):
    """Fetch the analysis grade from the totals.

    Related to `-cicd GRADE` option.
    """
    grade_note = (
        totals
        .get(get_detail("[cicd_grade]", replace=True), {})
        .get(get_detail("[cicd_grade_note]", replace=True), "")
    )
    return grade_note.split()[0].strip().upper() if grade_note else ""

finalize_export(final_filename, temp_filename, file_extension, export_all)

Manage the completion of exporting an analysis.

Handle file cleanup, renaming, and termination based on the export mode: in standalone mode (export_all=False), display the final path, remove temporary files, and exit; in batch mode (export_all=True), rename the generated file with the appropriate extension and update its content.

Related to -o option.

Source code in humble.py

def finalize_export(final_filename, temp_filename, file_extension, export_all):
    """Manage the completion of exporting an analysis.

    Handle file cleanup, renaming, and termination based on the export mode:
    in standalone mode (`export_all=False`), display the final path, remove
    temporary files, and exit; in batch mode (`export_all=True`), rename the
    generated file with the appropriate extension and update its content.

    Related to `-o` option.
    """
    if not export_all:
        print_export_path(final_filename, reliable)
        Path(temp_filename).unlink()
        sys.exit(0)
    new_filename = f"{final_filename[:-4]}.{file_extension}"
    Path(final_filename).rename(new_filename)
    dotted_ext = f".{file_extension}"
    if dotted_ext not in (EXPORT_EXTENSIONS[3], EXPORT_EXTENSIONS[5]):
        with Path(new_filename).open("r+", encoding="utf8") as processed_file:
            content = processed_file.read().replace(final_filename,
                                                    new_filename)
            processed_file.seek(0)
            processed_file.write(content)
            processed_file.truncate()

fix_pdf_all_export(tmp_filename)

Format and applies the correct extension to the PDF file.

Related to -o all option.

Source code in humble.py

def fix_pdf_all_export(tmp_filename):
    """Format and applies the correct extension to the PDF file.

    Related to `-o all` option.
    """
    base_pdffilename = Path(tmp_filename).stem[:-1]
    with (
        Path(tmp_filename).open("r+", encoding="utf8")
    ) as temp_pdffilename:
        content = "".join(temp_pdffilename.readlines()[6:])
        new_content = content.replace(f"{base_pdffilename}.all",
                                      f"{base_pdffilename}.pdf")
        temp_pdffilename.seek(0)
        temp_pdffilename.write(new_content)
        temp_pdffilename.truncate()
    return tmp_filename

fix_xlsx_all_export(csv_filename, export_all)

Apply the correct extension to the XSLX file.

Related to -o all option.

Source code in humble.py

def fix_xlsx_all_export(csv_filename, export_all):
    """Apply the correct extension to the XSLX file.

    Related to `-o all` option.
    """
    if export_all:
        identity = csv_filename.rsplit(".", 1)[0]
        with (
            Path(csv_filename).open("r+", encoding="utf8")
        ) as fixed_xlsxfilename:
            content = fixed_xlsxfilename.read()
            fixed_xlsxfilename.seek(0)
            fixed_xlsxfilename.write(content.replace(f"{identity}.all",
                                                     f"{identity}.xlsx"))
            fixed_xlsxfilename.truncate()

fng_statistics_term(fng_term)

Count fingerprint headers per provided term.

Exit if no results are found, related to -f option.

Source code in humble.py

def fng_statistics_term(fng_term):
    """Count fingerprint headers per provided term.

    Exit if no results are found, related to `-f` option.
    """
    print(
        f"\n{STYLE[0]}{get_detail('[fng_stats]', replace=True)}"
        f"{STYLE[4]}{get_detail('[fng_source]', replace=True)}\n",
    )
    fng_incl = PATHS["fingerprint_term"].read_text(encoding="utf8").splitlines(
        keepends=True,
    )[SLICE_INT[0]:]
    fng_groups, term_cnt = fng_statistics_term_groups(fng_incl, fng_term)
    if not fng_groups:
        print(
            f"{get_detail('[fng_zero]', replace=True)} '{fng_term}'.\n\n"
            f"{get_detail('[fng_zero_2]', replace=True)}.\n",
        )
        sys.exit(0)
    fng_statistics_term_content(fng_groups, fng_term, term_cnt, fng_incl)

fng_statistics_term_content(fng_groups, fng_term, term_cnt, fng_incl)

Compute percentage of fingerprint headers matching the search term.

Related to -f option.

Source code in humble.py

def fng_statistics_term_content(fng_groups, fng_term, term_cnt, fng_incl):
    """Compute percentage of fingerprint headers matching the search term.

    Related to `-f` option.
    """
    fng_pct = round(term_cnt / len(fng_incl) * 100, 2)
    print(f"{get_detail('[fng_add]', replace=True)} '{fng_term}': {fng_pct}%\
 ({term_cnt}{get_detail('[pdf_footer2]', replace=True)} {len(fng_incl)})")
    fng_statistics_term_sorted(fng_incl, fng_term.lower(), fng_groups)

fng_statistics_term_groups(fng_incl, fng_term)

Compute the total of fingerprint headers per provided term.

It takes into account also the service; related to -f option.

Source code in humble.py

def fng_statistics_term_groups(fng_incl, fng_term):
    """Compute the total of fingerprint headers per provided term.

    It takes into account also the service; related to `-f` option.
    """
    fng_matches = [match for line in fng_incl if
                   (match := re.search(RE_PATTERN[0], line)) and
                   fng_term.lower() in match[1].lower()]
    fng_groups = sorted({match[1].strip() for match in fng_matches})
    term_cnt = len(fng_matches)
    return fng_groups, term_cnt

fng_statistics_term_sorted(fng_incl, fng_term, fng_groups)

Print the service associated with the term provided.

In alphabetical order and terminate execution; related to -f option.

Source code in humble.py

def fng_statistics_term_sorted(fng_incl, fng_term, fng_groups):
    """Print the service associated with the term provided.

    In alphabetical order and terminate execution; related to `-f` option.
    """
    for content in fng_groups:
        print(f"\n [{STYLE[0]}{content}]")
        content_lower = content.lower()
        for line in fng_incl:
            line_lower = line.lower()
            if content_lower in line_lower and fng_term in line_lower:
                print(f"  {line[:line.find('(')].strip()}")
    sys.exit(0)

fng_statistics_top()

Print top 20 HTTP fingerprint header statistics.

Grouped by service and terminate execution, related to -f option.

Source code in humble.py

def fng_statistics_top():
    """Print top 20 HTTP fingerprint header statistics.

    Grouped by service and terminate execution, related to `-f` option.
    """
    print(
        f"\n{STYLE[0]}{get_detail('[fng_stats]', replace=True)}"
        f"{STYLE[4]}{get_detail('[fng_source]', replace=True)}\n",
    )
    fng_lines = PATHS["fingerprint_top"].read_text(encoding="utf8").splitlines(
        keepends=True,
    )
    fng_lines = fng_lines[SLICE_INT[0]:]
    fng_incl = len(fng_lines)
    fng_statistics_top_groups(fng_lines, fng_incl)
    sys.exit(0)

fng_statistics_top_groups(fng_lines, fng_incl)

Count fingerprint headers per service, related to -f option.

Source code in humble.py

def fng_statistics_top_groups(fng_lines, fng_incl):
    """Count fingerprint headers per service, related to `-f` option."""
    top_groups_pattern = re.compile(RE_PATTERN[3])
    fng_top_groups = Counter(match.strip() for line in fng_lines for match in
                             top_groups_pattern.findall(line))
    fng_statistics_top_result(fng_top_groups, fng_incl)

fng_statistics_top_result(fng_top_groups, fng_incl)

Print the Top 20 services.

By total number of fingerprint headers, related to -f option.

Source code in humble.py

def fng_statistics_top_result(fng_top_groups, fng_incl):
    """Print the Top 20 services.

    By total number of fingerprint headers, related to `-f` option.
    """
    max_ln_len = max(len(content) for content, _ in
                     fng_top_groups.most_common(20))
    print(f"{get_detail('[fng_top]', replace=True)} {fng_incl}\
{get_detail('[fng_top_2]', replace=True)}\n")
    for content, count in fng_top_groups.most_common(20):
        fng_global_pct = round(count / fng_incl * 100, 2)
        fng_padding = " " * (max_ln_len - len(content))
        print(f" [{content}]: {fng_padding}{fng_global_pct:.2f}% ({count})")

format_analysis_results(*diff, en_cnt_w, t_cnt)

Format differences in the analysis.

Between the last and the current one for the same URL.

Source code in humble.py

def format_analysis_results(*diff, en_cnt_w, t_cnt):
    """Format differences in the analysis.

    Between the last and the current one for the same URL.
    """
    results = [en_cnt, m_cnt, f_cnt, i_cnt[0], e_cnt, t_cnt]
    new_ln = ["\n" if int(en_cnt) > 0 else "", "", "", "", "", "\n\n"]
    totals = [f"{val:>2} ({diff[i]}){new_ln[i]}" for i, val in
              enumerate(results)]
    max_secl = get_max_lnlength(SECTION_S)
    print_analysis_results(totals, max_secl, en_cnt_w)

format_html_bold(html_final, ln_rstrip, inside_section)

Bold the section names.

Related to -o html option.

Source code in humble.py

def format_html_bold(html_final, ln_rstrip, inside_section):
    """Bold the section names.

    Related to `-o html` option.
    """
    if any(s in ln_rstrip for s in STRINGS_BOLD):
        if inside_section:
            html_final.write(HTML_TAGS[12])
        html_final.write(f"{HTML_TAGS[7]}{ln_rstrip}{HTML_TAGS[8]}")
        inside_section = True
        return True, inside_section
    return False, inside_section

format_html_compatibility(html_final, ln_rstrip)

Write formatted lines for browser compatibility section.

Related to -o html option.

Source code in humble.py

def format_html_compatibility(html_final, ln_rstrip):
    """Write formatted lines for browser compatibility section.

    Related to `-o html` option.
    """
    if URL_STRING[2] not in ln_rstrip:
        return False
    prefix, _, link = ln_rstrip.partition(": ")
    html_final.write(
        f"{HTML_TAGS[4]}{prefix[1:]}: {HTML_TAGS[5]}"
        f"{HTML_TAGS[1]}{link}{HTML_TAGS[2]}{link}{HTML_TAGS[0]}\
{HTML_TAGS[11]}")
    return True

format_html_csp(ln)

Bold Content-Security-Policy header directives.

Related to -o html option.

Source code in humble.py

def format_html_csp(ln):
    """Bold `Content-Security-Policy` header directives.

    Related to `-o html` option.
    """
    if "content-security-policy" not in ln.casefold():
        return ln
    return RE_CSP_DIRS.sub(
        lambda m: f"{m.group(1)}{HTML_TAGS[14]}{m.group(2)}{HTML_TAGS[15]}\
{m.group(3)}", ln)

format_html_empty(ln, ln_rstrip, l_empty)

Apply format to empty HTTP response headers.

By validating line content against a predefined list of empty headers.

Related to -o html option.

Source code in humble.py

def format_html_empty(ln, ln_rstrip, l_empty):
    """Apply format to empty HTTP response headers.

    By validating line content against a predefined list of empty headers.

    Related to `-o html` option.
    """
    ln_strip = ln_rstrip.lstrip().lower()
    for i in l_empty:
        if (i.lower() in ln_strip and "[" not in ln_strip and ":" not in
           ln_strip and HTML_TAGS[3] not in ln):
            ln = f"{HTML_TAGS[3]}{ln}{HTML_TAGS[5]}"
            break
    return ln

format_html_enabled(ln, html_final)

Write formatted lines for the section with enabled HTTP headers.

Related to -o html option.

Source code in humble.py

def format_html_enabled(ln, html_final):
    """Write formatted lines for the section with enabled HTTP headers.

    Related to `-o html` option.
    """
    if STYLE[8] not in ln:
        return ln, False
    ln = f" {ln[19:].rstrip()}"
    if ":" in ln:
        header, value = map(str.strip, ln.split(":", 1))
        ln = (f"{HTML_TAGS[6]} {header}{HTML_TAGS[5]}: "
              f"{escape(value, quote=False)}")
    else:
        ln = f"{HTML_TAGS[6]} {ln.strip()}{HTML_TAGS[5]}"
    html_final.write(f"{format_html_csp(ln)}{HTML_TAGS[11]}")
    return ln, True

format_html_fingerprint(args, ln, l_fng)

Write formatted lines for fingerprint headers.

Related to -o html option.

Source code in humble.py

def format_html_fingerprint(args, ln, l_fng):
    """Write formatted lines for fingerprint headers.

    Related to `-o html` option.
    """
    ln_cf = ln.casefold() if args.brief else ln
    for i in l_fng:
        i_match = i.casefold() if args.brief else i
        if i_match in ln_cf and ":" not in ln and HTML_TAGS[9] not in ln and \
           HTML_TAGS[10] not in ln:
            return f"{HTML_TAGS[3]}{ln}{HTML_TAGS[5]}"
    return ln

format_html_headers(ln, header_prefixes)

Format HTTP response header lines for an HTML export.

Sanitizes header values using HTML escaping to ensure that special characters (e.g., < or > in headers) are rendered correctly and do not interfere with the HTML structure; related to the -o html option.

Source code in humble.py

def format_html_headers(ln, header_prefixes):
    """Format HTTP response header lines for an HTML export.

    Sanitizes header values using HTML escaping to ensure that special
    characters (e.g., `<` or `>` in headers) are rendered correctly and do not
    interfere with the HTML structure; related to the `-o html` option.
    """
    for _header, header_prefix in header_prefixes:
        if header_prefix in ln:
            header_name, _, header_value = ln.partition(":")
            safe_value = escape(header_value.strip())
            ln = f"{HTML_TAGS[4]}{header_name}{HTML_TAGS[5]}: {safe_value}\n"
            ln = format_html_csp(ln)
            break
    return ln

format_html_info(html_final, ln_rstrip)

Write formatted lines.

For the GitHub URL of humble and the section with basic information.

Related to -o html option.

Source code in humble.py

def format_html_info(html_final, ln_rstrip):
    """Write formatted lines.

    For the GitHub URL of `humble` and the section with basic information.

    Related to `-o html` option.
    """
    if URL_STRING[0] in ln_rstrip:
        html_final.write(
            f"{HTML_TAGS[1]}{ln_rstrip[:32]}{HTML_TAGS[2]}"
            f"{ln_rstrip[:32]}{HTML_TAGS[0]}{ln_rstrip[32:]}",
        )
        return True
    if URL_STRING[1] in ln_rstrip:
        safe_url = escape(ln_rstrip[8:])
        html_final.write(
            f"{ln_rstrip[:8]}{HTML_TAGS[1]}{safe_url}"
            f"{HTML_TAGS[2]}{safe_url}{HTML_TAGS[0]}{HTML_TAGS[11]}",
        )
        return True
    return False

format_html_references(html_final, ln_rstrip, *, lang_slice)

Write formatted lines for references.

Related to -o html option.

Source code in humble.py

def format_html_references(html_final, ln_rstrip, *, lang_slice):
    """Write formatted lines for references.

    Related to `-o html` option.
    """
    for ref, off in ((REF_LINKS[1], 6), (REF_LINKS[0], 8), (REF_LINKS[4],
                                                            lang_slice)):
        if ref in ln_rstrip:
            content = ln_rstrip[off:].strip()
            html_final.write(
                f"{ln_rstrip[:off]}{HTML_TAGS[1]}{content}"
                f"{HTML_TAGS[2]}{content}{HTML_TAGS[0]}{HTML_TAGS[11]}",
            )
            return True
    return False

format_html_rest(html_final, ln, *, l_empty, l_total, fng_sorted, header_prefixes)

Write formatted lines for the rest of the sections.

E.g. highlighting in red the insecure headers; related to -o html option.

Source code in humble.py

def format_html_rest(html_final, ln, *, l_empty, l_total, fng_sorted,
                     header_prefixes):
    """Write formatted lines for the rest of the sections.

    E.g. highlighting in red the insecure headers; related to `-o html` option.
    """
    ln, ln_enabled = format_html_enabled(ln, html_final)
    ln_rstrip = ln.rstrip("\n")
    if ln and not ln_enabled:
        ln = format_html_headers(ln, header_prefixes)
        ln = format_html_fingerprint(args, ln, fng_sorted)
        ln = format_html_totals(ln, l_total)
        ln = format_html_empty(ln, ln_rstrip, l_empty)
        html_final.write(escape_html_value(ln))

format_html_totals(ln, l_total)

Highlight in red the header that fails any of the checks.

Related to -o html option.

Source code in humble.py

def format_html_totals(ln, l_total):
    """Highlight in red the header that fails any of the checks.

    Related to `-o html` option.
    """
    for i in l_total:
        if (not re.search(RE_PATTERN[11], ln)) and (
             ((i in ln) and ('"' not in ln)) or ("HTTP (" in ln) or
             (XFRAME_CHECK in ln)):
            ln = f"{HTML_TAGS[3]}{ln}{HTML_TAGS[5]}"
            break
    return ln

format_html_warnings(html_final, ln_rstrip, *, ko_strings, ok_string)

Write formatted lines for sections without results.

Either because they have passed all checks or because the headers could not be retrieved.

Related to -o html option.

Source code in humble.py

def format_html_warnings(html_final, ln_rstrip, *, ko_strings, ok_string):
    """Write formatted lines for sections without results.

    Either because they have passed all checks or because the headers could
    not be retrieved.

    Related to `-o html` option.
    """
    if ok_string in ln_rstrip:
        html_final.write(f"{HTML_TAGS[6]}{ln_rstrip}{HTML_TAGS[5]}\
{HTML_TAGS[11]}")
        return True
    if any(ko in ln_rstrip for ko in ko_strings):
        html_final.write(f"{HTML_TAGS[3]}{ln_rstrip}{HTML_TAGS[5]}\
                         {HTML_TAGS[11]}")
        return True
    return False

format_htmlpdf_all_export(line, export_format, target_state, in_browser)

Format the previously selected lines for HTML and PDF exports.

Related to -o all option.

Source code in humble.py

def format_htmlpdf_all_export(line, export_format, target_state, in_browser):
    """Format the previously selected lines for HTML and PDF exports.

    Related to `-o all` option.
    """
    if in_browser and line.strip() and not line.startswith("[6."):
        line = f" {line}" if export_format == "html" else f" {line.lstrip()}"
    if not (target_state and line.startswith(" ")):
        return line
    match export_format:
        case "html":
            return f" {STYLE[8]}{line[1:]}"
        case "pdf":
            return f" {STYLE[6]}{line[1:]}"
    return line

format_json(json_data, json_lns)

Format content, grouping duplicate keys, for a JSON export.

Related to -o json -b options.

Source code in humble.py

def format_json(json_data, json_lns):
    """Format content, grouping duplicate keys, for a JSON export.

    Related to `-o json -b` options.
    """
    for line in json_lns:
        if ":" in line:
            key, value = (part.strip() for part in line.split(":", 1))
            if key in json_data:
                if isinstance(json_data[key], list):
                    json_data[key].append(value)
                else:
                    json_data[key] = [json_data[key], value]
            else:
                json_data[key] = value
    return json_data

format_pdf_chunks(chunk, chunks, chunk_c, i, pdf)

Apply formatting and positioning to blocks of text.

Related to -o pdf option.

Source code in humble.py

def format_pdf_chunks(chunk, chunks, chunk_c, i, pdf):
    """Apply formatting and positioning to blocks of text.

    Related to `-o pdf` option.
    """
    if i > 0:
        chunk = f" {chunk}"
    if chunk_c == PDF_COLORS[2]:
        return chunk
    if i == 1 and len(chunks) >= LENGTH_BOUNDS[5]:
        pdf.set_y(pdf.get_y() - 1)
    elif len(chunks) == 1:
        pdf.set_y(pdf.get_y() - 0.5)
    return chunk

format_pdf_lines(line, pdf, ypos)

Identify lines, by length and content, to apply formatting.

Relatede to -o pdf option.

Source code in humble.py

def format_pdf_lines(line, pdf, ypos):
    """Identify lines, by length and content, to apply formatting.

    Relatede to `-o pdf` option.
    """
    if len(line) > LENGTH_BOUNDS[4]:
        chunks = [line[i:i + LENGTH_BOUNDS[4]] for i in range(0, len(line),
                                                              LENGTH_BOUNDS[4])]
        set_pdf_chunks(chunks, pdf)
        pdf.ln(h=2)
        return
    if re.search(RE_PATTERN[10], line):
        color_pdf_line(line[19:], PDF_COLORS[0], PDF_COLORS[1], None, None,
                       pdf)
        return
    if re.search(RE_PATTERN[7], line):
        color_pdf_line(line[19:], PDF_COLORS[2], PDF_COLORS[1], None, None,
                       pdf)
        return
    pdf.set_text_color(0, 0, 0)
    pdf.multi_cell(197, 6, text=line, align="L", new_y=ypos.LAST)

format_pdf_links(i, pdf_string, pdf, pdf_prefixes)

Apply a specific format to lines containing links.

Related to -o pdf option.

Source code in humble.py

def format_pdf_links(i, pdf_string, pdf, pdf_prefixes):
    """Apply a specific format to lines containing links.

    Related to `-o pdf` option.
    """
    pdf_link = set_pdf_links(i, pdf_string)
    if pdf_string in (URL_STRING[1], REF_LINKS[2], REF_LINKS[3]):
        pdf_prefix = pdf_prefixes.get(pdf_string, pdf_string)
        pdf.write(h=6, text=pdf_prefix)
    else:
        pdf.write(h=6, text=i[:i.index(": ")+2])
    pdf.set_text_color(0, 0, 255)
    pdf.cell(w=2000, h=6, text=i[i.index(": ")+2:], align="L", link=pdf_link)

generate_csv(final_filename, temp_filename, *, to_xlsx=False, export_all=False)

Export to CSV and XSLX and exit.

Related to -o csv option.

Source code in humble.py

def generate_csv(final_filename, temp_filename, *, to_xlsx=False,
                 export_all=False):
    """Export to CSV and XSLX and exit.

    Related to `-o csv` option.
    """
    with (
        Path(temp_filename).open(encoding="utf8") as txt_source,
        Path(final_filename).open(
            "w", newline="", encoding="utf8",
        ) as csv_final,
    ):
        write_csv_content(csv_final, txt_source)
    if to_xlsx:
        fix_xlsx_all_export(final_filename, export_all)
        generate_xlsx(final_filename, temp_filename, export_all=export_all)
        return
    finalize_export(final_filename, temp_filename, "csv", export_all)

generate_date_groups(year_cnt, url_ln)

Generate a formatted summary of analyses performed on a URL.

By year and month; related to -a option.

Source code in humble.py

def generate_date_groups(year_cnt, url_ln):
    """Generate a formatted summary of analyses performed on a URL.

    By year and month; related to `-a` option.
    """
    years_str = []
    for year in sorted(year_cnt.keys()):
        year_str = f" {year}: {year_cnt[year]} \
{get_detail('[analysis_y]').rstrip()}"
        month_cnts = get_month_counts(year, url_ln)
        months_str = "\n".join([f"  ({count}){month_name.rstrip()}" for
                                month_name, count in month_cnts.items()])
        year_str += f"\n{months_str}\n"
        years_str.append(year_str)
    return "\n".join(years_str)

generate_html()

Provide content for the variables of the template.

Source: /additional/html_template.html; related to -o html option.

Source code in humble.py

def generate_html():
    """Provide content for the variables of the template.

    Source: `/additional/html_template.html`; related to `-o html` option.
    """
    html_shell = Path(PATHS["html_source"]).read_text(encoding="utf8")
    html_replace = {"html_title": get_detail(METADATA_S[1]),
                    "html_desc": get_detail("[pdf_meta_title]"),
                    "html_keywords": get_detail(METADATA_S[0]),
                    "humble_URL": URL_LIST[4],
                    "humble_local_v": local_version,
                    "URL_analyzed": escape(URL),
                    "html_body": "<body><pre>"}
    replaced_html = Template(html_shell).substitute(html_replace)
    Path(final_filename).write_text(replaced_html, encoding="utf8")

generate_json(final_filename, temp_filename, *, export_all=False)

JSON export of a brief analysis and exit.

Related to -o json -b options.

Source code in humble.py

def generate_json(final_filename, temp_filename, *, export_all=False):
    """JSON export of a brief analysis and exit.

    Related to `-o json -b` options.
    """
    sections = tuple(get_detail(f"[{i}]", replace=True) for i in JSON_SECTION)
    with (
        Path(temp_filename).open(encoding="utf8") as txt_file,
        Path(final_filename).open("w", encoding="utf8") as json_file,
    ):
        txt_sections = re.split(RE_PATTERN[5], txt_file.read())[1:]
        dump(parse_json(sections, txt_sections), json_file,
             indent=4, ensure_ascii=False)
    finalize_export(final_filename, temp_filename, "json", export_all)

generate_json_detailed(final_filename, temp_filename, *, export_all=False)

JSON export of a detailed analysis and exit.

Related to -o json option.

Source code in humble.py

def generate_json_detailed(final_filename, temp_filename, *, export_all=False):
    """JSON export of a detailed analysis and exit.

    Related to `-o json` option.
    """
    with (
        Path(temp_filename).open(encoding="utf8") as txt_file,
        Path(final_filename).open("w", encoding="utf8") as json_file,
    ):
        txt_sections = re.split(RE_PATTERN[5], txt_file.read())[1:]
        data = {}
        json_detailed_parse(data, txt_sections)
        dump(data, json_file, indent=4, ensure_ascii=False)
    finalize_export(final_filename, temp_filename, "json", export_all)

generate_pdf(pdf, tmp_filename, pdf_links, pdf_prefixes, ypos, *, export_all=False)

Generate the required file structure, including metadata.

Related to -o pdf option.

Source code in humble.py

def generate_pdf(pdf, tmp_filename, pdf_links, pdf_prefixes, ypos, *,
                 export_all=False):
    """Generate the required file structure, including metadata.

    Related to `-o pdf` option.
    """
    set_pdf_file(pdf)
    ok_string = get_detail(DIR_MSG[2]).rstrip()
    no_headers = [get_detail(f"[{i}]").strip() for i in ("no_sec_headers",
                                                         "no_enb_headers")]
    set_pdf_content(tmp_filename, ok_string, no_headers, pdf, pdf_links,
                    pdf_prefixes, ypos)
    pdf.output(final_filename)
    finalize_export(final_filename, tmp_filename, "pdf", export_all)

generate_xlsx(final_filename, temp_filename, *, export_all=False)

XLSX spreadsheet export of the analysis and exit.

Related to -o xlsx option.

Tip

xlsxwriter is lazy-loaded to avoid unnecessary overhead when XLSX export is not used.

Source code in humble.py

def generate_xlsx(final_filename, temp_filename, *, export_all=False):
    """XLSX spreadsheet export of the analysis and exit.

    Related to `-o xlsx` option.

    ??? tip
        `xlsxwriter` is lazy-loaded to avoid unnecessary overhead when XLSX
        export is not used.
    """
    from xlsxwriter import Workbook
    workbook = Workbook(final_filename, {"in_memory": True})
    set_xlsx_metadata(workbook)
    set_xlsx_content(final_filename, workbook)
    workbook.close()
    finalize_export(final_filename, temp_filename, "xlsx", export_all)

generate_xml(final_filename, temp_filename, *, export_all=False)

XML export of the analysis, related to -o xml option.

Note

According to here (XML vulnerabilities section), here and here the minimum Python version required to run humble (3.11) should not be vulnerable to common XML attacks.

Additionally, the DTD used to generate the XML is defined in the constant DTD_CONTENT.

Source code in humble.py

def generate_xml(final_filename, temp_filename, *, export_all=False):
    """XML export of the analysis, related to `-o xml` option.

    ??? note
        According to [here](https://docs.python.org/3.11/library/xml.html){:target="_blank"}
        (*XML vulnerabilities* section), [here](https://github.com/python/cpython/pull/135294){:target="_blank"}
        and [here](https://github.com/python/cpython/issues/127502){:target="_blank"}
        the minimum Python version required to run *humble* (3.11) should not be
        vulnerable to common XML attacks.

        Additionally, the DTD used to generate the XML is defined in the
        constant `DTD_CONTENT`.
    """
    root = ET.Element("analysis", {"version": BANNER_VERSION,
                                   "generated": current_time})
    with Path(temp_filename).open(encoding="utf8") as txt_source:
        parse_xml(root, None, (line.strip() for line in txt_source))
    xml_decl = b'<?xml version="1.0" encoding="utf-8"?>\n'
    xml_content = ET.tostring(root, encoding="utf-8", xml_declaration=False)
    xml_dtd = f"<!DOCTYPE analysis [\n{DTD_CONTENT}]\n>\n".encode()
    with Path(final_filename).open("wb") as xml_final:
        xml_final.write(xml_decl + xml_dtd + xml_content)
    finalize_export(final_filename, temp_filename, "xml", export_all)

get_additional_metrics(adj_url_ln)

Compute the total of analyses performed on a URL, by month and year.

Related to -a option.

Source code in humble.py

def get_additional_metrics(adj_url_ln):
    """Compute the total of analyses performed on a URL, by month and year.

    Related to `-a` option.
    """
    avg_w = int(sum(int(line.split(" ; ")[-1]) for line in adj_url_ln) /
                len(adj_url_ln))
    year_a, avg_w_y, month_a = extract_date_metrics(adj_url_ln)
    return (avg_w, year_a, avg_w_y, month_a)

get_analysis_metrics(all_analysis)

Compute statistics for all analyses performed on a given URL.

Related to -a option.

Source code in humble.py

def get_analysis_metrics(all_analysis):
    """Compute statistics for all analyses performed on a given URL.

    Related to `-a` option.
    """
    url_ln = match_url_lines(all_analysis)
    if not url_ln:
        print_error_detail("[no_analysis]")
    adj_url_ln = adjust_old_analysis(url_ln)
    total_a = len(adj_url_ln)
    return print_metrics(
        get_analytics_length(SECTION_V[:5]),
        get_analytics_length(SECTION_V[5:12]),
        total_a,
        get_first_metrics(adj_url_ln),
        [get_second_metrics(adj_url_ln, i, total_a) for i in range(2, 7)],
        get_third_metrics(adj_url_ln),
        get_additional_metrics(adj_url_ln),
        get_highlights(adj_url_ln),
        get_trends(adj_url_ln),
    )

get_analysis_results()

Print analysis results and summary.

Source code in humble.py

def get_analysis_results():
    """Print analysis results and summary."""
    analysis_t = str(round(end - start, 2)).rstrip()
    print(f"{get_detail('[analysis_time]', replace=True)} {analysis_t}\
{get_detail('[analysis_time_sec]', replace=True)}")
    t_cnt = sum([m_cnt, f_cnt, i_cnt[0], e_cnt])
    analysis_totals = save_analysis_results(t_cnt)
    current = [en_cnt, m_cnt, f_cnt, i_cnt[0], e_cnt, t_cnt]
    analysis_diff = compare_analysis_results(analysis_totals, current)
    en_cnt_w = "1" if en_cnt == 0 else None
    format_analysis_results(*analysis_diff, en_cnt_w=en_cnt_w, t_cnt=t_cnt)
    analysis_grade = grade_analysis(en_cnt, m_cnt, f_cnt, i_cnt, e_cnt)
    print(get_detail(analysis_grade))
    print_detail("[experimental_header]", 3)

get_analysis_totals(url_ln)

Recover analysis totals, normalizing those performed before 11/28/2024.

Note

To avoid errors with analyses performed before 11/28/2024, the date on which enabled security headers began being considered when calculating differences between analyses of the same URL.

Therefore, analyses performed before that date are assumed to have no security headers enabled.

Source code in humble.py

def get_analysis_totals(url_ln):
    """Recover analysis totals, normalizing those performed before 11/28/2024.

    ??? note
        To avoid errors with analyses performed before 11/28/2024, the date on
        which enabled security headers began being considered when calculating
        differences between analyses of the same URL.

        Therefore, analyses performed before that date are assumed to have no
        security headers enabled.
    """
    updated_lines = []
    for line in url_ln:
        fields = line.strip().split(" ; ")
        if len(fields) == LENGTH_BOUNDS[1]:
            fields.insert(2, "0")
        updated_lines.append(" ; ".join(fields))
    url_ln = updated_lines
    analysis_date = max(line[:SLICE_INT[9]] for line in url_ln)
    for line in url_ln:
        if analysis_date in line:
            *totals, = line.strip().split(" ; ")
            break
    return tuple(totals[2:])

get_analytics_length(section)

Return the alignment padding for each item in a section.

Source code in humble.py

def get_analytics_length(section):
    """Return the alignment padding for each item in a section."""
    basic_l = get_max_lnlength(section) - 1
    section_l = []
    for i in section:
        section_l_item = " " * (basic_l - len(get_detail(i)))
        section_l.append(section_l_item)
    return section_l

get_averages_metrics(analytics_w, third_m)

Print average-related metrics details for a URL analysis.

Related to -a option.

Source code in humble.py

def get_averages_metrics(analytics_w, third_m):
    """Print average-related metrics details for a URL analysis.

    Related to `-a` option.
    """
    return {"[average_enb]": f"{analytics_w[2]}{third_m[0]}",
            "[average_miss]": f"{analytics_w[3]}{third_m[1]}",
            "[average_fng]": f"{analytics_w[4]}{third_m[2]}",
            "[average_dep]": f"{analytics_w[5]}{third_m[3]}",
            "[average_ety]": f"{analytics_w[6]}{third_m[4]}\n"}

get_basic_global_metrics(analytics_l, total_a, first_m)

Print metrics details across all URL analyses.

Related to -a option.

Source code in humble.py

def get_basic_global_metrics(analytics_l, total_a, first_m):
    """Print metrics details across all URL analyses.

    Related to `-a` option.
    """
    return {"[main]": "", "[total_analysis]": total_a,
            "[total_global_analysis]": str(first_m[2]),
            "[first_analysis_a]": first_m[0],
            "[latest_analysis]": f"{first_m[1]}\n",
            "[urls]": "",
            "[most_analyzed]": f"{analytics_l[0]}{first_m[3]}",
            "[least_analyzed]": f"{analytics_l[1]}{first_m[4]}\n",
            "[most_enabled]": f"{analytics_l[4]}{first_m[7]}",
            "[least_enabled]": f"{analytics_l[5]}{first_m[8]}\n",
            "[most_missing]": f"{analytics_l[6]}{first_m[9]}",
            "[least_missing]": f"{analytics_l[7]}{first_m[10]}\n",
            "[most_fingerprints]": f"{analytics_l[8]}{first_m[11]}",
            "[least_fingerprints]": f"{analytics_l[9]}{first_m[12]}\n",
            "[most_insecure]": f"{analytics_l[10]}{first_m[13]}",
            "[least_insecure]": f"{analytics_l[11]}{first_m[14]}\n",
            "[most_empty]": f"{analytics_l[12]}{first_m[15]}",
            "[least_empty]": f"{analytics_l[13]}{first_m[16]}\n",
            "[most_warnings]": f"{analytics_l[2]}{first_m[5]}",
            "[least_warnings]": f"{analytics_l[3]}{first_m[6]}\n"}

get_basic_metrics(total_a, first_m)

Print base metrics details for a URL analysis.

Related to -a option.

Source code in humble.py

def get_basic_metrics(total_a, first_m):
    """Print base metrics details for a URL analysis.

    Related to `-a` option.
    """
    return {"[main]": "", "[total_analysis]": total_a,
            "[first_analysis_a]": first_m[0], "[latest_analysis]": first_m[1],
            "[best_analysis]": f"{first_m[2]} \
{get_detail('[total_warnings]', replace=True)}{first_m[3]})",
            "[worst_analysis]": f"{first_m[4]} \
{get_detail('[total_warnings]', replace=True)}{first_m[5]})\n"}

get_cicd_labels()

Print literals related to the analysis designed for CI/CD.

Related to -cicd option.

Source code in humble.py

def get_cicd_labels():
    """Print literals related to the analysis designed for CI/CD.

    Related to `-cicd` option.
    """
    cidcd_labels = ["[cicd_total]", "[cicd_diff]", "[cicd_info]"]
    return tuple(get_detail(label, replace=True) for label in cidcd_labels)

get_date_metrics(additional_m)

Print date-related metrics details related.

Source code in humble.py

def get_date_metrics(additional_m):
    """Print date-related metrics details related."""
    return {"[analysis_year_month]": f"\n{additional_m[1]}"}

get_detail(id_mode, *, replace=False)

Print a message, optionally removing newlines.

Source code in humble.py

def get_detail(id_mode, *, replace=False):
    """Print a message, optionally removing newlines."""
    if match := next(
        (i for i, ln in enumerate(l10n_main) if ln.startswith(id_mode)),
        None,
    ):
        next_ln = l10n_main[match + 1]
        return next_ln.replace("\n", "") if replace else next_ln
    return None

get_enabled_headers(args, headers_l, t_enabled)

Print the contents of the section with enabled security headers.

Highlighting the experimental ones.

Note

The file associated with this check is security.txt.

The highlighted headers are defined in the EXP_HEADERS tuple and correspond to some of those indicated in the MDN list of HTTP headers.

Source code in humble.py

def get_enabled_headers(args, headers_l, t_enabled):
    """Print the contents of the section with enabled security headers.

    Highlighting the experimental ones.

    ??? note
        The file associated with this check is [security.txt](https://github.com/rfc-st/humble/blob/master/additional/security.txt){:target="_blank"}.

        The highlighted [headers](https://developer.mozilla.org/en-US/docs/
        MDN/Writing_guidelines/Experimental_deprecated_obsolete){:target="_blank"}
        are defined in the `EXP_HEADERS` tuple and correspond to some of those
        indicated in the MDN [list](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers){:target="_blank"}
        of HTTP headers.
    """
    headers_d = {key.title(): value for key, value in headers_l.items()}
    t_enabled = sorted({header.title() for header in t_enabled})
    enabled_headers = [header for header in t_enabled if header in headers_d]
    for header in enabled_headers:
        exp_s = get_detail("[exp_header]", replace=True) if header.lower() in\
          EXP_HEADERS else ""
        print_enabled_headers(args, exp_s, header, headers_d)
    None if enabled_headers else print_nosec_headers()
    en_cnt = len(enabled_headers)
    print("\n")
    return en_cnt

get_epilog_content(id_mode)

Return examples of use and contribution guidelines for humble.py.

Related to -h option.

Source code in humble.py

def get_epilog_content(id_mode):
    """Return examples of use and contribution guidelines for `humble.py`.

    Related to `-h` option.
    """
    target = id_mode + "\n"
    lines = PATHS["help_epilog"].read_text(encoding="utf8").splitlines(
        keepends=True,
    )
    start_idx = lines.index(target) + 1
    content = lines[start_idx : start_idx + SLICE_INT[12]]
    return "".join(content)

get_fingerprint_detail(header, headers, idx_fng, l_fng_ex, args)

Print the name, service and value of the fingerprint header.

Source: /additional/fingerprint.txt.

Source code in humble.py

def get_fingerprint_detail(header, headers, idx_fng, l_fng_ex, args):
    """Print the name, service and value of the fingerprint header.

    Source: `/additional/fingerprint.txt`.
    """
    if not args.brief:
        print_fng_header(l_fng_ex[idx_fng])
        header_value = headers_l.get(header.lower()) if "-if" in sys.argv else\
            headers[header]
        if header_value:
            print(f" {get_detail('[fng_value]', replace=True)} \
'{header_value}'")
        else:
            print(get_detail("[empty_fng]", replace=True))
        print()
    else:
        print_header(header)

get_fingerprint_headers()

Print the content in the section with fingerprint headers.

Note

The file associated with this check is fingerprint.txt.

Source code in humble.py

def get_fingerprint_headers():
    """Print the content in the section with fingerprint headers.

    ??? note
        The file associated with this check is [fingerprint.txt](https://github.com/rfc-st/humble/blob/master/additional/fingerprint.txt){:target="_blank"}.
    """
    lines = PATHS["fingerprint_header"].read_text(encoding="utf8").splitlines()
    l_fng_ex = lines[SLICE_INT[0]:]
    l_fng = [line.split(" (")[0] for line in l_fng_ex]
    titled_fng = [item.title() for item in l_fng]
    return l_fng_ex, l_fng, titled_fng

get_first_metrics(adj_url_ln)

Compute key analytics metrics of the analyses performed on a URL.

Related to -a option.

Source code in humble.py

def get_first_metrics(adj_url_ln):
    """Compute key analytics metrics of the analyses performed on a URL.

    Related to `-a` option.
    """
    first_a = min(line[:SLICE_INT[9]] for line in adj_url_ln)
    latest_a = max(line[:SLICE_INT[9]] for line in adj_url_ln)
    date_w = [(line[:SLICE_INT[9]], int(line.strip().split(" ; ")[-1]))
              for line in adj_url_ln]
    best_d, best_w = min(date_w, key=operator.itemgetter(1))
    worst_d, worst_w = max(date_w, key=operator.itemgetter(1))
    return (first_a, latest_a, best_d, best_w, worst_d, worst_w)

get_global_first_metrics(adj_url_ln)

Compute key analytics metrics across all URL analyses.

Related to -a option.

Source code in humble.py

def get_global_first_metrics(adj_url_ln):
    """Compute key analytics metrics across all URL analyses.

    Related to `-a` option.
    """
    split_lines = [line.split(" ; ") for line in adj_url_ln]
    url_lines = {}
    for entry in split_lines:
        url = entry[1]
        url_lines[url] = url_lines.get(url, 0) + 1
    return get_global_metrics(adj_url_ln, url_lines)

get_global_metrics(url_ln, url_lines)

Compute key analytics metrics across all URL analyses.

Related to -a option.

Source code in humble.py

def get_global_metrics(url_ln, url_lines):
    """Compute key analytics metrics across all URL analyses.

    Related to `-a` option.
    """
    first_a = min(line[:SLICE_INT[9]] for line in url_ln)
    latest_a = max(line[:SLICE_INT[9]] for line in url_ln)
    unique_u = len({line.split(" ; ")[1] for line in url_ln})
    most_analyzed_u = max(url_lines, key=url_lines.get)
    most_analyzed_c = url_lines[most_analyzed_u]
    most_analyzed_cu = f"({most_analyzed_c}) {most_analyzed_u}"
    least_analyzed_u = min(url_lines, key=url_lines.get)
    least_analyzed_c = url_lines[least_analyzed_u]
    least_analyzed_cu = f"({least_analyzed_c}) {least_analyzed_u}"
    fields = [-1, 2, 3, 4, 5, 6]
    totals = [get_global_totals(url_ln, field) for field in fields]
    return (first_a, latest_a, unique_u, most_analyzed_cu, least_analyzed_cu,
            *chain.from_iterable(totals))

get_global_totals(url_ln, field)

Compute totals metrics across all URL analyses.

Related to -a option.

Source code in humble.py

def get_global_totals(url_ln, field):
    """Compute totals metrics across all URL analyses.

    Related to `-a` option.
    """
    most_totals = max(url_ln, key=lambda line: int(line.split(" ; ")[field]))
    least_totals = min(url_ln, key=lambda line: int(line.split(" ; ")[field]))
    most_totals_c, most_totals_cu = most_totals.split(" ; ")[1], \
        str(most_totals.split(" ; ")[field]).strip()
    most_totals_p = f"({most_totals_cu}) {most_totals_c}"
    least_totals_c, least_totals_cu = least_totals.split(" ; ")[1], \
        str(least_totals.split(" ; ")[field]).strip()
    least_totals_p = f"({least_totals_cu}) {least_totals_c}"
    return (most_totals_p, least_totals_p)

get_highlights(adj_url_ln)

Compute highlight metrics of analyses performed on a URL.

Related to -a option.

Source code in humble.py

def get_highlights(adj_url_ln):
    """Compute highlight metrics of analyses performed on a URL.

    Related to `-a` option.
    """
    sections_h = SECTION_S[:-1]
    best_lbl = print_detail_l("[best_analysis]", analytics=True)
    worst_lbl = print_detail_l("[worst_analysis]", analytics=True)
    results = []
    for i, field in enumerate(range(2, 7)):
        fns_cond = (min, max) if i else (max, min)
        section_lbl = print_detail_l(sections_h[i], analytics=True)
        best_val = calculate_highlights(adj_url_ln, field, fns_cond[0])
        worst_val = calculate_highlights(adj_url_ln, field, fns_cond[1])
        results.append(f" {section_lbl}\n  {best_lbl}: {best_val}\n"
                       f"  {worst_lbl}: {worst_val}\n")
    return results

get_highlights_metrics(fourth_m)

Print highlight-related metrics details for a URL analysis.

Related to -a option.

Source code in humble.py

def get_highlights_metrics(fourth_m):
    """Print highlight-related metrics details for a URL analysis.

    Related to `-a` option.
    """
    return {"[highlights]": "\n" + "\n".join(fourth_m)}

get_insecure_checks()

Skips security checks for specified HTTP response headers.

Related to -s option.

Source code in humble.py

def get_insecure_checks():
    """Skips security checks for specified HTTP response headers.

    Related to `-s` option.
    """
    lines = PATHS["insecure_header"].read_text(encoding="utf8").splitlines()
    headers_name = set()
    for line in lines[SLICE_INT[2]:]:
        insecure_header = line.split(":")[0]
        headers_name.add(insecure_header.lower())
    return headers_name

get_l10n_content()

Load the localization file for the specified language.

Source code in humble.py

def get_l10n_content():
    """Load the localization file for the specified language."""
    l10n_path = (OS_PATH / HUMBLE_DIRS[1] /
                 (HUMBLE_FILES[4] if args.lang == "es" else HUMBLE_FILES[5]))
    with l10n_path.open(encoding="utf8") as l10n_content:
        return l10n_content.readlines()

get_max_lnlength(section)

Return the length of the longest item in a section.

Source code in humble.py

def get_max_lnlength(section):
    """Return the length of the longest item in a section."""
    sec_val = []
    max_secl = 0
    for i in section:
        sec_txt = get_detail(i)
        sec_val.append(sec_txt)
        max_secl = max(max_secl, len(sec_txt)+1)
    return max_secl

get_month_counts(year, url_ln)

Compute the total of analyses performed on a URL by month.

Related to -a option.

Source code in humble.py

def get_month_counts(year, url_ln):
    """Compute the total of analyses performed on a URL by month.

    Related to `-a` option.
    """
    month_cnts = defaultdict(int)
    for line in url_ln:
        date_str = line[:SLICE_INT[10]]
        line_year, line_month, _ = map(int, date_str.split("/"))
        if line_year == year:
            month_cnts[get_detail(f"[month_{line_month:02d}]")] += 1
    return month_cnts

get_second_metrics(adj_url_ln, index, total_a)

Compute the total of analyses performed on a URL that meet a key metric.

Related to -a option.

Source code in humble.py

def get_second_metrics(adj_url_ln, index, total_a):
    """Compute the total of analyses performed on a URL that meet a key metric.

    Related to `-a` option.
    """
    metric_c = len([line for line in adj_url_ln if int(line.split(" ; ")
                                                       [index])
                    == 0])
    return f"{metric_c / total_a:.0%} ({metric_c}\
{get_detail('[pdf_footer2]', replace=True)} {total_a})"

get_security_metrics(analytics_s, second_m)

Print security metrics for a URL analysis.

Related to -a option.

Source code in humble.py

def get_security_metrics(analytics_s, second_m):
    """Print security metrics for a URL analysis.

    Related to `-a` option.
    """
    return {"[analysis_y]": "",
            "[no_enabled]": f"{analytics_s[0]}{second_m[0]}",
            "[no_missing]": f"{analytics_s[1]}{second_m[1]}",
            "[no_fingerprint]": f"{analytics_s[2]}{second_m[2]}",
            "[no_ins_deprecated]": f"{analytics_s[3]}{second_m[3]}",
            "[no_empty]": f"{analytics_s[4]}{second_m[4]}\n"}

get_skipped_unsupported_headers(cli_headers, insecure_headers, file_skipped)

Validate skipped headers against the list of analyzed security headers.

Return unsupported header names and the list of headers to skip during analysis.

Source code in humble.py

def get_skipped_unsupported_headers(cli_headers, insecure_headers,
                                    file_skipped):
    """Validate skipped headers against the list of analyzed security headers.

    Return unsupported header names and the list of headers to skip during
    analysis.
    """
    cli_list = cli_headers or []
    combined_headers = cli_list + file_skipped
    insecure_set = {header.strip().lower() for header in combined_headers}
    skip_list = [header for header in insecure_set
                 if header in insecure_headers]
    unsupported_headers = list(insecure_set - insecure_headers)
    return unsupported_headers, skip_list

get_third_metrics(adj_url_ln)

Compute metrics related to averages of analyses performed on a URL.

Related to -a option.

Source code in humble.py

def get_third_metrics(adj_url_ln):
    """Compute metrics related to averages of analyses performed on a URL.

    Related to `-a` option.
    """
    fields = [line.strip().split(";") for line in adj_url_ln]
    total_enb, total_miss, total_fng, total_dep, total_ety = \
        [sum(int(f[i]) for f in fields) for i in range(2, 7)]
    num_a = len(adj_url_ln)
    avg_enb, avg_miss, avg_fng, avg_dep, avg_ety = \
        [t // num_a for t in (total_enb, total_miss, total_fng, total_dep,
                              total_ety)]
    return (avg_enb, avg_miss, avg_fng, avg_dep, avg_ety)

get_tmp_file(args, export_date)

Determine the temporary export file path for the current analysis.

Selects between a custom name (via -of) or a generated one based on date, language, and URL; applies the correct extension and resolves the absolute path if an output directory is specified (via -op option).

Related to -o option.

Source code in humble.py

def get_tmp_file(args, export_date):
    """Determine the temporary export file path for the current analysis.

    Selects between a custom name (via `-of`) or a generated one based on date,
    language, and URL; applies the correct extension and resolves the absolute
    path if an output directory is specified (via `-op` option).

    Related to `-o` option.
    """
    file_ext = ".txt" if args.output == "txt" else "t.txt"
    if args.output_file:
        name_part = normalize_output_file(args.output_file)
        tmp_file = f"{name_part}{file_ext}"
    else:
        url = urlparse(URL)
        humble_str = HUMBLE_DESC[1:7]
        lang = "_es" if args.lang else "_en"
        tmp_file = build_tmp_file(export_date, file_ext, lang, humble_str, url)
    if args.output_path:
        tmp_file = (Path(args.output_path) / tmp_file).resolve()
    return tmp_file

get_trend_metrics(fifth_m)

Print trend-related metrics details for a URL analysis.

Source code in humble.py

def get_trend_metrics(fifth_m):
    """Print trend-related metrics details for a URL analysis."""
    if "5" in fifth_m[0]:
        trends_s = get_detail("[t_insufficient]")
        return {"[trends]": "\n" + trends_s}
    return {"[trends]": "\n" + "\n".join(fifth_m) + "\n"}

get_trends(adj_url_ln)

Print trends based on the totals of analyses performed on a URL.

Source code in humble.py

def get_trends(adj_url_ln):
    """Print trends based on the totals of analyses performed on a URL."""
    sections_t = SECTION_S[1:]
    fields_t = [3, 4, 5, 6, 7]
    max_secl = (get_max_lnlength(SECTION_S))-2
    trends = []
    for section, field_idx in zip(sections_t, fields_t, strict=True):
        values = [int(parts[field_idx].strip()) for line in adj_url_ln
                  if len(parts := line.strip().split(";")) > field_idx]
        trends.append(f"{(get_detail(section, replace=True).ljust(max_secl))}\
 {calculate_trends(values)}")
    return trends

get_user_agent(user_agent_id)

Select and validate the User-Agent for a URL analysis.

Source: additional/user-agents.txt. Exit if it is not found; related to -ua option.

Source code in humble.py

def get_user_agent(user_agent_id):
    """Select and validate the User-Agent for a URL analysis.

    Source: `additional/user-agents.txt`. Exit if it is not found; related to
    `-ua` option.
    """
    lines = PATHS["user_agents"].read_text(encoding="utf8").splitlines()
    user_agents = lines[SLICE_INT[1]:]
    if user_agent_id == "0":
        print_user_agents(user_agents)
    for line in user_agents:
        if line.startswith(f"{user_agent_id}.-"):
            return line[4:].strip()
    print_error_detail("[ua_invalid]")
    return None

get_warnings_metrics(additional_m, analytics_w)

Print warning-related metrics details for a URL analysis.

Related to -a option.

Source code in humble.py

def get_warnings_metrics(additional_m, analytics_w):
    """Print warning-related metrics details for a URL analysis.

    Related to `-a` option.
    """
    return {"[averages]": "",
            "[average_warnings]": f"{analytics_w[0]}{additional_m[0]}",
            "[average_warnings_year]": f"{analytics_w[1]}{additional_m[2]}\n"}

grade_analysis(en_cnt, m_cnt, f_cnt, i_cnt, e_cnt)

Grade the analysis based on its results.

Source code in humble.py

def grade_analysis(en_cnt, m_cnt, f_cnt, i_cnt, e_cnt):
    """Grade the analysis based on its results."""
    if en_cnt == 0:
        return "[e_grade]"
    if i_cnt and sum(i_cnt) > 0:
        return "[d_grade]"
    if m_cnt > 0:
        return "[c_grade]"
    if f_cnt > 0:
        return "[b_grade]"
    return "[a_grade]" if e_cnt > 0 else "[perfect_grade]"

header_eligible(header)

Check if an HTTP response header is eligible to be analyzed.

Returns True if the header is present in headers_l (a dictionary storing the lowercased HTTP response headers of the analyzed URL) and not in the list of headers skipped via the -s option; False otherwise.

Source code in humble.py

def header_eligible(header):
    """Check if an HTTP response header is eligible to be analyzed.

    Returns `True` if the header is present in `headers_l` (a dictionary
    storing the lowercased HTTP response headers of the analyzed URL)
    **and not** in the list of headers skipped via the `-s` option;
    `False` otherwise.
    """
    return header in headers_l and header not in skip_set

initialize_pdf(pdf, tmp_filename, ypos, *, export_all=False)

Retrieve literals to apply the appropriate formatting.

Related to -o pdf option.

Source code in humble.py

def initialize_pdf(pdf, tmp_filename, ypos, *, export_all=False):
    """Retrieve literals to apply the appropriate formatting.

    Related to `-o pdf` option.
    """
    pdf_links = (URL_STRING[1], REF_LINKS[2], REF_LINKS[3], URL_LIST[0],
                 REF_LINKS[4])
    pdf_prefixes = {REF_LINKS[2]: REF_LINKS[0], REF_LINKS[3]: REF_LINKS[1]}
    generate_pdf(pdf, tmp_filename, pdf_links, pdf_prefixes, ypos,
                 export_all=export_all)

json_detailed_empty(json_lns)

Print the contents of empty HTTP response headers values.

Related to -o json option.

Source code in humble.py

def json_detailed_empty(json_lns):
    """Print the contents of empty HTTP response headers values.

    Related to `-o json` option.
    """
    desc_key = get_detail("[json_det_empty]", replace=True)
    status_key = get_detail("[json_det_empty_s]", replace=True)
    empty_key = get_detail("[json_det_empty_h]", replace=True)
    lines = [line.strip() for line in json_lns if line.strip()]
    result = {desc_key: lines[0][:-1]}
    if e_cnt == 0:
        result[status_key] = lines[1]
    else:
        result[empty_key] = l_empty
    return result

json_detailed_fng(json_lns, fingerprint_set)

Select the lines to include in the fingerprint section.

Related to -o json option.

Source code in humble.py

def json_detailed_fng(json_lns, fingerprint_set):
    """Select the lines to include in the fingerprint section.

    Related to `-o json` option.
    """
    result, entry, current_header = [], {}, None
    fng_header = get_detail("[json_det_fngheader]", replace=True)
    fng_val = get_detail(JSON_L10N[2], replace=True)
    for line in json_lns:
        new_entry, current_header = json_detailed_fng_process(
            line, fingerprint_set, entry, current_header, fng_header, fng_val)
        if new_entry != entry:
            if entry:
                result.append(entry)
            entry = new_entry
    if entry:
        result.append(entry)
    return result

json_detailed_fng_process(line, fingerprint_set, entry, current_header, fng_header, fng_val)

Format the lines to include in the fingerprint section.

Related to -o json option.

Source code in humble.py

def json_detailed_fng_process(line, fingerprint_set, entry, current_header,
                              fng_header, fng_val):
    """Format the lines to include in the fingerprint section.

    Related to `-o json` option.
    """
    line_s = line.strip()
    for f in fingerprint_set:
        if line_s.startswith(f):
            return {fng_header: f}, f
    if current_header and line_s.startswith(fng_val):
        entry[fng_val] = line_s.split(": ", 1)[1].strip("'\" ")
        return entry, current_header
    return entry, current_header

json_detailed_format(json_lns, *, is_compat=False, is_l10n=False)

Format lines in specific sections.

Related to -o json option.

Source code in humble.py

def json_detailed_format(json_lns, *, is_compat=False, is_l10n=False):
    """Format lines in specific sections.

    Related to `-o json` option.
    """
    l10n_txt = JSON_L10N[1] if is_l10n else JSON_L10N[2]
    header_t = get_detail(JSON_L10N[0], replace=True)
    value_t = get_detail(l10n_txt, replace=True)
    if is_compat:
        value_t = value_t[:-1]
    return json_detailed_format_add(json_lns, header_t, value_t)

json_detailed_format_add(json_lns, header_t, value_t)

Convert raw header lines into a list using the given header.

Related to -o json option.

Source code in humble.py

def json_detailed_format_add(json_lns, header_t, value_t):
    """Convert raw header lines into a list using the given header.

    Related to `-o json` option.
    """
    result = []
    for line in map(str.strip, json_lns):
        if not line:
            continue
        if line.startswith("(*)") or ":" in line:
            key, value = line.split(":", 1)
            result.append({header_t: key.strip(), value_t: value.strip()})
        else:
            result.append({header_t: line, value_t: ""})
    return result

json_detailed_info(json_lns)

Print the contents of basic info.

Related to -o json option.

Source code in humble.py

def json_detailed_info(json_lns):
    """Print the contents of basic info.

    Related to `-o json` option.
    """
    info = {get_detail("[json_gen]", replace=True): BANNER_VERSION}
    for line in json_lns:
        key, value = line.split(":", 1)
        key = key.strip()
        info[key] = value.strip()
    return info

json_detailed_ins(json_lns, insecure_checks)

Select the lines to include in the deprecated/insecure section.

Related to -o json option.

Source code in humble.py

def json_detailed_ins(json_lns, insecure_checks):
    """Select the lines to include in the deprecated/insecure section.

    Related to `-o json` option.
    """
    header_t, detail_t, ref_t = (get_detail(text, replace=True)
                                 for text in (
        "[json_det_inscheck]", "[json_det_details]", JSON_L10N[1]))
    if args.lang:
        insecure_checks = {check.split(": ")[0] + ":"
                           for check in insecure_checks}
    checks_list = []
    json_detailed_ins_checks(checks_list, insecure_checks)
    return json_detailed_ins_process(
        json_lns, checks_list, ref_t, PDF_CONDITIONS[0], header_t, detail_t,
    )

json_detailed_ins_append(line, ref_t, ref_o, entry, header, header_t, detail_t, result, is_header)

Add lines to the deprecated/insecure headers section.

Related to -o json option.

Source code in humble.py

def json_detailed_ins_append(line, ref_t, ref_o, entry, header, header_t,
                             detail_t, result, is_header):
    """Add lines to the deprecated/insecure headers section.

    Related to `-o json` option.
    """
    if is_header:
        if entry:
            result.append(entry)
        header = line
        entry = {header_t: header, detail_t: [], ref_t: []}
    elif header:
        if line.startswith(ref_o):
            entry[ref_t].append(line[len(ref_o):].strip())
        else:
            entry[detail_t].append(line)
    return entry, header

json_detailed_ins_checks(checks_list, insecure_checks)

Select the content to include in the deprecated/insecure section.

Related to -o json option.

Source code in humble.py

def json_detailed_ins_checks(checks_list, insecure_checks):
    """Select the content to include in the deprecated/insecure section.

    Related to `-o json` option.
    """
    for check in insecure_checks:
        check_s = check.strip()
        key, val = check_s.split(":", 1)
        checks_list.append((key.strip(), val.strip()))

json_detailed_ins_headers(line, line_s, checks_list, ref_t)

Determine if a line represents a deprecated/insecure header.

Related to -o json option.

Source code in humble.py

def json_detailed_ins_headers(line, line_s, checks_list, ref_t):
    """Determine if a line represents a deprecated/insecure header.

    Related to `-o json` option.
    """
    header_cond = line.startswith("(*)")
    header_cond2 = not line.startswith(ref_t)
    header_cond3 = any(
        (val and key in line and val in line)
        or (not val and line_s.startswith(key))
        for key, val in checks_list
    )
    return header_cond or (header_cond2 and header_cond3)

json_detailed_ins_process(json_lns, checks_list, ref_t, ref_o, header_t, detail_t)

Format the lines to include in the deprecated/insecure section.

Related to -o json option.

Source code in humble.py

def json_detailed_ins_process(json_lns, checks_list, ref_t, ref_o, header_t,
                              detail_t):
    """Format the lines to include in the deprecated/insecure section.

    Related to `-o json` option.
    """
    result, entry, header = [], {}, None
    for line in json_lns:
        if line := line.strip():
            is_header = json_detailed_ins_headers(line, line, checks_list,
                                                  ref_t)
            entry, header = json_detailed_ins_append(
                line, ref_t, ref_o, entry, header, header_t,
                detail_t, result, is_header,
            )
    if entry:
        result.append(entry)
    return result

json_detailed_miss(json_lns, l_miss, json_miss_h, json_miss_d, json_miss_r)

Select the lines to include in the missing section.

Related to -o json option.

Source code in humble.py

def json_detailed_miss(json_lns, l_miss, json_miss_h, json_miss_d,
                       json_miss_r):
    """Select the lines to include in the missing section.

    Related to `-o json` option.
    """
    json_det_mref = PDF_CONDITIONS[0]
    json_miss_keys = (json_miss_h, json_miss_d, json_miss_r)
    result = json_detailed_miss_add(
        json_lns, l_miss, json_miss_keys, json_det_mref,
    )
    for e in result:
        if len(e[json_miss_d]) == 1:
            e[json_miss_d] = e[json_miss_d][0]
    return result

json_detailed_miss_add(json_lns, l_miss, json_miss_keys, json_det_mref)

Add lines to the missing section.

Related to -o json option.

Source code in humble.py

def json_detailed_miss_add(json_lns, l_miss, json_miss_keys, json_det_mref):
    """Add lines to the missing section.

    Related to `-o json` option.
    """
    result, entry = [], {}
    for line in json_lns:
        if line := line.strip():
            entry = json_detailed_miss_process(
                line, l_miss, json_miss_keys, json_det_mref, result, entry,
            )
    if entry:
        result.append(entry)
    return result

json_detailed_miss_process(line, l_miss, json_miss_keys, json_det_mref, result, entry)

Format lines in the missing section.

Related to -o json option.

Source code in humble.py

def json_detailed_miss_process(line, l_miss, json_miss_keys, json_det_mref,
                               result, entry):
    """Format lines in the missing section.

    Related to `-o json` option.
    """
    json_miss_h, json_miss_d, json_miss_r = json_miss_keys
    if line in l_miss or line.startswith("(*)"):
        result.extend(filter(None, (entry,)))
        return {json_miss_h: line, json_miss_d: [], json_miss_r: []}
    if entry and line.startswith(json_det_mref):
        entry[json_miss_r].append(line.removeprefix(json_det_mref).strip())
    elif entry:
        entry[json_miss_d].append(line)
    return entry

json_detailed_parse(data, txt_sections)

Parse sections for a JSON export, related to -o json option.

Source code in humble.py

def json_detailed_parse(data, txt_sections):
    """Parse sections for a JSON export, related to `-o json` option."""
    params = [JSON_L10N[0], "[json_det_details]", JSON_L10N[1]]
    details = [get_detail(p, replace=True) for p in params]
    for i in range(0, len(txt_sections), 2):
        section = f"[{txt_sections[i]}]"
        lines = [line.strip() for line in txt_sections[i + 1].split("\n")
                 if line.strip()]
        data[section] = json_detailed_write(
            lines, section, *details,
        )

json_detailed_response(json_lns)

Print the contents of HTTP response headers.

Related to -o json option.

Source code in humble.py

def json_detailed_response(json_lns):
    """Print the contents of HTTP response headers.

    Related to `-o json` option.
    """
    header_key = get_detail(JSON_L10N[0], replace=True)
    value_key = get_detail(JSON_L10N[2], replace=True)
    result = []
    for line in json_lns:
        line_strip = line.strip()
        if not line_strip or ":" not in line_strip:
            continue
        header, value = line_strip.split(":", 1)
        result.append({
            header_key: header.strip(),
            value_key: value.strip(),
        })
    return result

json_detailed_results(json_lns)

Add detailed content to the sections.

Related to -o json option.

Source code in humble.py

def json_detailed_results(json_lns):
    """Add detailed content to the sections.

    Related to `-o json` option.
    """
    result = {}
    duration_t = get_detail("[analysis_time]", replace=True)
    duration_key = get_detail("[json_det_analysis]", replace=True)
    for line in json_lns:
        if line.startswith(duration_t.strip()):
            result[duration_key] = line
        elif ":" in line:
            key, value = line.split(":", 1)
            result[key.strip()] = value.strip()
    return result

json_detailed_sources(file_idx, slice_idx)

Read source file contents from the /additional path for a JSON export.

Extracts a sliced subset of lines for use in fingerprint and deprecated/insecure header sections; related to -o json option.

Source code in humble.py

def json_detailed_sources(file_idx, slice_idx):
    """Read source file contents from the `/additional` path for a JSON export.

    Extracts a sliced subset of lines for use in fingerprint and
    deprecated/insecure header sections; related to `-o json` option.
    """
    file_path = OS_PATH / HUMBLE_DIRS[0] / HUMBLE_FILES[file_idx]
    with Path(file_path).open(encoding="utf8") as json_file:
        return {line.strip() for line in islice(json_file, SLICE_INT[slice_idx],
                                                None) if line.strip()}

json_detailed_write(json_lns, json_section, json_miss_h, json_miss_d, json_miss_r)

Write sections for a JSON export, related to -o json option.

Source code in humble.py

def json_detailed_write(json_lns, json_section, json_miss_h, json_miss_d,
                        json_miss_r):
    """Write sections for a JSON export, related to `-o json` option."""
    match json_section:
        case s if s.startswith(STRINGS_BOLD[0]):
            return json_detailed_info(json_lns)
        case s if any(s.startswith(p) for p in (STRINGS_BOLD[1],
                                                STRINGS_BOLD[9])):
            return json_detailed_response(json_lns)
        case s if s.startswith(STRINGS_BOLD[2]):
            return json_detailed_format(json_lns)
        case s if s.startswith(STRINGS_BOLD[3]):
            return json_detailed_miss(json_lns, l_miss, json_miss_h,
                                      json_miss_d, json_miss_r)
        case s if s.startswith(STRINGS_BOLD[4]):
            return json_detailed_fng(json_lns, json_detailed_sources(2, 0))
        case s if s.startswith(STRINGS_BOLD[5]):
            return json_detailed_ins(json_lns, json_detailed_sources(7, 2))
        case s if s.startswith(STRINGS_BOLD[6]):
            return json_detailed_empty(json_lns)
        case s if s.startswith(STRINGS_BOLD[7]):
            return json_detailed_format(json_lns, is_compat=True, is_l10n=True)
        case s if s.startswith(STRINGS_BOLD[8]):
            return json_detailed_results(json_lns)
    return list(json_lns)

make_http_request(custom_headers, proxy)

Make the request to the provided URL, disabling certain checks.

Note

I have disabled the following checks to allow the analysis of URLs in certain cases (e.g., development environments, hosts with outdated servers/software or self-signed certificates) and because they exceed the scope of this tool:

• Certificate Verification
• Hostname Verification
• Certificate Requirement

If -df option is provided (args.redirects) the exact URL will be analyzed; otherwise the last redirected URL will be analyzed.

Source code in humble.py

def make_http_request(custom_headers, proxy):  # sourcery skip: extract-method
    """Make the request to the provided URL, disabling certain checks.

    ??? note
        I have disabled the following checks to allow the analysis of URLs in
        certain cases (e.g., development environments, hosts with outdated
        servers/software or self-signed certificates) and because they exceed
        the scope of this tool:

        - Certificate Verification
        - Hostname Verification
        - Certificate Requirement

        If `-df` option is provided (`args.redirects`) the exact URL will
        be analyzed; otherwise the last redirected URL will be analyzed.
    """
    try:
        session = requests.Session()
        session.mount("https://", SSLContextAdapter())
        session.mount("http://", HTTPAdapter())
        r = session.get(
            URL,
            allow_redirects=not args.redirects,
            verify=False,
            headers=custom_headers,
            timeout=REQ_TIMEOUT,
            proxies=proxy,
        )
    except requests.exceptions.Timeout as timeout_err:
        return None, None, timeout_err
    except requests.exceptions.SSLError:
        return None, None, None
    except requests.exceptions.RequestException as request_err:
        return None, None, request_err
    except Exception as unexpected_err: # noqa: BLE001
        return None, None, unexpected_err
    else:
        return r, None, None

match_url_lines(all_analysis)

Return history lines whose URL field exactly matches URL.

Splitting on " ; " and comparing the URL field (index 1) avoids substring collisions (e.g. example.com vs example.com.evil.com); related to -a option and to the analysis history file.

Source code in humble.py

def match_url_lines(all_analysis):
    """Return history lines whose URL field exactly matches `URL`.

    Splitting on `" ; "` and comparing the URL field (index 1) avoids
    substring collisions (e.g. `example.com` vs `example.com.evil.com`);
    related to `-a` option and to the analysis history file.
    """
    return [line for line in all_analysis
            if line.split(" ; ")[1:2] == [URL]]

normalize_htmlpdf_all_export(export_format, tmp_filename, final_filename=None)

Normalize section and line formatting for HTML and PDF exports.

Processes the analysis output and delegates to the appropriate export function; related to -o all option.

Source code in humble.py

def normalize_htmlpdf_all_export(export_format, tmp_filename,
                                 final_filename=None):
    """Normalize section and line formatting for HTML and PDF exports.

    Processes the analysis output and delegates to the appropriate export
    function; related to `-o all` option.
    """
    is_html = (export_format == "html")
    path = Path(tmp_filename)
    lines = path.read_text(encoding="utf-8").splitlines(keepends=True)
    idx = next((i for i, ln in enumerate(lines) if INFO_SECTION in ln), None)
    processed = process_htmlpdf_all_export(lines, idx, export_format, is_html)
    path.write_text("".join(processed), encoding="utf-8")
    if is_html:
        export_html_file(final_filename, tmp_filename, export_all=True)
    else:
        export_pdf_file(fix_pdf_all_export(tmp_filename), export_all=True)

normalize_output_file(filename)

Normalize the filename by stripping paths and removing extensions.

Aborts execution if the resulting filename is invalid (e.g., '.html').

Source code in humble.py

def normalize_output_file(filename):
    """Normalize the filename by stripping paths and removing extensions.

    Aborts execution if the resulting filename is invalid (e.g., '.html').
    """
    base_name = Path(filename).name
    while base_name.lower().endswith(EXPORT_EXTENSIONS):
        dot_index = base_name.rfind(".")
        if dot_index <= 0:
            break
        base_name = base_name[:dot_index]
    if not base_name or base_name.startswith("."):
        delete_lines()
        print_error_detail("[export_filename_error]")
    return base_name

normalize_txt_all_export(tmp_filename)

Apply format to section and lines to the TXT file.

Related to -o all option.

Source code in humble.py

def normalize_txt_all_export(tmp_filename):
    """Apply format to section and lines to the TXT file.

    Related to `-o all` option.
    """
    txt_path = Path(tmp_filename)
    identity = txt_path.stem[:-1]
    txt_content = (txt_path.read_text(encoding="utf-8")
                   .replace(STYLE[6], "").replace(STYLE[8], "")
                   .replace(f"{identity}.pdf", f"{identity}.txt"))
    txt_path.write_text(txt_content, encoding="utf-8")
    txt_path.rename(txt_path.with_name(f"{identity}.txt"))

nourl_user_agent(user_agent_id)

Display available User-Agents if the ID is 0.

Otherwise, display an error indicating that a URL is required for the provided User-Agent ID; related to -ua option.

Source code in humble.py

def nourl_user_agent(user_agent_id):
    """Display available User-Agents if the ID is `0`.

    Otherwise, display an error indicating that a URL is required for the
    provided User-Agent ID; related to `-ua` option.
    """
    if user_agent_id == "0":
        return get_user_agent("0")
    print_error_detail("[args_useragent]")
    return None

parse_cicd_lines(line, pattern, cicd_total_t, cicd_diff_t)

Process lines associated with analysis designed for CI/CD.

Related to -cicd option.

Source code in humble.py

def parse_cicd_lines(line, pattern, cicd_total_t, cicd_diff_t):
    """Process lines associated with analysis designed for CI/CD.

    Related to `-cicd` option.
    """
    if match := pattern.match(line):
        key = match[1].strip()
        cicd_total_v = int(match[2])
        cicd_diff_v = match[3].strip()
        return key, {cicd_total_t: cicd_total_v, cicd_diff_t: cicd_diff_v}
    return None

parse_cicd_sections(cicd_diff_t, cicd_total_t, lines)

Parse analysis output into CI/CD info and totals sections.

Extracts metadata and scoring lines from the analysis, computes totals, and appends the grade detail; related to --cicd option.

Source code in humble.py

def parse_cicd_sections(cicd_diff_t, cicd_total_t, lines):
    """Parse analysis output into CI/CD info and totals sections.

    Extracts metadata and scoring lines from the analysis, computes totals,
    and appends the grade detail; related to `--cicd` option.
    """
    cicd_info_start = lines.index(next(line for line in lines if
                                       STRINGS_BOLD[0] in line))
    cicd_info_lines = lines[cicd_info_start + 1:cicd_info_start + 4]
    cicd_totals_start = lines.index(next(line for line in lines if
                                         STRINGS_BOLD[8] in line))
    cicd_totals_lines = lines[cicd_totals_start + 2:-3]
    cicdi_grade_lines = lines[cicd_totals_start + 8]
    line_pattern = re.compile(RE_PATTERN[21])
    cicd_totals_result = parse_cicd_totals(cicd_totals_lines, cicd_total_t,
                                           cicd_diff_t, line_pattern)
    cicd_totals_result[get_detail("[cicd_grade]", replace=True)] = (
        {get_detail("[cicd_grade_note]", replace=True):
         cicdi_grade_lines.split(":", 1)[1].strip()}
    )
    return cicd_info_lines, cicd_totals_result

parse_cicd_totals(cicd_totals_lines, cicd_total_t, cicd_diff_t, pattern)

Print the total of findings per section.

Along with the differences between the current analysis and the last one performed against the URL; designed for CI/CD.

Related to -cicd option.

Source code in humble.py

def parse_cicd_totals(cicd_totals_lines, cicd_total_t, cicd_diff_t, pattern):
    """Print the total of findings per section.

    Along with the differences between the current analysis and the last one
    performed against the URL; designed for CI/CD.

    Related to `-cicd` option.
    """
    return {
        k: v for line in cicd_totals_lines
        if (processed := parse_cicd_lines(line, pattern, cicd_total_t,
                                          cicd_diff_t))
        for k, v in [processed]}

parse_csv(csv_section, csv_source, csv_writer)

Extract and write data for matching section items for a CSV export.

Related to -o csv option.

Source code in humble.py

def parse_csv(csv_section, csv_source, csv_writer):
    """Extract and write data for matching section items for a CSV export.

    Related to `-o csv` option.
    """
    for i in (item for item in csv_section if item in csv_source):
        csv_content = csv_source.split(i)[1].split("[")[0]
        info_list = [line.strip() for line in csv_content.split("\n") if
                     line.strip()]
        for csv_ln in info_list:
            clean_ln = ": ".join([part.strip() for part in csv_ln.split(":",
                                                                        1)])
            csv_writer.writerow([i, clean_ln])

parse_har_file(file_path)

Parse a HAR file.

Source code in humble.py

def parse_har_file(file_path):
    """Parse a HAR file."""
    try:
        with file_path.open(encoding="utf8") as har_file:
            har_data = load(har_file)
    except (ValueError, KeyError, AttributeError, TypeError):
        print_error_detail("[args_malformedhar]")
    return extract_har_content(har_data)

parse_input_file(input_headers, input_source, status_code)

Parse the headers and values.

Related to -if option.

Source code in humble.py

def parse_input_file(input_headers, input_source, status_code):
    """Parse the headers and values.

    Related to `-if` option.
    """
    parts = input_source.readline().strip().split()
    if len(parts) == LENGTH_BOUNDS[5] and parts[-1].isdigit():
        status_code = int(parts[-1])
    for line in input_source:
        line_strip = line.strip()
        if ":" in line:
            input_header, input_value = line_strip.split(":", 1)
            input_headers[input_header.title()] = input_value.strip()
    if not input_headers:
        print_error_detail("[args_inputlines]")
    return input_headers, status_code

parse_json(sections, txt_sections)

Parse sections for a JSON export; related to -o json -b options.

Source code in humble.py

def parse_json(sections, txt_sections):
    """Parse sections for a JSON export; related to `-o json -b` options."""
    data = {}
    for i in range(0, len(txt_sections), 2):
        json_section = f"[{txt_sections[i]}]"
        json_lns = [line.strip() for line in txt_sections[i + 1].split("\n")
                    if line.strip()]
        data[json_section] = write_json(json_lns, json_section, sections)
    return data

parse_request_headers(request_headers)

Add the provided headers to the request.

Exit if any of them are not well-formed; related to -H option.

Source code in humble.py

def parse_request_headers(request_headers):
    """Add the provided headers to the request.

    Exit if any of them are not well-formed; related to `-H` option.
    """
    headers, malformed_headers = process_request_headers(request_headers)
    if malformed_headers:
        delete_lines()
        print()
        quoted = ", ".join(f'"{h}"' for h in malformed_headers)
        print(f"{get_detail('[e_custom_headers]', replace=True)}: {quoted}")
        sys.exit(1)
    return headers

parse_user_agent(*, user_agent=False)

Select and validate the provided user agent, related to -ua option.

Source code in humble.py

def parse_user_agent(*, user_agent=False):
    """Select and validate the provided user agent, related to `-ua` option."""
    if not user_agent:
        return get_user_agent("1")
    user_agent_id = sys.argv[sys.argv.index("-ua") + 1].lstrip("-ua")
    if not URL:
        nourl_user_agent(user_agent_id)
    else:
        return get_user_agent(user_agent_id)
    return None

parse_xml(root, section, stripped_txt)

Parse sections of an XML export; related to -o xml option.

Source code in humble.py

def parse_xml(root, section, stripped_txt):
    """Parse sections of an XML export; related to `-o xml` option."""
    for line in stripped_txt:
        if not line:
            continue
        if line.startswith("["):
            section = ET.SubElement(root, "section", {"name": line})
            continue
        if section is None:
            continue
        add_xml_item(line, section)
    return section

permissions_analyze_content(perm_header, i_cnt)

Permissions-Policy header analysis.

Source code in humble.py

def permissions_analyze_content(perm_header, i_cnt):
    """`Permissions-Policy` header analysis."""
    if any(value in perm_header for value in t_per_dep):
        permissions_print_deprecated(perm_header)
    if "none" in perm_header:
        print_details("[ifpoli_h]", "[ifpoli]", "d", i_cnt)
    if perm_broad_dirs := permissions_check_broad(perm_header):
        permissions_print_broad(perm_broad_dirs, i_cnt)

permissions_check_broad(perm_header)

Permissions-Policy header check related to broad values.

Source code in humble.py

def permissions_check_broad(perm_header):
    """`Permissions-Policy` header check related to broad values."""
    if sum(
        directive in perm_header for directive in t_per_ft
    ) < HEADERS_CHECKS:
        return None
    try:
        result = []
        for directive in perm_header.split(","):
            if "=" in directive:
                feature, value = directive.split("=")
                feature = feature.strip()
                value = value.strip()
                if any(broad in value for broad in t_per_broad):
                    result.append(feature)
    except (IndexError, ValueError):
        print_details("[ifpolf_h]", "[ifpolf]", "d", i_cnt)
        return None
    else:
        return result or None

permissions_print_broad(perm_broad_dirs, i_cnt)

Print the broad values in the Permissions-Policy header.

Source code in humble.py

def permissions_print_broad(perm_broad_dirs, i_cnt):
    """Print the broad values in the `Permissions-Policy` header."""
    print_detail_r("[ifpol_h]", is_red=True)
    if not args.brief:
        print_detail_l(DIR_MSG[0] if len(perm_broad_dirs) > 1 else DIR_MSG[1])
        print(" " + ", ".join(f"'{directive}'" for directive in
                              sorted(perm_broad_dirs)) + ".")
        print_detail("[ifpol]", num_lines=2)
    i_cnt[0] += 1

permissions_print_deprecated(perm_header)

Print deprecated directives in the Permissions-Policy header.

Source code in humble.py

def permissions_print_deprecated(perm_header):
    """Print deprecated directives in the `Permissions-Policy` header."""
    print_detail_r("[ifpold_h]", is_red=True)
    if not args.brief:
        matches_perm = [x for x in t_per_dep if x in perm_header]
        print_detail_l("[ifpold_h_s]")
        print(", ".join(f"'{x}'" for x in matches_perm))
        print_detail("[ifpold]")
    i_cnt[0] += 1

print_analysis_results(totals, max_secl, en_cnt_w)

Print the totals for the current analysis.

Source code in humble.py

def print_analysis_results(totals, max_secl, en_cnt_w):
    """Print the totals for the current analysis."""
    for idx, (literal, total) in enumerate(zip(SECTION_S, totals, strict=True)):
        print(f"{print_detail_s(literal, max_ln=True):<{max_secl}} {total}",
              end="")
        if idx == 0 and en_cnt_w:
            val1 = print_detail_s("[enabled_cnt_w]", max_ln=True)
            val2 = get_detail("[enabled_cnt_wt]")
            print(f"{val1:<{max_secl}} {val2}", end="")

print_basic_info(export_filename)

Print basic analysis details.

Date, time, URL, User-Agent (-ua option), input file (-if option) and exported filename (-o option).

Source code in humble.py

def print_basic_info(export_filename):
    """Print basic analysis details.

    Date, time, URL, User-Agent (`-ua` option), input file (`-if` option) and
    exported filename (`-o` option).
    """
    print(end="\n\n" if args.output in ("html", "pdf", None) else "")
    print_detail_r("[0section]")
    print_detail_l("[analysis_date]")
    print(f" {current_time}")
    print(f"{URL_STRING[1]}{URL}")
    if args.user_agent not in (None, "", "0"):
        print(f"{get_detail('[ua_custom]', replace=True)} '{args.user_agent}'"
              f"{get_detail('[ua_custom2]', replace=True)}")
    if args.input_file:
        print(f"{get_detail('[input_filename]', replace=True)} \
{args.input_file}")
    if export_filename:
        print(f"{get_detail('[export_filename]', replace=True)} \
{export_filename}")
    validate_file_access(VALIDATE_FILE, context="basic")

print_browser_compatibility(compat_headers)

Print links for browser compatibility of enabled HTTP headers.

Note

References provided by Can I use.

Source code in humble.py

def print_browser_compatibility(compat_headers):
    """Print links for browser compatibility of enabled HTTP headers.

    ??? note
        References provided by [Can I use](https://caniuse.com/){:target="_blank"}.
    """
    style_blanks = "  " if args.output == "html" else " "
    for key in compat_headers:
        styled_header = key if args.output else f"{STYLE[2]}{key}{STYLE[5]}"
        csp_key = "contentsecuritypolicy2" if key == "Content-Security-Policy"\
            else key
        print(f"{style_blanks}{styled_header}{URL_LIST[0]}{csp_key}")

print_cicd_totals(tmp_filename, threshold_grade=None)

Print a JSON-formatted CI/CD summary and exits.

If threshold_grade is provided, appends a 'Security Gate' field and exits if the analysis grade does not reach that threshold; equal grades pass.

Source code in humble.py

def print_cicd_totals(tmp_filename, threshold_grade=None):
    """Print a JSON-formatted CI/CD summary and exits.

    If `threshold_grade` is provided, appends a 'Security Gate' field and exits
    if the analysis grade does not reach that threshold; equal grades pass.
    """
    try:
        cicd_labels = get_cicd_labels() # sourcery skip: extract-method
        with Path(tmp_filename).open(encoding="utf8") as f:
            lines = [ln.strip() for ln in f if ln.strip()]
        info_lines, totals = parse_cicd_sections(cicd_labels[1], cicd_labels[0],
                                                  lines)
        threshold, failed = (
            threshold_cicd(threshold_grade, totals)
            if threshold_grade else (None, False)
        )
        cicd_output = build_cicd_totals(tmp_filename, info_lines, totals,
                                        cicd_labels, threshold=threshold)
        print(dumps(cicd_output, indent=2, ensure_ascii=False))
        sys.exit(int(failed))
    except Exception as exc:  # noqa: BLE001
        err_key = get_detail("[cicd_error]", replace=True)
        print(dumps({err_key: str(exc)}, ensure_ascii=False))
        sys.exit(1)

print_detail(id_mode, num_lines=1)

Print detailed information about the finding across multiple lines.

Source code in humble.py

def print_detail(id_mode, num_lines=1):
    """Print detailed information about the finding across multiple lines."""
    idx = l10n_main.index(id_mode + "\n")
    print(l10n_main[idx+1], end="")
    for i in range(1, num_lines+1):
        if idx+i+1 < len(l10n_main):
            print(l10n_main[idx+i+1], end="")

print_detail_l(id_mode, *, analytics=False, no_headers=False)

Print detailed information about the finding.

Removing lines from the output based on it.

Note

pairwise is used to match each bracketed ID with its corresponding descriptive text on the following line (bridging the line break) from l10n_main, which contains the multilingual strings and descriptions used for the final report.

Source code in humble.py

def print_detail_l(id_mode, *, analytics=False, no_headers=False):
    """Print detailed information about the finding.

    Removing lines from the output based on it.

    ??? note
        `pairwise` is used to match each bracketed ID with its corresponding
        descriptive text on the following line (bridging the line break) from
        `l10n_main`, which contains the multilingual strings and descriptions
        used for the final report.
    """
    for idmode_ln, idnext_ln in pairwise(l10n_main):
        if idmode_ln.startswith(id_mode):
            if no_headers:
                print(idnext_ln, end="")
            elif not analytics:
                print(idnext_ln.replace("\n", ""), end="")
            else:
                return idnext_ln.replace("\n", "").replace(":", "")[1:]
    return None

print_detail_r(id_mode, *, is_red=False)

Print detailed information about the finding using a distinctive format.

Note

pairwise is used to match each bracketed ID with its corresponding descriptive text on the following line (bridging the line break) from l10n_main, which contains the multilingual strings and descriptions used for the final report.

Source code in humble.py

def print_detail_r(id_mode, *, is_red=False):
    """Print detailed information about the finding using a distinctive format.

    ??? note
        `pairwise` is used to match each bracketed ID with its corresponding
        descriptive text on the following line (bridging the line break) from
        `l10n_main`, which contains the multilingual strings and descriptions
        used for the final report.
    """
    style_str = STYLE[1] if is_red else STYLE[0]
    for idmode_ln, idnext_ln in pairwise(l10n_main):
        if idmode_ln.startswith(id_mode):
            if not args.output:
                print(f"{style_str}{idnext_ln}", end="")
            else:
                print(idnext_ln, end="")
            if not is_red:
                print()

print_detail_s(id_mode, *, max_ln=False)

Print message with leading newline and optional whitespace preservation.

Note

pairwise is used to match each bracketed ID with its corresponding descriptive text on the following line (bridging the line break) from l10n_main, which contains the multilingual strings and descriptions used for the final report.

Source code in humble.py

def print_detail_s(id_mode, *, max_ln=False):
    """Print message with leading newline and optional whitespace preservation.

    ??? note
        `pairwise` is used to match each bracketed ID with its corresponding
        descriptive text on the following line (bridging the line break) from
        `l10n_main`, which contains the multilingual strings and descriptions
        used for the final report.
    """
    if match := next(
        (pr for pr in pairwise(l10n_main) if pr[0].startswith(id_mode)),
        None,
    ):
        _, idnext_ln = match
        return (
            f"\n{idnext_ln.rstrip()}" if max_ln else f"\n{idnext_ln.strip()}"
        )
    return None

print_details(short_d, long_d, id_mode, i_cnt)

Print detailed information about the finding.

Source code in humble.py

def print_details(short_d, long_d, id_mode, i_cnt):
    """Print detailed information about the finding."""
    print_detail_r(short_d, is_red=True)
    if not args.brief:
        print_detail(long_d, 2) if id_mode == "d" else print_detail(long_d, 3)
    i_cnt[0] += 1
    return i_cnt

print_empty_headers(headers, l_empty)

Print the contents of the section with empty HTTP response headers.

Source code in humble.py

def print_empty_headers(headers, l_empty):
    """Print the contents of the section with empty HTTP response headers."""
    e_cnt = 0
    for key in sorted(headers):
        if not headers[key]:
            l_empty.append(key)
            print_header(key.title())
            e_cnt += 1
    return e_cnt

print_enabled_headers(args, exp_s, header, headers_d)

Print enabled HTTP response headers.

Source: additional/security.txt.

Source code in humble.py

def print_enabled_headers(args, exp_s, header, headers_d):
    """Print enabled HTTP response headers.

    Source: `additional/security.txt`.
    """
    prefix = STYLE[8] if args.output in ("html", "pdf") else ""
    header_display = f"{prefix}{exp_s}{header}"
    if not args.output:
        header_display = f"{STYLE[7]}{header_display}{STYLE[5]}"[18:]
    output_str = f" {header_display}" if args.brief else f" {header_display}: \
{headers_d[header]}"
    print(output_str)

print_error_detail(id_mode, *, clean_lines=False)

Print an error message and exit, optionally clearing previous output.

Source code in humble.py

def print_error_detail(id_mode, *, clean_lines=False):
    """Print an error message and exit, optionally clearing previous output."""
    if clean_lines:
        delete_lines()
    print(f"\n{get_detail(id_mode, replace=True)}")
    sys.exit(1)

print_export_path(filename, reliable, *, export_all=False)

Print the export path.

Displays the file path for single reports (e.g. 'html') or the directory for bulk exports (e.g. 'all'); related to -o option.

Source code in humble.py

def print_export_path(filename, reliable, *, export_all=False):
    """Print the export path.

    Displays the file path for single reports (e.g. 'html') or the directory
    for bulk exports (e.g. 'all'); related to `-o` option.
    """
    delete_lines(reliable=False) if reliable else delete_lines()
    if "-c" in sys.argv:
        return
    export_path = Path(filename).resolve()
    if export_all:
        all_reports = print_detail_s("[all_reports]").lstrip()
        msg = f"{all_reports} '{export_path.parent}'."
    else:
        single_report = print_detail_s("[report]").lstrip()
        msg = f"{single_report} '{export_path}'."
    print(f"\n {msg}")

print_extended_info(args, reliable, status_code, headers_skipped, skip_set)

Print extended analysis details.

Request (-H option) and skipped (-s option) headers, proxy usage (-p option) and specific HTTP 4xx errors.

Source code in humble.py

def print_extended_info(args, reliable, status_code, headers_skipped, skip_set):
    """Print extended analysis details.

    Request (`-H` option) and skipped (`-s` option) headers, proxy usage
    (`-p` option) and specific HTTP 4xx errors.
    """
    if args.request_header:
        print_request_headers(added_request_headers)
    if headers_skipped:
        print_skipped_headers(skip_set)
    if args.proxy:
        print_detail_l("[proxy_analysis_note]")
        print(f" {args.proxy}")
    if (
        (status_code is not None and
         ERROR_CODES_MIXED[0] <= status_code <= ERROR_CODES_MIXED[1]) or
        reliable or args.redirects or args.skip_headers
    ):
        print_extra_info(reliable)

print_extra_info(reliable)

Print supplementary analysis details.

Specific 4xx errors, reliability warnings and redirects (-df option).

Source code in humble.py

def print_extra_info(reliable):
    """Print supplementary analysis details.

    Specific 4xx errors, reliability warnings and redirects (`-df` option).
    """
    if status_code in ERROR_CODES_CLIENT:
        id_mode = f"[http_{status_code}]"
        print_detail(id_mode, 0)
        print(f"{URL_LIST[2]}{status_code}")
    elif (
        status_code is not None and
        ERROR_CODES_MIXED[0] <= status_code <= ERROR_CODES_MIXED[1]
    ):
        print(f"{get_detail('[http_4xx]', replace=True)} {status_code})")
    if reliable:
        print(get_detail("[unreliable_analysis_note]", replace=True))
    if args.redirects:
        print(get_detail("[analysis_redirects_note]", replace=True))

print_fingerprint_headers(headers_l, l_fng_ex, titled_fng)

Identify and print fingerprint headers.

Source code in humble.py

def print_fingerprint_headers(headers_l, l_fng_ex, titled_fng):
    """Identify and print fingerprint headers."""
    f_cnt = 0
    sorted_headers = sorted({header.title() for header in headers_l})
    for header in sorted_headers:
        if header in titled_fng:
            idx_fng = titled_fng.index(header)
            get_fingerprint_detail(header, headers, idx_fng, l_fng_ex, args)
            f_cnt += 1
    return f_cnt

print_fng_header(header)

Print the header name in the section with fingerprint headers.

Source code in humble.py

def print_fng_header(header):
    """Print the header name in the section with fingerprint headers."""
    if args.output:
        print(f" {header}")
    else:
        print(f"{STYLE[1]} {header}")

print_general_info(reliable, export_filename, headers_skipped, skip_set)

Print the content in the section with basic information.

Source code in humble.py

def print_general_info(reliable, export_filename, headers_skipped, skip_set):
    """Print the content in the section with basic information."""
    if not args.output:
        delete_lines(reliable=False) if reliable else delete_lines()
        print(f"\n{BANNER}\n ({BANNER_VERSION})")
    elif args.output != "pdf":
        humble_desc = get_detail("[humble_desc]", replace=True)
        print(f"\n\n{humble_desc}\n{BANNER_VERSION}\n")
    print_basic_info(export_filename)
    print_extended_info(args, reliable, status_code, headers_skipped, skip_set)

print_global_metrics(analytics_l, analytics_s, analytics_w, total_a, first_m, second_m, third_m, additional_m)

Print metrics across all URL analyses, related to -a option.

Source code in humble.py

def print_global_metrics(analytics_l, analytics_s, analytics_w,
                         total_a, first_m, second_m, third_m, additional_m):
    """Print metrics across all URL analyses, related to `-a` option."""
    basic_m = get_basic_global_metrics(analytics_l, total_a, first_m)
    error_m = get_security_metrics(analytics_s, second_m)
    warning_m = get_warnings_metrics(additional_m, analytics_w)
    averages_m = get_averages_metrics(analytics_w, third_m)
    analysis_year_m = get_date_metrics(additional_m)
    totals_m = {**basic_m, **error_m, **warning_m, **averages_m,
                **analysis_year_m}
    return {get_detail(key, replace=True): value for key, value in
            totals_m.items()}

print_header(header)

Print the header name.

Source code in humble.py

def print_header(header):
    """Print the header name."""
    print(f" {header}" if args.output else f"{STYLE[1]} {header}")

print_http_exception(exception_id, exception_v)

Print the exception received during analysis.

Source code in humble.py

def print_http_exception(exception_id, exception_v):
    """Print the exception received during analysis."""
    delete_lines()
    print()
    print_detail(exception_id)
    raise SystemExit from exception_v

print_l10n_file(args, l10n_file, *, slice_ln=False)

Print the contents of a file in the specified language and exit.

Source code in humble.py

def print_l10n_file(args, l10n_file, *, slice_ln=False):
    """Print the contents of a file in the specified language and exit."""
    lang_es = args.lang == "es"
    lang_idx = 1 if lang_es else 0
    l10n_file = HUMBLE_FILES[L10N_IDXS[l10n_file][lang_idx]]
    l10n_slice = SLICE_INT[2 if lang_es else 3]
    file_path = OS_PATH / HUMBLE_DIRS[1] / l10n_file
    with file_path.open(encoding="utf8") as l10n_source:
        l10n_lines = islice(l10n_source, l10n_slice, None) if slice_ln else \
            l10n_source
        for line in l10n_lines:
            prefix = f" {STYLE[0]}" if line.startswith("[") else "  "
            print(f"{prefix}{line}", end="")
    sys.exit(0)

print_metrics(analytics_s, analytics_w, total_a, *m_data)

Build the metrics dictionary for the final statistics output of a URL.

Consolidates and formats security, analysis, and trend values with localized headers.

Source code in humble.py

def print_metrics(analytics_s, analytics_w, total_a, *m_data):
    """Build the metrics dictionary for the final statistics output of a URL.

    Consolidates and formats security, analysis, and trend values with localized
    headers.
    """
    basic_m = get_basic_metrics(total_a, m_data[0])
    error_m = get_security_metrics(analytics_s, m_data[1])
    warning_m = get_warnings_metrics(m_data[3], analytics_w)
    averages_m = get_averages_metrics(analytics_w, m_data[2])
    fourth_m = get_highlights_metrics(m_data[4])
    trend_m = get_trend_metrics(m_data[5])
    analysis_year_m = get_date_metrics(m_data[3])
    totals_m = {**basic_m, **error_m, **warning_m, **averages_m, **fourth_m,
                **trend_m, **analysis_year_m}
    return {get_detail(key, replace=True): value for key, value in
            totals_m.items()}

print_missing_headers(args, headers_l, l_detail, l_miss)

Print the contents of the section with missing HTTP Security Headers.

Note

The file associated with this check is missing.txt.

Source code in humble.py

def print_missing_headers(args, headers_l, l_detail, l_miss):
    """Print the contents of the section with missing HTTP Security Headers.

    ??? note
        The file associated with this check is [missing.txt](https://github.com/rfc-st/humble/blob/master/additional/missing.txt){:target="_blank"}.
    """
    m_cnt = 0
    headers_set = set(headers_l)
    l_miss_set = {header.lower() for header in l_miss}
    skip_headers = [h.lower() for h in (args.skip_headers or [])]
    skip_missing = {header for header in skip_headers if header in l_miss_set}
    merged_set = headers_set | skip_missing
    xfo_skipped = "x-frame-options" in skip_missing
    m_cnt = check_missing_headers(m_cnt, l_miss, l_detail, merged_set,
                                  xfo_skipped)
    m_cnt = check_frame_options(args, headers_l, l_miss, m_cnt, skip_headers)
    return m_cnt, skip_missing

print_nosec_headers(*, enabled=True)

Print a message if no security-related HTTP response headers are enabled.

Or if none was received.

Source code in humble.py

def print_nosec_headers(*, enabled=True):
    """Print a message if no security-related HTTP response headers are enabled.

    Or if none was received.
    """
    id_mode = "[no_sec_headers]" if enabled else "[no_enb_headers]"
    if args.output:
        print_detail_l(id_mode, no_headers=True)
    else:
        print_detail_r(id_mode, is_red=True)

print_nowarnings()

Check if no results were found in sections of the analysis.

Source code in humble.py

def print_nowarnings():
    """Check if no results were found in sections of the analysis."""
    if not args.output:
        print(f"{STYLE[10]}{get_detail(DIR_MSG[2])}{STYLE[5]}")
    else:
        print_detail(DIR_MSG[2])

print_owasp_findings(header_dict, header_list)

Print OWASP Secure Headers Project check results.

Related to -c option.

Source code in humble.py

def print_owasp_findings(header_dict, header_list):
    """Print `OWASP Secure Headers Project` check results.

    Related to `-c` option.
    """
    print(end="\n\n")
    print(f"{STYLE[0]}{get_detail('[comp_analysis]')}")
    print(" ", end="")
    print_detail_l("[analysis_date]")
    print(f" {current_time}")
    print(f" {URL_STRING[1]}{URL}")
    print_detail("[comp_ref]", num_lines=2)
    missing_owasp = print_owasp_missing(header_list)
    wrong_owasp = print_owasp_wrong(header_dict)
    if wrong_owasp:
        print_owasp_rec(wrong_owasp, header_dict)
    print_owasp_summary(missing_owasp, wrong_owasp)
    print()
    print_detail("[comp_experimental]", 2)

print_owasp_missing(header_list)

Print OWASP Secure Headers Project missing check results.

Related to -c option.

Source code in humble.py

def print_owasp_missing(header_list):
    """Print `OWASP Secure Headers Project` missing check results.

    Related to `-c` option.
    """
    print(f"\n{STYLE[0]}{get_detail('[comp_rec]')}{STYLE[5]}")
    missing_owasp = [header.title() for header in header_list if header not in
                     headers_l]
    if not missing_owasp:
        print(f"{STYLE[10]}  {get_detail(DIR_MSG[2])}{STYLE[5]}", end="")
        return []
    for header in missing_owasp:
        prefix = "(*) " if header == "Permissions-Policy" else ""
        print(f"{STYLE[1]}  {prefix}{header}{STYLE[5]}")
    return missing_owasp

print_owasp_rec(wrong_owasp, header_dict)

Print OWASP Secure Headers Project recommended values check results.

Related to -c option.

Source code in humble.py

def print_owasp_rec(wrong_owasp, header_dict):
    """Print `OWASP Secure Headers Project` recommended values check results.

    Related to `-c` option.
    """
    print(f"\n\n{STYLE[0]}{get_detail('[comp_rec_val]')}{STYLE[5]}")
    for header, _ in sorted(wrong_owasp):
        prefix = "(*) " if header == "Permissions-Policy" else ""
        if rec_val := header_dict.get(header):
            print(f"{STYLE[10]}  {prefix}{header}{STYLE[4]}: {rec_val}")

print_owasp_summary(missing, wrong)

Format lines for results of OWASP Secure Headers Project checks.

Related to -c option.

Source code in humble.py

def print_owasp_summary(missing, wrong):
    """Format lines for results of `OWASP Secure Headers Project` checks.

    Related to `-c` option.
    """
    missing_txt = get_detail("[comp_missing]", replace=True)
    wrong_txt = get_detail("[comp_noncompliant]", replace=True)
    max_len = len(wrong_txt)
    print(end="\n\n")
    print(f"{STYLE[0]}{get_detail('[comp_summary]')}")
    print(f" {missing_txt:{max_len}} : {len(missing)}")
    print(f" {wrong_txt:{max_len}} : {len(wrong)}")

print_owasp_wrong(header_dict)

Print OWASP Secure Headers Project enabled headers check results.

Related to -c option.

Source code in humble.py

def print_owasp_wrong(header_dict):
    """Print `OWASP Secure Headers Project` enabled headers check results.

    Related to `-c` option.
    """
    wrong_owasp = [
        (header.title(), value)
        for header, value in headers_l.items()
        if (owasp_value := header_dict.get(header.title())) and value !=
        owasp_value
    ]
    print(f"\n\n{STYLE[0]}{get_detail('[comp_val]')}{STYLE[5]}")
    if not wrong_owasp:
        print(f"{STYLE[10]} {get_detail(DIR_MSG[2])}{STYLE[5]}", end="")
        return []
    for header, value in sorted(wrong_owasp):
        prefix = "(*) " if header == "Permissions-Policy" else ""
        print(f"{STYLE[1]}  {prefix}{header}{STYLE[4]}: {value}")
    return wrong_owasp

print_request_headers(added_request_headers)

Print requested HTTP request headers.

Source code in humble.py

def print_request_headers(added_request_headers):
    """Print requested HTTP request headers."""
    print_detail_l("[analysis_request_note]")
    request_headers = ", ".join(
        f"{header!r}: {value!r}"
        for header, value in added_request_headers.items()
    )
    print(f" {request_headers}")

print_response_headers()

Print response headers, related to -r option.

Source code in humble.py

def print_response_headers():
    """Print response headers, related to `-r` option."""
    print(end="\n\n")
    print_detail_r("[0headers]")
    if not headers:
        print_nosec_headers(enabled=False)
        print("\n")
        return
    pdf_style = STYLE[6] if args.output == "pdf" else ""
    for key, value in sorted(headers.items()):
        print(f" {pdf_style}{key}:", value) if args.output else \
            print(f" {STYLE[2]}{key}:", value)
    print("\n")

print_skipped_headers(skip_set)

Print skipped HTTP response headers.

Source code in humble.py

def print_skipped_headers(skip_set):
    """Print skipped HTTP response headers."""
    note = "[analysis_skipped_note]" if len(skip_set) > 1 \
        else "[analysis_skipped_note_single]"
    print_detail_l(note)
    print(" " + ", ".join(f"'{h.title()}'" for h in
                          sorted(skip_set, key=str.lower)) + ".")

print_unsafe_cookies(unsafe_cks)

Print unsafe cookies in the Set-Cookie header.

Source code in humble.py

def print_unsafe_cookies(unsafe_cks):
    """Print unsafe cookies in the `Set-Cookie` header."""
    print_detail_l("[icooks_s]" if len(unsafe_cks) > 1 else "[icook_s]")
    print(", ".join(f"'{ck}'" for ck in sorted(unsafe_cks)) + ".")
    print_detail("[iset]", num_lines=2)

print_unsupported_headers(unsupported_headers)

Print unsupported HTTP response headers.

For those which it has been expressly indicated to skip their security analysis and exit.

Source code in humble.py

def print_unsupported_headers(unsupported_headers):
    # sourcery skip: use-fstring-for-concatenation
    """Print unsupported HTTP response headers.

    For those which it has been expressly indicated to skip their security
    analysis and exit.
    """
    quoted = ", ".join("'" + h + "'" for h in unsupported_headers)
    print(f"\n {get_detail('[args_skipped_unknown]', replace=True)} \
({quoted})")
    sys.exit(1)

print_user_agents(user_agents)

Print available User-Agents and exit.

Related to -ua option.

Source code in humble.py

def print_user_agents(user_agents):
    """Print available User-Agents and exit.

    Related to `-ua` option.
    """
    print(f"\n{STYLE[0]}{get_detail('[ua_available]', replace=True)}\
{STYLE[4]}{get_detail('[ua_source]', replace=True)}\n")
    for line in user_agents:
        print(f" {line}")
    sys.exit(0)

process_htmlpdf_all_export(lines, start_index, export_format, is_html)

Process line formatting and section prefixing for HTML and PDF exports.

Applies state-based formatting to each line; related to -o all option.

Source code in humble.py

def process_htmlpdf_all_export(lines, start_index, export_format, is_html):
    """Process line formatting and section prefixing for HTML and PDF exports.

    Applies state-based formatting to each line; related to `-o all` option.
    """
    content = lines[:start_index]
    states = ExportStates(response=False, enabled=False, browser=False)
    for line in lines[start_index:]:
        prefix, states = sections_htmlpdf_all_export(line, states)
        if prefix:
            content.append(prefix)
        target_state = states.enabled if is_html else states.response
        content.append(format_htmlpdf_all_export(line, export_format,
                                                 target_state,
                                                 states.browser))
    return content

process_http_error(r, exception_d)

Print error messages based on HTTP response code during analysis.

Source code in humble.py

def process_http_error(r, exception_d):
    """Print error messages based on HTTP response code during analysis."""
    if r is None:
        return
    try:
        r.raise_for_status()
    except requests.exceptions.HTTPError as err_http:
        status = err_http.response.status_code
        l10n_id = f"[server_{status}]"
        if ERROR_CODES_MIXED[2] <= status <= ERROR_CODES_MIXED[4]:
            process_server_error(status, l10n_id)
    except Exception as http_err: # noqa: BLE001
        ex = exception_d.get(type(http_err))
        if ex and (not callable(ex) or ex(http_err)):
            print_http_exception(ex, http_err)

process_http_request(status_code, reliable, body, proxy, custom_headers)

Perform an HTTP request using a background thread.

It manages response processing and ensure the target URL is reachable within a set timeout.

The analysis is flagged as unreliable if a response isn't received within the (REQ_TIMEOUT - REQ_WARNING) constants. The function will terminate execution if the request exceeds the REQ_TIMEOUT constant.

Source code in humble.py

def process_http_request(status_code, reliable, body, proxy, custom_headers):
    """Perform an HTTP request using a background thread.

    It manages response processing and ensure the target URL is reachable within
    a set timeout.

    The analysis is flagged as unreliable if a response isn't received within
    the (`REQ_TIMEOUT` - `REQ_WARNING`) constants. The function will terminate
    execution if the request exceeds the `REQ_TIMEOUT` constant.
    """
    result = {}
    done = Event()

    def worker():
        try:
            r, _, exception = make_http_request(custom_headers, proxy)
            result["r"] = r
            result["exception"] = exception
        except Exception as thread_err: # noqa: BLE001
            result["exception"] = thread_err
        finally:
            done.set()

    thread = Thread(target=worker, daemon=True)
    thread.start()
    if not done.wait(timeout=REQ_TIMEOUT - REQ_WARNING):
        print(get_detail("[unreliable_analysis]"))
        reliable = True
    if not done.wait(timeout=REQ_TIMEOUT):
        delete_lines()
        delete_lines()
        print(f"\n{get_detail('[e_timeout]', replace=True)}")
        sys.exit(1)
    r = result.get("r")
    exception = result.get("exception")
    return process_http_response(r, exception, status_code, reliable, body)

process_http_response(r, exception, status_code, reliable, body)

Process an HTTP response and its exceptions.

Storing headers, status code and body, and determining if the analyzed URL returns an HTML document.

Note

References:

• Exceptions in the HTTP library 'requests'
• Generic HTTP 5xx errors
• Cloudflare 5xx HTTP errors
• MDN docs regarding HTML <meta> and content-type

Source code in humble.py

def process_http_response(r, exception, status_code, reliable, body):
    """Process an HTTP response and its exceptions.

    Storing headers, status code and body, and determining if the analyzed URL
    returns an HTML document.

    ??? note
        References:<br>

        - [Exceptions in the HTTP library 'requests'](https://requests.readthedocs.io/en/latest/_modules/requests/exceptions/){:target="_blank"}
        - [Generic HTTP 5xx errors](https://en.wikipedia.org/wiki/List_of_HTTP_status_codes#5xx_server_errors){:target="_blank"}
        - [Cloudflare 5xx HTTP errors](https://developers.cloudflare.com/support/troubleshooting/http-status-codes/cloudflare-5xx-errors/){:target="_blank"}
        - [MDN docs regarding HTML `<meta>` and `content-type`](https://developer.mozilla.org/en-US/docs/Web/HTML/Reference/Elements/meta/http-equiv#content-type){:target="_blank"}
    """
    if exception:
        process_requests_exception(exception)
        return {}, status_code, reliable, body, False
    if r is None:
        return {}, status_code, reliable, body, False
    process_http_error(r, exception_d)
    status_code = r.status_code
    headers = CaseInsensitiveDict({
        k: re.sub(RE_PATTERN[20], " ", v).strip()
        for k, v in r.headers.items()})
    body = r.text
    is_html = headers.get("content-type", "").lower().startswith("text/html")
    return headers, status_code, reliable, body, is_html

process_proxy_url(proxy_url, timeout)

Parse and validate proxy URL accessibility, related to -p option.

Source code in humble.py

def process_proxy_url(proxy_url, timeout):
    """Parse and validate proxy URL accessibility, related to `-p` option."""
    parsed_proxy_url = urlparse(proxy_url)
    proxy_host = parsed_proxy_url.hostname
    if not proxy_host:
        print_error_detail("[proxy_host]", clean_lines=True)
    try:
        proxy_port = parsed_proxy_url.port or 8080
    except ValueError:
        print_error_detail("[proxy_port]", clean_lines=True)
    failed_proxy = Event()
    proxy_thread = Thread(target=check_proxy_url, args=(proxy_host, proxy_port,
                                                        timeout, failed_proxy),
                          daemon=True)
    proxy_thread.start()
    proxy_thread.join(timeout)
    if proxy_thread.is_alive() or failed_proxy.is_set():
        print_error_detail("[proxy_url]", clean_lines=True)
    return True

process_request_headers(request_headers)

Verify that the request headers provided are well-formed.

Exit if empty entries are found, while returning a list of any malformed headers.

Source code in humble.py

def process_request_headers(request_headers):
    """Verify that the request headers provided are well-formed.

    Exit if empty entries are found, while returning a list of any malformed
    headers.
    """
    headers = {}
    malformed_headers = []
    for header in request_headers:
        if not header:
            delete_lines()
            print()
            print(get_detail("[e_custom_eheaders]", replace=True))
            sys.exit(1)
        if ":" not in header:
            malformed_headers.append(header)
            continue
        key, value = header.split(":", 1)
        key, value = key.strip(), value.strip()
        if not key or not value:
            malformed_headers.append(header)
            continue
        headers[key] = value
    return headers, malformed_headers

process_requests_exception(exception)

Print error messages for request timeout and unhandled exceptions.

Source code in humble.py

def process_requests_exception(exception):
    """Print error messages for request timeout and unhandled exceptions."""
    if isinstance(exception, requests.exceptions.Timeout):
        delete_lines()
        delete_lines()
        print(f"\n{get_detail('[e_timeout]', replace=True)}")
        sys.exit(1)
    if exception_id := exception_d.get(type(exception)):
        print_http_exception(exception_id, exception)
    else:
        print_detail_l("[unhandled_exception]")
        print(f" {type(exception).__name__}")
        sys.exit(1)

process_server_error(http_status_code, l10n_id)

Print error message for specific server error (5xx) during analysis.

Source code in humble.py

def process_server_error(http_status_code, l10n_id):
    """Print error message for specific server error (5xx) during analysis."""
    delete_lines()
    if http_status_code in CDN_HTTP_CODES:
        print()
        print_detail(l10n_id, 0)
        if ERROR_CODES_MIXED[2] <= http_status_code <= ERROR_CODES_MIXED[3]:
            print(URL_LIST[2])
        else:
            print(URL_LIST[1])
    else:
        print_error_detail("[server_serror]")
    sys.exit(1)

save_analysis_results(t_cnt)

Save analysis results to the analysis history file, analysis_h.txt.

Info

Totals, in order, of each entry in the history file:

• Date of analysis
• URL analyzed
• Total number of enabled headers
• Total number of missing headers
• Total number of fingerprint headers
• Total number of deprecated/insecure headers
• Total number of empty headers
• Total number of warnings (the four previous totals)

Source code in humble.py

def save_analysis_results(t_cnt):
    """Save analysis results to the analysis history file, `analysis_h.txt`.

    ??? info
        Totals, in order, of each entry in the history file:<br>

        - Date of analysis
        - URL analyzed
        - Total number of enabled headers
        - Total number of missing headers
        - Total number of fingerprint headers
        - Total number of deprecated/insecure headers
        - Total number of empty headers
        - Total number of warnings (the four previous totals)
    """
    ok, fallback = validate_file_access(VALIDATE_FILE, context="history")
    if not ok:
        return fallback
    with Path(HUMBLE_FILES[0]).open("a+", encoding="utf8") as all_analysis:
        all_analysis.seek(0)
        url_ln = match_url_lines(all_analysis)
        analysis_totals = [current_time, URL, en_cnt, m_cnt, f_cnt, i_cnt[0],
                           e_cnt, t_cnt]
        all_analysis.write(" ; ".join(map(str, analysis_totals)) + "\n")
    return get_analysis_totals(url_ln) if url_ln else ("First",) * 6

sections_htmlpdf_all_export(line, states)

Identify lines requiring specific formatting for HTML and PDF exports.

Matches each line against known section prefixes and returns the appropriate formatting state; related to -o all option.

Source code in humble.py

def sections_htmlpdf_all_export(line, states):
    """Identify lines requiring specific formatting for HTML and PDF exports.

    Matches each line against known section prefixes and returns the appropriate
    formatting state; related to `-o all` option.
    """
    for prefix, new_states in SECTIONS_EXPORT_STATES.items():
        if line.startswith(prefix):
            return "\n", ExportStates(*new_states)
    if any(line.startswith(s) for s in STRINGS_BOLD + RESP_SECTION):
        return "\n", states
    return "", states

set_pdf_chunks(chunks, pdf)

Identify blocks of text to format them.

Related to -o pdf option.

Source code in humble.py

def set_pdf_chunks(chunks, pdf):
    """Identify blocks of text to format them.

    Related to `-o pdf` option.
    """
    chunk_c = None
    for i, chunk in enumerate(chunks):
        if re.search(RE_PATTERN[10], chunk):
            chunk_c = color_pdf_line(chunk[19:], PDF_COLORS[0], PDF_COLORS[1],
                                     chunks, i, pdf)
        elif re.search(RE_PATTERN[7], chunk):
            chunk_c = color_pdf_line(chunk[19:], PDF_COLORS[2], PDF_COLORS[1],
                                     chunks, i, pdf)
        else:
            pdf.set_text_color(0, 0, 0)
            formatted_chunk = format_pdf_chunks(chunk, chunks, chunk_c, i, pdf)
            pdf.cell(104, 6, text=formatted_chunk, align="L")
            pdf.ln(h=6)

set_pdf_conditions(line, pdf, ypos)

Determine whether to apply specific formatting to a line in a PDF export.

Checks the line against response header conditions and delegates formatting to set_pdf_warnings if matched; related to -o pdf option.

Source code in humble.py

def set_pdf_conditions(line, pdf, ypos):
    """Determine whether to apply specific formatting to a line in a PDF export.

    Checks the line against response header conditions and delegates formatting
    to `set_pdf_warnings` if matched; related to `-o pdf` option.
    """
    combined_h = l_miss + l_ins + l_fng + titled_fng
    combined_h.append(XFRAME_CHECK)
    return (
        all(condition not in line for condition in PDF_CONDITIONS[:3]) and
        (PDF_CONDITIONS[3] in line or any(item in line for item in combined_h))
        and set_pdf_warnings(line, pdf, ypos))

set_pdf_content(tmp_filename, ok_string, no_headers, pdf, pdf_links, pdf_prefixes, ypos)

Set the format and sections.

Related to -o pdf option.

Source code in humble.py

def set_pdf_content(tmp_filename, ok_string, no_headers, pdf, pdf_links,
                    pdf_prefixes, ypos):
    """Set the format and sections.

    Related to `-o pdf` option.
    """
    with Path(tmp_filename).open(encoding="utf8") as txt_source:
        for line in txt_source:
            if any(no_header in line for no_header in no_headers):
                set_pdf_warnings(line, pdf, ypos)
                continue
            if "[" in line:
                set_pdf_sections(line, pdf)
            if set_pdf_format(line, ok_string, pdf, pdf_links, pdf_prefixes,
                              ypos):
                continue

set_pdf_empty(l_empty, line, pdf, ypos)

Format line with empty headers.

Related to -o pdf option.

Source code in humble.py

def set_pdf_empty(l_empty, line, pdf, ypos):
    """Format line with empty headers.

    Related to `-o pdf` option.
    """
    ln_strip = line.lstrip().lower()
    if (
        any(i.lower() in ln_strip for i in l_empty)
        and "[" not in ln_strip
        and ":" not in ln_strip
    ):
        pdf.set_text_color(255, 0, 0)
        pdf.multi_cell(197, 6, text=line, align="L", new_y=ypos.LAST)
        return True
    return False

set_pdf_file(pdf)

Set display parameters along with metadata.

Related to -o pdf option.

Note

The defined zoom=100 percentage may be ignored by some PDF readers. For optimal viewing, set the zoom level to 100% manually after opening the file.

Source code in humble.py

def set_pdf_file(pdf):
    """Set display parameters along with metadata.

    Related to `-o pdf` option.

    ??? note
        The defined `zoom=100` percentage may be ignored by some PDF readers.
        For optimal viewing, set the zoom level to 100% manually after opening
        the file.

    """
    pdf.alias_nb_pages()
    set_pdf_metadata(pdf)
    pdf.set_display_mode(zoom=100)
    pdf.add_page()
    pdf.set_font("Courier", size=9)

set_pdf_format(line, ok_string, pdf, pdf_links, pdf_prefixes, ypos)

Apply specific format to lines based on its content.

Related to -o pdf option.

Source code in humble.py

def set_pdf_format(line, ok_string, pdf, pdf_links, pdf_prefixes, ypos):
    """Apply specific format to lines based on its content.

    Related to `-o pdf` option.
    """
    if any(bold_str in line for bold_str in STRINGS_BOLD):
        pdf.set_font(style="B")
    else:
        pdf.set_font(style="")
    next((format_pdf_links(line, string, pdf, pdf_prefixes) for string in
          pdf_links if string in line), None)
    if set_pdf_conditions(line, pdf, ypos):
        return True
    if ok_string in line:
        set_pdf_nowarnings(line, pdf, ypos)
        return True
    pdf.set_text_color(255, 0, 0)
    if set_pdf_empty(l_empty, line, pdf, ypos):
        return True
    format_pdf_lines(line, pdf, ypos)
    return False

set_pdf_links(i, pdf_string)

Check if the line includes a link, to display it with a specific format.

Related to -o pdf option.

Source code in humble.py

def set_pdf_links(i, pdf_string):
    """Check if the line includes a link, to display it with a specific format.

    Related to `-o pdf` option.
    """
    pdf_links_d = {URL_STRING[1]: URL,
                   REF_LINKS[2]: i.partition(REF_LINKS[2])[2].strip(),
                   REF_LINKS[3]: i.partition(REF_LINKS[3])[2].strip(),
                   REF_LINKS[4]: i.partition(REF_LINKS[4])[2].strip(),
                   URL_LIST[0]: i.partition(": ")[2].strip()}
    return pdf_links_d.get(pdf_string)

set_pdf_metadata(pdf)

Set metadata values for a PDF export; related to -o pdf option.

Source code in humble.py

def set_pdf_metadata(pdf):
    """Set metadata values for a PDF export; related to `-o pdf` option."""
    title = f"{get_detail('[pdf_meta_title]', replace=True)} {URL}"
    git_urlc = BANNER_VERSION
    pdf.set_author(git_urlc)
    pdf.set_creator(git_urlc)
    pdf.set_keywords(get_detail(METADATA_S[0], replace=True))
    pdf.set_lang(get_detail("[pdf_meta_language]"))
    pdf.set_subject(get_detail(METADATA_S[1], replace=True))
    pdf.set_title(title)
    pdf.set_producer(git_urlc)

set_pdf_nowarnings(line, pdf, ypos)

Format line without warnings.

Related to -o pdf option.

Source code in humble.py

def set_pdf_nowarnings(line, pdf, ypos):
    """Format line without warnings.

    Related to `-o pdf` option.
    """
    pdf.set_text_color(0, 128, 0)
    pdf.multi_cell(197, 6, text=line, align="L", new_y=ypos.LAST)

set_pdf_sections(line, pdf)

Set the sections of the analysis.

Related to -o pdf option.

Source code in humble.py

def set_pdf_sections(line, pdf):
    """Set the sections of the analysis.

    Related to `-o pdf` option.
    """
    for section_key, section_val in PDF_SECTION.items():
        if line.startswith(section_key):
            pdf.start_section(get_detail(section_val))
            break

set_pdf_warnings(line, pdf, ypos)

Format warnings-related lines.

Related to -o pdf option.

Source code in humble.py

def set_pdf_warnings(line, pdf, ypos):
    """Format warnings-related lines.

    Related to `-o pdf` option.
    """
    if STYLE[8] not in line:
        pdf.set_text_color(255, 0, 0)
        pdf.multi_cell(197, 6, text=line, align="L", new_y=ypos.LAST)
        return True
    return None

set_xlsx_content(final_filename, workbook)

Define the content and format of the data for a XLSX export.

Related to -o xlsx option.

Source code in humble.py

def set_xlsx_content(final_filename, workbook):
    """Define the content and format of the data for a XLSX export.

    Related to `-o xlsx` option.
    """
    worksheet = workbook.add_worksheet(get_detail(METADATA_S[1], replace=True))
    bold_fmt = workbook.add_format({"bold": True, "text_wrap": True,
                                    "align": "center", "valign": "vcenter"})
    cell_fmt = workbook.add_format({"text_wrap": True, "valign": "top"})
    hidden_fmt = workbook.add_format({"font_color": "#FFFFFF",
                                      "text_wrap": True, "valign": "top"})
    col_wd = {}
    set_xlsx_format(bold_fmt, cell_fmt, col_wd, final_filename, hidden_fmt,
                    worksheet)
    set_xlsx_width(col_wd, worksheet)
    worksheet.autofilter(0, 0, 0, 1)

set_xlsx_format(bold_fmt, cell_fmt, col_wd, final_filename, hidden_fmt, worksheet)

Write formatted content with dynamic column widths for a XLSX export.

Related to -o xlsx option.

Note

This function uses defusedcsv to mitigate formula injection attacks by sanitizing potentially dangerous values.

Source code in humble.py

def set_xlsx_format(bold_fmt, cell_fmt, col_wd, final_filename, hidden_fmt,
                    worksheet):
    """Write formatted content with dynamic column widths for a XLSX export.

    Related to `-o xlsx` option.

    ??? note
        This function uses `defusedcsv` to mitigate formula injection attacks
        by sanitizing potentially dangerous values.
    """
    prev_section = None
    with Path(final_filename).open(encoding="utf-8", newline="") as csv_final:
        for row_index, row_data in enumerate(
                defusedcsv_logic.reader(csv_final)):
            for col_index, cell_value in enumerate(row_data):
                fmt, prev_section = choose_xlsx_format(bold_fmt, cell_fmt,
                                                       cell_value, col_index,
                                                       hidden_fmt, row_index,
                                                       prev_section)
                worksheet.write(row_index, col_index, cell_value, fmt)
                col_wd[col_index] = max(col_wd.get(col_index, 0),
                                        len(cell_value))

set_xlsx_metadata(workbook)

Set metadata values for a XLSX export, related to -o xlsx option.

Source code in humble.py

def set_xlsx_metadata(workbook):
    """Set metadata values for a XLSX export, related to `-o xlsx` option."""
    workbook.set_properties({
        "author": BANNER_VERSION,
        "category": get_detail(METADATA_S[1], replace=True),
        "keywords": get_detail(METADATA_S[0], replace=True),
        "subject": get_detail(METADATA_S[1], replace=True),
        "title": f"{get_detail('[pdf_meta_title]', replace=True)} {URL}",
        "comments": f"{get_detail('[excel_meta_generated]', replace=True)} \
{BANNER_VERSION}",
    })

set_xlsx_width(col_wd, worksheet)

Set column widths for a XLSX export, related to -o xlsx option.

Source code in humble.py

def set_xlsx_width(col_wd, worksheet):
    """Set column widths for a XLSX export, related to `-o xlsx` option."""
    for col_idx, width in col_wd.items():
        if col_idx == 0:
            actual_width = width
        elif col_idx == 1:
            col_a_width = col_wd.get(0, 0)
            adjusted_b_width = max(width + 2, col_a_width * 2)
            actual_width = min(adjusted_b_width, 100)
        worksheet.set_column(col_idx, col_idx, actual_width)

testssl_analysis(testssl_cmd)

Run TLS/SSL analysis with testssl.sh, related to -e option.

Note

This function is safe from injection: shell=True is not used, arguments are passed as a list and testssl.sh options are defined in the TESTSSL_OPTIONS constant.

Source code in humble.py

def testssl_analysis(testssl_cmd):
    """Run TLS/SSL analysis with testssl.sh, related to `-e` option.

    ??? note
        This function is safe from injection: `shell=True` is not used,
        arguments are passed as a list and `testssl.sh` options are defined in
        the `TESTSSL_OPTIONS` constant.
    """
    try:
        # nosemgrep: dangerous-subprocess-use-audit
        process = Popen(testssl_cmd, stdout=PIPE, stderr=STDOUT,
                        text=True) # false-positive
        for ln in iter(process.stdout.readline, ""):
            print(ln, end="")
            if "Done" in ln:
                process.terminate()
                break
        process.wait()
    except (OSError, ValueError):
        print_error_detail("[testssl_error]")

testssl_command(testssl_temp_path, uri)

Prepare the TLS/SSL analysis.

Validate the URI and prompt the user to accept terms and exit if declined; related to -e option.

Tip

The options used in the analysis are defined in the TESTSSL_OPTIONS constant:

• -f: checks robust forward secrecy key exchange
• -g: checks several server implementation bugs
• -p: checks the availability of SSL/TLS protocols
• -U: tests all vulnerabilities, like Heartbleed, ROBOT and sweet32
• -s: tests lists of cipher suites/categories by strength
• --hints: (available in the future) give hints how to fix a finding

Check the testssl.sh documentation for a list of all available options.

Source code in humble.py

def testssl_command(testssl_temp_path, uri):
    """Prepare the TLS/SSL analysis.

    Validate the URI and prompt the user to accept terms and exit if declined;
    related to `-e` option.

    ??? tip
        The options used in the analysis are defined in the `TESTSSL_OPTIONS`
        constant:<br>

        - `-f`: checks robust forward secrecy key exchange
        - `-g`: checks several server implementation bugs
        - `-p`: checks the availability of SSL/TLS protocols
        - `-U`: tests all vulnerabilities, like Heartbleed, ROBOT and sweet32
        - `-s`: tests lists of cipher suites/categories by strength
        - `--hints`: (available in the future) give hints how to fix a finding

        Check the testssl.sh [documentation](https://testssl.sh/doc/testssl.1.html){:target="_blank"}
        for a list of all available options.
    """
    validate_testssl_uri(uri)
    args_path = Path(testssl_temp_path).resolve()
    if not args_path.is_dir():
        print_error_detail("[notestssl_path]")
    testssl_path = next(
        (args_path / filename for filename in TESTSSL_FILE if
         (args_path / filename).is_file()), None)
    if not testssl_path or not which(testssl_path):
        print_error_detail("[notestssl_fileexec]")
    print()
    print(f"{get_detail('[testssl_warning]', replace=True)} '{testssl_path}'")
    choice = input( # false-positive
        f"{get_detail('[testssl_choice]', replace=True)} ")[:1].strip().lower()
    if choice != "y":
        sys.exit(0)
    delete_lines()
    testssl_cmd = [testssl_path, *TESTSSL_OPTIONS, uri]
    testssl_analysis(testssl_cmd)
    sys.exit(0)

threshold_cicd(threshold_grade, totals)

CI/CD threshold validation.

Related to -cicd GRADE option.

Source code in humble.py

def threshold_cicd(threshold_grade, totals):
    """CI/CD threshold validation.

    Related to `-cicd GRADE` option.
    """
    threshold_norm = threshold_grade.strip().upper()
    analysis_grade = validate_cicd_grade(threshold_norm, threshold_grade,
                                         totals)
    fails_security_gate = check_cicd(analysis_grade, threshold_norm)
    msg_key = "[cicd_ko]" if fails_security_gate else "[cicd_ok]"
    return (
        f"{get_detail(msg_key, replace=True)}, '{threshold_norm}'",
        fails_security_gate,
    )

url_analytics(*, is_global=False)

Print analysis statistics for all analyses performed on a URL and exit.

Related to the -a option.

Source code in humble.py

def url_analytics(*, is_global=False):
    """Print analysis statistics for all analyses performed on a URL and exit.

    Related to the `-a` option.
    """
    url_scope = extract_global_metrics if is_global else get_analysis_metrics
    with Path(HUMBLE_FILES[0]).open(encoding="utf8") as all_analysis:
        analysis_metrics = url_scope(all_analysis)
    l10n_det = "[global_stats_analysis]" if is_global else "[stats_analysis]"
    url_string = "" if is_global else URL
    print(f"\n{get_detail(l10n_det, replace=True)} {url_string}\n")
    for key, value in analysis_metrics.items():
        key_style = f"{STYLE[0]}{key}{STYLE[4]}" if not value or not \
            key.startswith(" ") else key
        print(f"{key_style}: {value}")
    sys.exit(0)

validate_cicd_grade(threshold_norm, threshold_grade, totals)

Validate the target threshold and the actual analysis grade.

Ensures both grades exist within GRADE_ORDER: exits in case of an invalid threshold or an undeterminable analysis grade.

Source code in humble.py

def validate_cicd_grade(threshold_norm, threshold_grade, totals):
    """Validate the target threshold and the actual analysis grade.

    Ensures both grades exist within `GRADE_ORDER`: exits in case of an invalid
    threshold or an undeterminable analysis grade.
    """
    analysis_grade = fetch_cicd_grade(totals)
    if threshold_norm in GRADE_ORDER and analysis_grade in GRADE_ORDER:
        return analysis_grade
    bad_thold = threshold_norm not in GRADE_ORDER
    msg = (
        f"'{threshold_grade}' {get_detail('[cicd_invalid]', replace=True)}; "
        f"{get_detail('[cicd_valid]', replace=True)} "
        + ", ".join(f"'{g}'" for g in GRADE_ORDER) + "."
        if bad_thold else get_detail("[cicd_no_grade]", replace=True)
    )
    print(f"\n{get_detail('[cicd_error]', replace=True)}: {msg}")
    sys.exit(2 if bad_thold else 1)

validate_file_access(target_path, *, context='history')

Check if the history or export files can be accessed or created.

Exit if an error occurs during the export of an analysis.

Source code in humble.py

def validate_file_access(target_path, *, context="history"):
    """Check if the history or export files can be accessed or created.

    Exit if an error occurs during the export of an analysis.
    """
    try:
        with Path(target_path).open("a+", encoding="utf8"):
            pass
    except OSError as file_err:
        err_str = file_err.strerror or type(file_err).__name__
        if context == "history":
            return False, ("Not available",) * 6
        if context == "basic":
            msg = get_detail("[analysis_history_note]", replace=True)
            print(f"{msg} ({err_str})")
            return False, None
        if context == "export":
            delete_lines()
            err_msg = get_detail("[e_export_analysis]", replace=True)
            print(f"\n{err_msg} ({err_str}).")
            sys.exit(1)
    return True, None

validate_path(output_path)

Validate permissions in the provided path.

Exit in case of error, related to -op option.

Source code in humble.py

def validate_path(output_path):
    """Validate permissions in the provided path.

    Exit in case of error, related to `-op` option.
    """
    try:
        validate_path = Path(output_path) / HUMBLE_FILES[1]
        with validate_path.open("w", encoding="utf8"):
            pass
    except OSError as path_err:
        print(f"\n {get_detail('[args_pathe]', replace=True)} "
              f"'{output_path}' ({path_err.strerror})")
        sys.exit(1)
    else:
        validate_path.unlink()

validate_testssl_uri(uri)

Check that the URI is well-formed before analyzing it.

Exit with a specific message if the URI has no scheme, an unsupported one, or no host; related to -e option.

Source code in humble.py

def validate_testssl_uri(uri):
    """Check that the URI is well-formed before analyzing it.

    Exit with a specific message if the URI has no scheme, an unsupported
    one, or no host; related to `-e` option.
    """
    parsed_uri = urlparse(uri)
    if not parsed_uri.scheme:
        print_error_detail("[e_mschema]")
    if parsed_uri.scheme not in HTTP_SCHEMES_S:
        print_error_detail("[e_ischema]")
    if not parsed_uri.netloc:
        print_error_detail("[e_url]")

validate_url(url)

Exit cleanly if the URL cannot be parsed.

urlparse() raises ValueError on malformed URLs (e.g. an unclosed IPv6 bracket), and accessing .port does so on an invalid port; this guards check_russian_scope() and every later consumer from an uncaught traceback. Scheme and host remain the responsibility of the requests library, reported via exception_d.

Related to -u option.

Source code in humble.py

def validate_url(url):
    """Exit cleanly if the URL cannot be parsed.

    `urlparse()` raises `ValueError` on malformed URLs (e.g. an unclosed
    IPv6 bracket), and accessing `.port` does so on an invalid port; this
    guards `check_russian_scope()` and every later consumer from an
    uncaught traceback. Scheme and host remain the responsibility of the
    `requests` library, reported via `exception_d`.

    Related to `-u` option.
    """
    try:
        _ = urlparse(url).port
    except ValueError:
        print_error_detail("[e_url]")

write_csv_content(csv_file, txt_source)

Write headers and parse content.

Related to -o csv option.

Note

defusedcsv is used to mitigate formula injection attacks by sanitizing potentially dangerous values

Source code in humble.py

def write_csv_content(csv_file, txt_source):
    """Write headers and parse content.

    Related to `-o csv` option.

    ??? note
        `defusedcsv` is used to mitigate formula injection attacks by
        sanitizing potentially dangerous values
    """
    writer = defusedcsv_logic.writer(csv_file,
                                     quoting=defusedcsv_logic.QUOTE_ALL)
    writer.writerow([
        get_detail("[csv_section]", replace=True),
        get_detail("[csv_values]", replace=True),
    ])
    writer.writerow([
        get_detail("[0section]", replace=True),
        f"{get_detail('[json_gen]', replace=True)}: {BANNER_VERSION}",
    ])
    section_titles = [get_detail(f"[{i}]", replace=True) for i in CSV_SECTION]
    parse_csv(section_titles, txt_source.read(), writer)

write_html_line(html_final, ln, html_writers, html_rest, inside_section)

Write a single line of the analysis, applying the first matching rule.

Section names are handled apart, being the only stateful case; lines matching no rule fall through to format_html_rest().

Related to -o html option.

Source code in humble.py

def write_html_line(html_final, ln, html_writers, html_rest,
                    inside_section):
    """Write a single line of the analysis, applying the first matching rule.

    Section names are handled apart, being the only stateful case; lines
    matching no rule fall through to `format_html_rest()`.

    Related to `-o html` option.
    """
    ln_rstrip = ln.rstrip("\n")
    if format_html_info(html_final, ln_rstrip):
        return inside_section
    matched_bold, inside_section = format_html_bold(html_final, ln_rstrip,
                                                    inside_section)
    if matched_bold or any(writer(html_final, ln_rstrip) for writer in
                           html_writers):
        return inside_section
    html_rest(html_final, ln)
    return inside_section

write_json(json_lns, json_section, sections)

Format content for a JSON export.

Related to -o json -b options.

Source code in humble.py

def write_json(json_lns, json_section, sections):
    """Format content for a JSON export.

    Related to `-o json -b` options.
    """
    if json_section not in sections:
        return list(json_lns)
    json_data = {}
    format_json(json_data, json_lns)
    if json_section == sections[0]:
        json_data = {get_detail("[json_gen]", replace=True):
                     BANNER_VERSION, **json_data}
    return json_data