diff --git a/docs/Compiler-Hardening-Guides/README.md b/docs/Compiler-Hardening-Guides/README.md index 2582ceec8..1e42d0c73 100644 --- a/docs/Compiler-Hardening-Guides/README.md +++ b/docs/Compiler-Hardening-Guides/README.md @@ -9,7 +9,7 @@ The objective of compiler options hardening is to produce application binaries ( ## Usage in Tools -A python script is also provided (in the `compiler-options-scraper` directory) that can fetch the latest version of the OpenSSF compiler hardening guide from the internet, obtain the recommended options tables from it and convert them to a machine readable JSON for usage in tools. +A python script is also provided that can fetch the latest version of the OpenSSF compiler hardening guide from the internet, obtain the recommended options tables from it and convert them to a machine readable JSON for usage in tools. ## How to Contribute diff --git a/docs/Compiler-Hardening-Guides/compiler-options-scraper/.gitignore b/docs/Compiler-Hardening-Guides/compiler-options-scraper/.gitignore deleted file mode 100644 index 890a7834d..000000000 --- a/docs/Compiler-Hardening-Guides/compiler-options-scraper/.gitignore +++ /dev/null @@ -1 +0,0 @@ -compiler-options.json diff --git a/docs/Compiler-Hardening-Guides/compiler-options-scraper/README.md b/docs/Compiler-Hardening-Guides/compiler-options-scraper/README.md deleted file mode 100644 index 7529b9f30..000000000 --- a/docs/Compiler-Hardening-Guides/compiler-options-scraper/README.md +++ /dev/null @@ -1,17 +0,0 @@ -# Compiler Options Scraper - -This directory contains a Python3 script that can fetch the latest OpenSSF Compiler Hardening Guide -from the website, obtain the recommended options tables and convert them to a machine readable format (JSON). -The output is saved in a JSON file: `compiler-options.json` -This can be changed by changing the global variable at the top of the script. - -## Usage - -`python3 main.py` - -## Dependencies - -Dependencies are specified in `requirements.txt`. The main dependencies are: - -1. requests: To fetch the HTML page from the given OpenSSF URL. -2. BeautifulSoup4: To parse the HTML page to convert it to JSON diff --git a/docs/Compiler-Hardening-Guides/compiler-options-scraper/main.py b/docs/Compiler-Hardening-Guides/compiler-options-scraper/main.py deleted file mode 100644 index 65e126c10..000000000 --- a/docs/Compiler-Hardening-Guides/compiler-options-scraper/main.py +++ /dev/null @@ -1,149 +0,0 @@ -# compiler-options-scraper.py -# Author: mayank-ramnani -# Date: 2024-06-26 -# Description: Python script to scrape recommended compiler options from the OpenSSF \ - # Compiler Options Hardening Guide HTML page - # Scrapes the webpage and stores the options in JSON format in `compiler-options.json` \ - # file in the execution directory - -import requests -from bs4 import BeautifulSoup -import json -import re -from typing import Optional, List, Dict, Tuple, Any - - -OPENSSF_URL = ("https://best.openssf.org/Compiler-Hardening-Guides/" - "Compiler-Options-Hardening-Guide-for-C-and-C++.html") -DB_FILE = "compiler-options.json" - - -def scrape_document(url: str) -> Optional[BeautifulSoup]: - """Scrape the document from the given URL.""" - response = requests.get(url) - if response.status_code == 200: - return BeautifulSoup(response.text, 'html.parser') - print("Failed to fetch HTML content") - return None - -# Assumes version date is in the first paragraph element -def extract_version_from_soup(soup: BeautifulSoup) -> Optional[str]: - """Extract version date from the document's subtitle.""" - subtitle = soup.find('p').get_text() - if not subtitle: - print("No subtitle found in the document") - return None - - date_pattern = r'\b\d{4}-\d{2}-\d{2}\b' - match = re.search(date_pattern, subtitle) - if not match: - print("No version date found in the subtitle of document.") - return None - - version_date = match.group(0) - return version_date - - -def table_to_dicts(table: BeautifulSoup) -> List[Dict[str, str]]: - """Convert a BeautifulSoup table to a list of dictionaries.""" - # get table headers - headers = [header.get_text() for header in table.find_all('th')] - # get table rows - rows = table.find_all('tr')[1:] # Skip the header row, start from index 1 - - # convert rows to dictionaries - data = [] - for row in rows: - row_data = [] - for cell in row.find_all('td'): - for r in cell: - if (r.string is None): - r.string = ' ' - row_data.append(cell.get_text()) - row_dict = dict(zip(headers, row_data)) - data.append(row_dict) - - return data - - -def split_description(desc: str) -> Tuple[str, str]: - """Split description into main description and prerequisite.""" - index = desc.find("Requires") - if index != -1: - return desc[:index], desc[index:] - return desc, "" - - -def convert_to_json(table_data: List[Dict[str, str]]) -> List[Dict[str, Any]]: - """Convert table data to JSON format.""" - json_data = [] - for entry in table_data: - flags = [entry['Compiler Flag']] - for flag in flags: - desc, prereq = split_description(entry['Description']) - json_entry = {} - json_entry["opt"] = flag - json_entry["desc"] = desc - if prereq: - json_entry["prereq"] = prereq - json_entry["requires"] = extract_versions(entry['Supported since']) - - json_data.append(json_entry) - return json_data - - -def extract_versions(input_string: str) -> Dict[str, str]: - """Extract version information of dependencies from the input string.""" - versions = {} - # Regex for various dependencies - # NOTE: the last version node is assumed to be single digit - # if you need to support multiple digits, d+ can be added - # however, it will start including the superscript references in the version number - # example: -D_FORTIFY_SOURCE=3 - version_patterns = { - 'gcc': r'GCC\s+(\d+\.\d+\.\d)', - 'clang': r'Clang\s+(\d+\.\d+\.\d)', - 'binutils': r'Binutils\s+(\d+\.\d+\.\d)', - 'libc++': r'libc\+\+\s+(\d+\.\d+\.\d)', - 'libstdc++': r'libstdc\+\+\s+(\d+\.\d+\.\d)' - } - - versions = {} - for key, pattern in version_patterns.items(): - match = re.search(pattern, input_string) - if match: - versions[key] = match.group(1) - - return versions - - -def main(): - """Main function to scrape and process the document.""" - soup = scrape_document(OPENSSF_URL) - if not soup: - print("Error: Unable to scrape document") - return - - # extract document version info - version = extract_version_from_soup(soup) - # extract all tables from soup: finds all tags - tables = soup.find_all('table') - - # NOTE: we only care about tables 1 and 2, since those contain recommended options - # convert tables to list of dictionaries and merge entries - recommended_data = table_to_dicts(tables[1]) + table_to_dicts(tables[2]) - - # convert table to JSON format - json_data = convert_to_json(recommended_data) - - output_db = {"version": version, "options": {"recommended": json_data}} - - with open(DB_FILE, "w") as fp: - # json_formatted_str = json.dumps(output_db, indent=4) - # fp.write(json_formatted_str) - json.dump(output_db, fp, indent=4) - print("Write compiler options in json to:", DB_FILE) - - -if __name__ == "__main__": - main() diff --git a/docs/Compiler-Hardening-Guides/compiler-options-scraper/requirements.txt b/docs/Compiler-Hardening-Guides/compiler-options-scraper/requirements.txt deleted file mode 100644 index 821e35baa..000000000 --- a/docs/Compiler-Hardening-Guides/compiler-options-scraper/requirements.txt +++ /dev/null @@ -1,8 +0,0 @@ -beautifulsoup4==4.12.3 -bs4==0.0.2 -certifi==2024.7.4 -charset-normalizer==3.3.2 -idna==3.7 -requests==2.32.4 -soupsieve==2.5 -urllib3==2.5.0 diff --git a/docs/Compiler-Hardening-Guides/example-cmake/CMakeLists.txt b/docs/Compiler-Hardening-Guides/example-cmake/CMakeLists.txt new file mode 100644 index 000000000..d98f28a3f --- /dev/null +++ b/docs/Compiler-Hardening-Guides/example-cmake/CMakeLists.txt @@ -0,0 +1,88 @@ +cmake_minimum_required(VERSION 3.27 FATAL_ERROR) +project(demo LANGUAGES C) + +set(CMAKE_C_STANDARD 23) +set(CMAKE_C_STANDARD_REQUIRED ON) + +# +# compiler hardening +# + +if(CMAKE_C_COMPILER_ID MATCHES "GNU" AND CMAKE_C_COMPILER_VERSION VERSION_LESS 14.0.0) + message(FATAL_ERROR "GCC is outdated: ${CMAKE_C_COMPILER_VERSION}") +endif() +if(CMAKE_C_COMPILER_ID MATCHES "Clang" AND CMAKE_C_COMPILER_VERSION VERSION_LESS 16.0.0) + message(FATAL_ERROR "Clang is outdated: ${CMAKE_C_COMPILER_VERSION}") +endif() + +add_compile_options( + -O2 + -Wall + -Wextra + -Wformat + -Wformat=2 + -Wconversion + -Wsign-conversion + -Wimplicit-fallthrough + -Werror=format-security + # more portable and explicit than -fhardened + -U_FORTIFY_SOURCE + -D_FORTIFY_SOURCE=3 + -D_GLIBCXX_ASSERTIONS + -fstrict-flex-arrays=3 + -fstack-protector-strong + # deprecated c calls + -Werror=implicit + -Werror=incompatible-pointer-types + -Werror=int-conversion + # multithreading with pthreads + -fexceptions + # for shared libraries use `-fPIC` + -fPIE +) + +# build mode specific +if(CMAKE_BUILD_TYPE STREQUAL "Release") + add_compile_options( + -fno-delete-null-pointer-checks -fno-strict-overflow -fno-strict-aliasing -ftrivial-auto-var-init=zero + ) +else() + add_compile_options(-Werror) +endif() + +# os specific +if(CMAKE_SYSTEM_NAME STREQUAL "Linux") + add_compile_options(-fstack-clash-protection) + add_link_options( + -pie + -Wl,-z,nodlopen + -Wl,-z,noexecstack + -Wl,-z,relro + -Wl,-z,now + -Wl,--as-needed + -Wl,--no-copy-dt-needed-entries + ) +endif() + +# compiler specific +if(CMAKE_C_COMPILER_ID MATCHES "GNU") + # from gcc-15 add `-fzero-init-padding-bits=all` + add_compile_options(-Wtrampolines -Wbidi-chars=any) +endif() +if(CMAKE_C_COMPILER_ID MATCHES "Clang") + # nop +endif() + +# architecture specific +if(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|AMD64|i686|i386") + add_compile_options(-fcf-protection=full) +endif() +if(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|arm64|ARM64") + add_compile_options(-mbranch-protection=standard) +endif() + +# +# sources +# + +add_executable(demo demo.c) \ No newline at end of file diff --git a/docs/Compiler-Hardening-Guides/example-cmake/Dockerfile b/docs/Compiler-Hardening-Guides/example-cmake/Dockerfile new file mode 100644 index 000000000..decfc44d8 --- /dev/null +++ b/docs/Compiler-Hardening-Guides/example-cmake/Dockerfile @@ -0,0 +1,14 @@ +FROM --platform=linux/amd64 ubuntu:24.04 + +RUN apt-get update && apt-get install -y --no-install-recommends \ + gcc-14 g++-14 \ + clang-19 \ + cmake make \ + valgrind \ + && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-14 100 \ + && update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-14 100 \ + && update-alternatives --install /usr/bin/clang clang /usr/bin/clang-19 100 \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /workspace +COPY . /workspace diff --git a/docs/Compiler-Hardening-Guides/example-cmake/Makefile b/docs/Compiler-Hardening-Guides/example-cmake/Makefile new file mode 100644 index 000000000..bb5a4615a --- /dev/null +++ b/docs/Compiler-Hardening-Guides/example-cmake/Makefile @@ -0,0 +1,17 @@ +DOCKER_RUN = docker run --rm -v $(PWD):/workspace main sh -c + +.PHONY: build-image +build-image: + docker build -t main . + +.PHONY: run-gcc +run-gcc: build-image + $(DOCKER_RUN) "mkdir -p /tmp/gcc-build && cd /tmp/gcc-build && cmake -DCMAKE_C_COMPILER=gcc /workspace && make && ./demo" + +.PHONY: run-clang +run-clang: build-image + $(DOCKER_RUN) "mkdir -p /tmp/clang-build && cd /tmp/clang-build && cmake -DCMAKE_C_COMPILER=clang /workspace && make && ./demo" + +.PHONY: clean +clean: + docker rmi main diff --git a/docs/Compiler-Hardening-Guides/demo.c b/docs/Compiler-Hardening-Guides/example-cmake/demo.c similarity index 100% rename from docs/Compiler-Hardening-Guides/demo.c rename to docs/Compiler-Hardening-Guides/example-cmake/demo.c diff --git a/docs/Compiler-Hardening-Guides/Makefile b/docs/Compiler-Hardening-Guides/example-minimal/Makefile similarity index 100% rename from docs/Compiler-Hardening-Guides/Makefile rename to docs/Compiler-Hardening-Guides/example-minimal/Makefile diff --git a/docs/Compiler-Hardening-Guides/example-minimal/demo.c b/docs/Compiler-Hardening-Guides/example-minimal/demo.c new file mode 100644 index 000000000..d61fc5499 --- /dev/null +++ b/docs/Compiler-Hardening-Guides/example-minimal/demo.c @@ -0,0 +1,27 @@ +// Test C/C++ hardening flags + +// Copyright Open Source Security Foundation (OpenSSF) and its contributors +// SPDX-License-Identifier: Apache-2.0 OR MIT + +#include + +// Linux 5.10 solution: +#if __has_attribute(__fallthrough__) +# define fallthrough __attribute__((__fallthrough__)) +#else +# define fallthrough do {} while (0) /* fallthrough */ +#endif + +int main(void) { + int c = 0; + switch (c) { + case 1: + printf("Hello\n"); + fallthrough; + case 0: + printf("Goodbye\n"); + fallthrough; + default: + printf("Default\n"); + } +} diff --git a/docs/Compiler-Hardening-Guides/get_options.py b/docs/Compiler-Hardening-Guides/get_options.py new file mode 100644 index 000000000..fa5698596 --- /dev/null +++ b/docs/Compiler-Hardening-Guides/get_options.py @@ -0,0 +1,95 @@ +# +# Usage: `uv run get_options.py > options.json` +# See: https://docs.astral.sh/uv/guides/scripts/#declaring-script-dependencies +# +# +# /// script +# requires-python = ">=3.10" +# dependencies = [ +# "requests==2.32.4", +# "markdown==3.6", +# "beautifulsoup4==4.12.3", +# ] +# /// + +import json +import re +import sys +from pathlib import Path +from typing import Any, Dict, List, Tuple + +import markdown +import requests +from bs4 import BeautifulSoup + +__author__ = "Yahya Jabary" +__copyright__ = "The OpenSSF Best Practices WG" +__license__ = "Apache-2.0" + + +def extract_versions(input_string: str) -> Dict[str, str]: + version_patterns = { + "gcc": r"GCC\s+(\d+\.\d+\.\d)", + "clang": r"Clang\s+(\d+\.\d+\.\d)", + "binutils": r"Binutils\s+(\d+\.\d+\.\d)", + "libc++": r"libc\+\+\s+(\d+\.\d+\.\d)", + "libstdc++": r"libstdc\+\+\s+(\d+\.\d+\.\d)", + } + return {key: match.group(1) for key, pattern in version_patterns.items() if (match := re.search(pattern, input_string))} + + +def get_desc_preq_pair(desc: str) -> Tuple[str, str]: + split_index = desc.find("Requires") + return (desc[:split_index], desc[split_index:]) if split_index != -1 else (desc, "") + + +def create_option_dict(row_data: Dict[str, str]) -> Dict[str, Any]: + description, prerequisite = get_desc_preq_pair(row_data["Description"]) + + option_dict = { + "option": row_data["Compiler Flag"], + "description": description, + "requires": extract_versions(row_data["Supported since"]), + } + + if prerequisite: + option_dict["prerequisite"] = prerequisite + + return option_dict + + +def table_to_dict(table: BeautifulSoup) -> List[Dict[str, Any]]: + headers = [header.get_text().strip() for header in table.find_all("th")] + rows = table.find_all("tr")[1:] + + header_value_dicts = [dict(zip(headers, [cell.get_text().strip() for cell in row.find_all("td")])) for row in rows] + + return [create_option_dict(row_data) for row_data in header_value_dicts] + + +def get_content() -> str: + filename = "Compiler-Options-Hardening-Guide-for-C-and-C++.md" + cwd_files = list(Path().cwd().glob(filename)) + if cwd_files: + return cwd_files[0].read_text() + + # remote fallback if not found in current working directory + fallback = "https://raw.githubusercontent.com/ossf/wg-best-practices-os-developers/refs/heads/main/docs/Compiler-Hardening-Guides/Compiler-Options-Hardening-Guide-for-C-and-C%2B%2B.md" + response = requests.get(fallback) + assert response.status_code == 200 + return response.text + + +if __name__ == "__main__": + content = get_content() + + html = markdown.markdown(content, extensions=["tables"]) + soup = BeautifulSoup(html, "html.parser") + tables = soup.find_all("table") + + version = re.search(r"\b\d{4}-\d{2}-\d{2}\b", content).group(0) + compile_time_options = table_to_dict(tables[1]) + runtime_options = table_to_dict(tables[2]) + + output = {"version": version, "options": compile_time_options + runtime_options} + json.dump(output, fp=sys.stdout, indent=4)