Merge branch 'master' into python-web-applications

Author: martin-martin

README.md

@@ -1,10 +1,18 @@
 # Real Python Materials
 
-Bonus materials, exercises, and example projects for our [Python tutorials](https://realpython.com).
+Bonus materials, exercises, and example projects for Real Python's [Python tutorials](https://realpython.com).
 
 Build Status: [![CircleCI](https://circleci.com/gh/realpython/materials.svg?style=svg)](https://circleci.com/gh/realpython/materials)
 
-## Running Code Style Checks
+## Got a Question?
+
+The best way to get support for Real Python courses & articles and code in this repository is to join one of our [weekly Office Hours calls](https://realpython.com/office-hours/) or to ask your question in the [RP Community Slack](https://realpython.com/community/). 
+
+Due to time constraints we cannot provide 1:1 support via GitHub. See you on Slack or on the next Office Hours call 🙂
+
+## Adding Source Code & Sample Projects to This Repo (RP Contributors)
+
+### Running Code Style Checks
 
 We use [flake8](http://flake8.pycqa.org/en/latest/) and [black](https://github.com/ambv/black) to ensure a consistent code style for all of our sample code in this repository.
 
@@ -15,7 +23,7 @@ $ flake8
 $ black --check .
 ```
 
-## Running Python Code Formatter
+### Running Python Code Formatter
 
 We're using a tool called [black](https://github.com/ambv/black) on this repo to ensure consistent formatting. On CI it runs in "check" mode to ensure any new files added to the repo are following PEP 8. If you see linter warnings that say something like "would reformat some_file.py" it means black disagrees with your formatting. 
 

nlp-sentiment-analysis/sentiment_analyzer.py

@@ -79,19 +79,19 @@ def evaluate_model(tokenizer, textcat, test_data: list) -> dict:
     true_negatives = 0
     false_negatives = 1e-8
     for i, review in enumerate(textcat.pipe(reviews)):
-        true_label = labels[i]
+        true_label = labels[i]["cats"]
         for predicted_label, score in review.cats.items():
             # Every cats dictionary includes both labels, you can get all
             # the info you need with just the pos label
             if predicted_label == "neg":
                 continue
-            if score >= 0.5 and true_label == "pos":
+            if score >= 0.5 and true_label["pos"]:
                 true_positives += 1
-            elif score >= 0.5 and true_label == "neg":
+            elif score >= 0.5 and true_label["neg"]:
                 false_positives += 1
-            elif score < 0.5 and true_label == "neg":
+            elif score < 0.5 and true_label["neg"]:
                 true_negatives += 1
-            elif score < 0.5 and true_label == "pos":
+            elif score < 0.5 and true_label["pos"]:
                 false_negatives += 1
     precision = true_positives / (true_positives + false_positives)
     recall = true_positives / (true_positives + false_negatives)

python-bitwise-operators/README.md

@@ -0,0 +1,22 @@
+# Bitwise Operators in Python
+
+Source code of the least-significant bit **steganography** example from the [Bitwise Operators in Python](https://realpython.com/python-bitwise-operators/) article.  
+
+## Installation
+
+There are no external dependencies except for a Python interpreter.
+
+## Running
+
+Change directory to the current folder, where this `README.md` file is located, and then execute Python module:
+
+```shell
+$ python -m stegano /path/to/bitmap (--encode /path/to/file | --decode | --erase)
+```
+
+For example, to extract a secret file from the attached bitmap, type the following:
+
+```shell
+$ python -m stegano example.bmp -d
+Extracted a secret file: podcast.mp4
+```

python-bitwise-operators/example.bmp

@@ -0,0 +1,28 @@
+"""
+Runnable module, can be executed as:
+$ python -m stegano
+"""
+
+from .bitmap import Bitmap
+from .cli import CommandLineArguments, parse_args
+from .decoder import decode, DecodingError
+from .encoder import encode, EncodingError
+from .eraser import erase
+
+
+def main(args: CommandLineArguments) -> None:
+    """Entry point to the script."""
+    with Bitmap(args.bitmap) as bitmap:
+        if args.encode:
+            encode(bitmap, args.encode)
+        elif args.decode:
+            decode(bitmap)
+        elif args.erase:
+            erase(bitmap)
+
+
+if __name__ == "__main__":
+    try:
+        main(parse_args())
+    except (EncodingError, DecodingError) as ex:
+        print(ex)

python-bitwise-operators/stegano/__main__.py

@@ -0,0 +1,116 @@
+"""
+Bitmap read/write operations.
+"""
+
+import pathlib
+from dataclasses import dataclass
+from itertools import islice
+from mmap import mmap, ACCESS_WRITE
+from struct import pack, unpack
+from typing import Any, Union, Iterator
+
+
+class Bitmap:
+    """High-level interface to a bitmap file."""
+
+    def __init__(self, path: pathlib.Path) -> None:
+        self._file = path.open(mode="r+b")
+        self._file_bytes = mmap(self._file.fileno(), 0, access=ACCESS_WRITE)
+        self._header = Header.from_bytes(self._file_bytes[:50])
+
+    def __enter__(self) -> "Bitmap":
+        return self
+
+    def __exit__(self, *args, **kwargs) -> None:
+        self._file_bytes.close()
+        self._file.close()
+
+    def __getattr__(self, name: str) -> Any:
+        return getattr(self._header, name)
+
+    def __getitem__(self, offset: Union[int, slice]) -> Union[int, bytes]:
+        return self._file_bytes[offset]
+
+    def __setitem__(
+        self, offset: Union[int, slice], value: Union[int, bytes]
+    ) -> None:
+        self._file_bytes[offset] = value
+
+    @property
+    def max_bytes(self) -> int:
+        """The maximum number of bytes the bitmap can hide."""
+        return self.width * self.height * 3
+
+    @property
+    def byte_offsets(self) -> Iterator[int]:
+        """Return an iterator over byte offsets (skip the padding)."""
+
+        start_index = self.pixels_offset
+        end_index = self.pixels_offset + self.pixel_size_bytes
+        scanline_bytes = self.pixel_size_bytes // self.height
+
+        for scanline in range(start_index, end_index, scanline_bytes):
+            yield from range(scanline, scanline + self.width * 3)
+
+    @property
+    def byte_slices(self) -> Iterator[slice]:
+        """Generator iterator of 8-byte long slices."""
+        for byte_index in islice(self.byte_offsets, 0, self.max_bytes, 8):
+            yield slice(byte_index, byte_index + 8)
+
+    @property
+    def reserved_field(self) -> int:
+        """Return a little-endian 32-bit unsigned integer."""
+        return unsigned_int(self._file_bytes, 0x06)
+
+    @reserved_field.setter
+    def reserved_field(self, value: int) -> None:
+        """Store a little-endian 32-bit unsigned integer."""
+        self._file_bytes.seek(0x06)
+        self._file_bytes.write(pack("<I", value))
+
+
+@dataclass
+class Header:
+    """Bitmap metadata from the file header."""
+
+    signature: bytes
+    file_size_bytes: int
+    pixel_size_bytes: int
+    pixels_offset: int
+    width: int
+    height: int
+    bit_depth: int
+    compressed: bool
+    has_palette: bool
+
+    def __post_init__(self):
+        assert self.signature == b"BM", "Unknown file signature"
+        assert not self.compressed, "Compression unsupported"
+        assert not self.has_palette, "Color palette unsupported"
+        assert self.bit_depth == 24, "Only 24-bit depth supported"
+
+    @staticmethod
+    def from_bytes(data: bytes) -> "Header":
+        """Factory method to deserialize the header from bytes."""
+        return Header(
+            signature=data[0x00:2],
+            file_size_bytes=unsigned_int(data, 0x02),
+            pixels_offset=unsigned_int(data, 0x0A),
+            width=unsigned_int(data, 0x12),
+            height=unsigned_int(data, 0x16),
+            bit_depth=unsigned_short(data, 0x1C),
+            compressed=unsigned_int(data, 0x1E) != 0,
+            has_palette=unsigned_int(data, 0x2E) != 0,
+            pixel_size_bytes=unsigned_int(data, 0x22),
+        )
+
+
+def unsigned_int(data: Union[bytes, mmap], offset: int) -> int:
+    """Read a little-endian 32-bit unsigned integer."""
+    return unpack("<I", data[offset : offset + 4])[0]
+
+
+def unsigned_short(data: Union[bytes, mmap], offset: int) -> int:
+    """Read a little-endian 16-bit unsigned integer."""
+    return unpack("<H", data[offset : offset + 2])[0]

python-bitwise-operators/stegano/bitmap.py

@@ -0,0 +1,51 @@
+"""
+Command line arguments parsing.
+"""
+
+import argparse
+import pathlib
+from dataclasses import dataclass
+from typing import Optional
+
+
+@dataclass
+class CommandLineArguments:
+    """Parsed command line arguments."""
+
+    bitmap: pathlib.Path
+    encode: Optional[pathlib.Path]
+    decode: bool
+    erase: bool
+
+
+def parse_args() -> CommandLineArguments:
+    """Parse command line arguments."""
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("bitmap", type=path_factory)
+
+    modes = parser.add_mutually_exclusive_group()
+    modes.add_argument("--encode", "-e", metavar="file", type=path_factory)
+    modes.add_argument("--decode", "-d", action="store_true")
+    modes.add_argument("--erase", "-x", action="store_true")
+
+    args = parser.parse_args()
+
+    if not any([args.encode, args.decode, args.erase]):
+        parser.error("Mode required: --encode file | --decode | --erase")
+
+    return CommandLineArguments(**vars(args))
+
+
+def path_factory(argument: str) -> pathlib.Path:
+    """Convert the argument to a path instance."""
+
+    path = pathlib.Path(argument)
+
+    if not path.exists():
+        raise argparse.ArgumentTypeError("file doesn't exist")
+
+    if not path.is_file():
+        raise argparse.ArgumentTypeError("must be a file")
+
+    return path

python-bitwise-operators/stegano/cli.py

@@ -0,0 +1,39 @@
+"""
+Secret file decoder.
+"""
+
+from itertools import islice, takewhile
+from pathlib import Path
+from typing import Iterator
+
+from .bitmap import Bitmap
+
+
+class DecodingError(Exception):
+    pass
+
+
+def decode(bitmap: Bitmap) -> None:
+    """Extract a secret file from the bitmap."""
+
+    if bitmap.reserved_field <= 0:
+        raise DecodingError("Secret file not found in the bitmap")
+
+    iterator = secret_bytes(bitmap)
+
+    filename = "".join(map(chr, takewhile(lambda x: x != 0, iterator)))
+    with Path(filename).open(mode="wb") as file:
+        file.write(bytes(islice(iterator, bitmap.reserved_field)))
+
+    print(f"Extracted a secret file: {filename}")
+
+
+def secret_bytes(bitmap) -> Iterator[int]:
+    """Return an iterator over secret bytes."""
+    for eight_bytes in bitmap.byte_slices:
+        yield sum(
+            [
+                (byte & 1) << (7 - i)
+                for i, byte in enumerate(bitmap[eight_bytes])
+            ]
+        )

python-bitwise-operators/stegano/decoder.py

@@ -0,0 +1,54 @@
+"""
+Secret file encoder.
+"""
+
+import pathlib
+from typing import Iterator
+
+from .bitmap import Bitmap
+
+
+class EncodingError(Exception):
+    pass
+
+
+class SecretFile:
+    """Convenience class for serializing secret data."""
+
+    def __init__(self, path: pathlib.Path):
+        self.path = path
+        self.filename = path.name.encode("utf-8") + b"\x00"
+        self.size_bytes = path.stat().st_size
+
+    @property
+    def num_secret_bytes(self) -> int:
+        """Total number of bytes including the null-terminated string."""
+        return len(self.filename) + self.size_bytes
+
+    @property
+    def secret_bytes(self) -> Iterator[int]:
+        """Null-terminated name followed by the file content."""
+        yield from self.filename
+        with self.path.open(mode="rb") as file:
+            yield from file.read()
+
+
+def encode(bitmap: Bitmap, path: pathlib.Path) -> None:
+    """Embed a secret file in the bitmap."""
+
+    file = SecretFile(path)
+
+    if file.num_secret_bytes > bitmap.max_bytes:
+        raise EncodingError("Not enough pixels to embed a secret file")
+
+    bitmap.reserved_field = file.size_bytes
+    for secret_byte, eight_bytes in zip(file.secret_bytes, bitmap.byte_slices):
+        secret_bits = [(secret_byte >> i) & 1 for i in reversed(range(8))]
+        bitmap[eight_bytes] = bytes(
+            [
+                byte | 1 if bit else byte & ~1
+                for byte, bit in zip(bitmap[eight_bytes], secret_bits)
+            ]
+        )
+
+    print("Secret file was embedded in the bitmap")

python-bitwise-operators/stegano/encoder.py

@@ -0,0 +1,24 @@
+"""
+Secret file eraser.
+"""
+
+from itertools import islice
+from random import random
+
+from .bitmap import Bitmap
+
+
+def erase(bitmap: Bitmap) -> None:
+    """Scramble a previously hidden data."""
+    if bitmap.reserved_field > 0:
+        for byte_offset in islice(bitmap.byte_offsets, bitmap.reserved_field):
+            bitmap[byte_offset] = randomize_lsb(bitmap[byte_offset])
+        bitmap.reserved_field = 0
+        print("Erased a secret file from the bitmap")
+    else:
+        print("Secret file not found in the bitmap")
+
+
+def randomize_lsb(value: int) -> int:
+    """Set a random bit on the least-significant position."""
+    return value & ~1 if random() < 0.5 else value | 1