gen_ift_segmentation_plan.cc

#include <google/protobuf/text_format.h>

#include <chrono>
#include <cstdint>
#include <iostream>
#include <vector>

#include "absl/container/btree_map.h"
#include "absl/container/flat_hash_map.h"
#include "absl/flags/flag.h"
#include "absl/flags/parse.h"
#include "absl/flags/usage.h"
#include "absl/log/globals.h"
#include "absl/log/initialize.h"
#include "absl/status/statusor.h"
#include "absl/strings/str_cat.h"
#include "hb.h"
#include "ift/common/font_data.h"
#include "ift/common/font_helper.h"
#include "ift/common/int_set.h"
#include "ift/common/try.h"
#include "ift/config/auto_segmenter_config.h"
#include "ift/config/load_codepoints.h"
#include "ift/config/segmentation_plan.pb.h"
#include "ift/config/segmenter_config.pb.h"
#include "ift/config/segmenter_config_util.h"
#include "ift/encoder/closure_glyph_segmenter.h"
#include "ift/encoder/glyph_segmentation.h"
#include "ift/encoder/merge_strategy.h"
#include "ift/encoder/subset_definition.h"
#include "ift/freq/unicode_frequencies.h"
#include "util/auto_config_flags.h"

using ift::config::CLOSURE_ONLY;
using ift::config::PATCH;
using ift::config::SegmentationPlan;
using ift::config::SegmenterConfig;

/*
 * Given a code point based segmentation creates an appropriate glyph based
 * segmentation and associated activation conditions that maintain the "closure
 * requirement".
 */

ABSL_FLAG(std::string, input_font, "in.ttf",
          "Name of the font to convert to IFT.");

ABSL_FLAG(
    std::string, config, "auto",
    "Path to a text proto file containing the configuration for the segmenter. "
    "Should contain a single SegmenterConfig message. If set to \"auto\", then "
    "segmenter configuration will be automatically generated "
    "based on the input font.");

ABSL_FLAG(bool, output_segmentation_plan, true,
          "If set a segmentation plan representing the determined segmentation "
          "will be output to stdout. If not set, then a plain text summary of "
          "the segmentation will be output to stdout instead.");

ABSL_FLAG(bool, include_initial_codepoints_in_config, true,
          "If set the generated encoder config will include the initial "
          "codepoint set.");

ABSL_FLAG(bool, output_segmentation_analysis, false,
          "If set an analysis of the segmentation will be output to stderr.");

ABSL_FLAG(
    bool, output_fallback_glyph_count, false,
    "If set the number of fallback glyphs in the segmentation will be output.");

ABSL_FLAG(
    int, verbosity, 0,
    "Log verbosity level from. 0 is least verbose, higher values are more.");

using absl::btree_map;
using absl::btree_set;
using absl::flat_hash_map;
using absl::Status;
using absl::StatusOr;
using absl::StrCat;
using google::protobuf::TextFormat;
using ift::common::CodepointSet;
using ift::common::FontData;
using ift::common::FontHelper;
using ift::common::GlyphSet;
using ift::common::hb_face_unique_ptr;
using ift::common::SegmentSet;
using ift::config::AutoSegmenterConfig;
using ift::config::SegmenterConfigUtil;
using ift::encoder::ClosureGlyphSegmenter;
using ift::encoder::GlyphSegmentation;
using ift::encoder::MergeStrategy;
using ift::encoder::Segment;
using ift::encoder::SegmentationCost;
using ift::encoder::SubsetDefinition;
using ift::freq::UnicodeFrequencies;

static StatusOr<SegmenterConfig> LoadConfig(hb_face_t* font) {
  if (absl::GetFlag(FLAGS_config) == "auto") {
    std::optional<int> quality_level = std::nullopt;
    if (absl::GetFlag(FLAGS_auto_config_quality) > 0) {
      quality_level = absl::GetFlag(FLAGS_auto_config_quality);
    }
    return AutoSegmenterConfig::GenerateConfig(
        font, absl::GetFlag(FLAGS_auto_config_primary_script), quality_level);
  }

  FontData config_text =
      TRY(ift::config::LoadFile(absl::GetFlag(FLAGS_config).c_str()));
  SegmenterConfig config;
  if (!google::protobuf::TextFormat::ParseFromString(config_text.str(),
                                                     &config)) {
    return absl::InvalidArgumentError(StrCat(
        "Failed to parse the input config: ", absl::GetFlag(FLAGS_config)));
  }
  return config;
}

StatusOr<hb_face_unique_ptr> LoadFont(const char* filename) {
  return TRY(ift::config::LoadFile(filename)).face();
}

static Status Analysis(hb_face_t* font,
                       const btree_map<SegmentSet, MergeStrategy>& merge_groups,
                       const GlyphSegmentation& segmentation) {
  unsigned group_index = 0;
  double overall_cost = 0.0;
  for (const auto& [_, strategy] : merge_groups) {
    if (!strategy.UseCosts()) {
      // Can only evaluate costs for strategies that utilize costs.
      continue;
    }

    auto calculator = strategy.ProbabilityCalculator();
    ClosureGlyphSegmenter segmenter(11, 11, PATCH, CLOSURE_ONLY);
    SegmentationCost cost =
        TRY(segmenter.TotalCost(font, segmentation, *calculator));

    overall_cost += cost.total_cost;

    std::cerr << "non_ift_cost_bytes[" << group_index
              << "] = " << (uint64_t)cost.cost_for_non_segmented << std::endl;
    std::cerr << "total_cost_bytes[" << group_index
              << "] = " << (uint64_t)cost.total_cost << std::endl;
    std::cerr << "ideal_cost_bytes[" << group_index
              << "] = " << (uint64_t)cost.ideal_cost << std::endl;
    std::cerr << std::endl;

    group_index++;
  }

  std::cerr << "total_cost_across_groups = " << (uint64_t)overall_cost
            << std::endl;

  return absl::OkStatus();
}

static Status OutputFallbackGlyphCount(hb_face_t* original_face,
                                       const ClosureGlyphSegmenter& segmenter,
                                       const GlyphSegmentation& segmentation) {
  uint32_t num_fallback_glyphs = segmentation.UnmappedGlyphs().size();
  uint32_t fallback_glyphs_size = 0;
  uint32_t all_glyphs_size = 0;
  TRYV(segmenter.FallbackCost(original_face, segmentation, fallback_glyphs_size,
                              all_glyphs_size));

  GlyphSet all_glyphs;
  for (const auto& [_, gids] : segmentation.GidSegments()) {
    all_glyphs.union_set(gids);
  }

  uint32_t num_glyphs = all_glyphs.size() + num_fallback_glyphs;
  std::cout << "num_fallback_glyphs, " << num_fallback_glyphs << ", "
            << num_glyphs << ", " << fallback_glyphs_size << ", "
            << all_glyphs_size << std::endl;

  return absl::OkStatus();
}

static Status Main(const std::vector<char*> args) {
  hb_face_unique_ptr font =
      TRY(LoadFont(absl::GetFlag(FLAGS_input_font).c_str()));
  SegmenterConfig config = TRY(LoadConfig(font.get()));

  SegmenterConfigUtil config_util((absl::GetFlag(FLAGS_config) == "auto")
                                      ? ""
                                      : absl::GetFlag(FLAGS_config));

  auto start_time = std::chrono::high_resolution_clock::now();
  auto result = TRY(config_util.RunSegmenter(font.get(), config));
  auto end_time = std::chrono::high_resolution_clock::now();
  std::chrono::duration<double> duration = end_time - start_time;
  std::cerr << "CodepointToGlyphSegments took: " << duration.count()
            << " seconds" << std::endl;

  GlyphSegmentation segmentation = std::move(result.segmentation);
  SegmentationPlan plan = std::move(result.plan);

  if (absl::GetFlag(FLAGS_output_segmentation_plan)) {
    if (!absl::GetFlag(FLAGS_include_initial_codepoints_in_config)) {
      // Requested to not include init codepoints in the generated config.
      plan.clear_initial_codepoints();
    }

    // TODO(garretrieger): assign a basic (single segment) table keyed config.
    // Later on the input to this util should include information on how the
    // segments should be grouped together for the table keyed portion of the
    // font.
    std::string config_string;
    TextFormat::PrintToString(plan, &config_string);
    std::cout << config_string;
  } else {
    // No config requested, just output a simplified plain text representation
    // of the segmentation.
    std::cout << segmentation.ToString() << std::endl;
  }

  if (absl::GetFlag(FLAGS_output_fallback_glyph_count)) {
    ClosureGlyphSegmenter segmenter(
        config.brotli_quality(),
        config.brotli_quality_for_initial_font_merging(),
        config.unmapped_glyph_handling(), config.condition_analysis_mode());
    TRYV(OutputFallbackGlyphCount(font.get(), segmenter, segmentation));
  }

  if (!absl::GetFlag(FLAGS_output_segmentation_analysis)) {
    return absl::OkStatus();
  }

  std::cerr << ">> Analysis" << std::endl;
  return Analysis(font.get(), result.merge_groups, segmentation);
}

int main(int argc, char** argv) {
  absl::SetProgramUsageMessage(
      "Generates a segmentation plan for a font.\n"
      "\n"
      "Usage: gen_ift_segmentation_plan --input_font=\"myfont.ttf\" "
      "[--config=\"config.txtpb\"]\n");
  absl::SetStderrThreshold(absl::LogSeverityAtLeast::kInfo);
  absl::SetGlobalVLogLevel(absl::GetFlag(FLAGS_verbosity));
  auto args = absl::ParseCommandLine(argc, argv);
  absl::InitializeLog();

  auto sc = Main(args);
  if (!sc.ok()) {
    std::cerr << "Error: " << sc << std::endl;
    return -1;
  }
  return 0;
}