fp_http.c

/*
   p0f - HTTP fingerprinting
   -------------------------

   Copyright (C) 2012 by Michal Zalewski <lcamtuf@coredump.cx>

   Distributed under the terms and conditions of GNU LGPL.

 */

#define _FROM_FP_HTTP
#define _GNU_SOURCE

#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <ctype.h>
#include <time.h>

#include <netinet/in.h>
#include <sys/types.h>

#include "types.h"
#include "config.h"
#include "debug.h"
#include "alloc-inl.h"
#include "process.h"
#include "readfp.h"
#include "p0f.h"
#include "tcp.h"
#include "hash.h"

#include "fp_http.h"
#include "languages.h"

static u8** hdr_names;                 /* List of header names by ID         */
static u32  hdr_cnt;                   /* Number of headers registered       */

static u32* hdr_by_hash[SIG_BUCKETS];  /* Hashed header names                */
static u32  hbh_cnt[SIG_BUCKETS];      /* Number of headers in bucket        */

/* Signatures aren't bucketed due to the complex matching used; but we use
   Bloom filters to go through them quickly. */

static struct http_sig_record* sigs[2];
static u32 sig_cnt[2];

static struct ua_map_record* ua_map;   /* Mappings between U-A and OS        */
static u32 ua_map_cnt;

#define SLOF(_str) (u8*)_str, strlen((char*)_str)


/* Ghetto Bloom filter 4-out-of-64 bitmask generator for adding 32-bit header
   IDs to a set. We expect around 10 members in a set. */

static inline u64 bloom4_64(u32 val) {
  u32 hash = hash32(&val, 4, hash_seed);
  u64 ret;
  ret = (1LL << (hash & 63));
  ret ^= (1LL << ((hash >> 8) & 63));
  ret ^= (1LL << ((hash >> 16) & 63));
  ret ^= (1LL << ((hash >> 24) & 63));
  return ret;
}


/* Look up or register new header */

static s32 lookup_hdr(u8* name, u32 len, u8 create) {

  u32  bucket = hash32(name, len, hash_seed) % SIG_BUCKETS;

  u32* p = hdr_by_hash[bucket];
  u32  i = hbh_cnt[bucket];

  while (i--) {
    if (!memcmp(hdr_names[*p], name, len) /* ASAN won't like this... */ &&
        !hdr_names[*p][len]) return *p;
    p++;
  }

  /* Not found! */

  if (!create) return -1;

  hdr_names = DFL_ck_realloc(hdr_names, (hdr_cnt + 1) * sizeof(u8*));
  hdr_names[hdr_cnt] = DFL_ck_memdup_str(name, len);

  hdr_by_hash[bucket] = DFL_ck_realloc(hdr_by_hash[bucket],
    (hbh_cnt[bucket] + 1) * 4);

  hdr_by_hash[bucket][hbh_cnt[bucket]++] = hdr_cnt++;

  return hdr_cnt - 1;

}


/* Pre-register essential headers. */

void http_init(void) {
  u32 i;

  /* Do not change - other code depends on the ordering of first 6 entries. */

  lookup_hdr(SLOF("User-Agent"), 1);      /* 0 */
  lookup_hdr(SLOF("Server"), 1);          /* 1 */
  lookup_hdr(SLOF("Accept-Language"), 1); /* 2 */
  lookup_hdr(SLOF("Via"), 1);             /* 3 */
  lookup_hdr(SLOF("X-Forwarded-For"), 1); /* 4 */
  lookup_hdr(SLOF("Date"), 1);            /* 5 */

#define HDR_UA  0
#define HDR_SRV 1
#define HDR_AL  2
#define HDR_VIA 3
#define HDR_XFF 4
#define HDR_DAT 5

  i = 0;
  while (req_optional[i].name) {
    req_optional[i].id = lookup_hdr(SLOF(req_optional[i].name), 1);
    i++;
  }

  i = 0;
  while (resp_optional[i].name) {
    resp_optional[i].id = lookup_hdr(SLOF(resp_optional[i].name), 1);
    i++;
  }

  i = 0;
  while (req_skipval[i].name) {
    req_skipval[i].id = lookup_hdr(SLOF(req_skipval[i].name), 1);
    i++;
  }

  i = 0;
  while (resp_skipval[i].name) {
    resp_skipval[i].id = lookup_hdr(SLOF(resp_skipval[i].name), 1);
    i++;
  }

  i = 0;
  while (req_common[i].name) {
    req_common[i].id = lookup_hdr(SLOF(req_common[i].name), 1);
    i++;
  }

  i = 0;
  while (resp_common[i].name) {
    resp_common[i].id = lookup_hdr(SLOF(resp_common[i].name), 1);
    i++;
  }

}


/* Find match for a signature. */

static void http_find_match(u8 to_srv, struct http_sig* ts, u8 dupe_det) {

  struct http_sig_record* gmatch = NULL;
  struct http_sig_record* ref = sigs[to_srv];
  u32 cnt = sig_cnt[to_srv];

  while (cnt--) {

    struct http_sig* rs = ref->sig;
    u32 ts_hdr = 0, rs_hdr = 0;

    if (rs->http_ver != -1 && rs->http_ver != ts->http_ver) goto next_sig;

    /* Check that all the headers listed for the p0f.fp signature (probably)
       appear in the examined traffic. */

    if ((ts->hdr_bloom4 & rs->hdr_bloom4) != rs->hdr_bloom4) goto next_sig;

    /* Confirm the ordering and values of headers (this is relatively
       slow, hence the Bloom filter first). */

    while (rs_hdr < rs->hdr_cnt) {

      u32 orig_ts = ts_hdr;

      while (rs->hdr[rs_hdr].id != ts->hdr[ts_hdr].id &&
             ts_hdr < ts->hdr_cnt) ts_hdr++;

      if (ts_hdr == ts->hdr_cnt) {

        if (!rs->hdr[rs_hdr].optional) goto next_sig;

        /* If this is an optional header, check that it doesn't appear
           anywhere else. */

        for (ts_hdr = 0; ts_hdr < ts->hdr_cnt; ts_hdr++)
          if (rs->hdr[rs_hdr].id == ts->hdr[ts_hdr].id) goto next_sig;

        ts_hdr = orig_ts;
        rs_hdr++;
        continue;

      }

      if (rs->hdr[rs_hdr].value &&
          (!ts->hdr[ts_hdr].value ||
          !strstr((char*)ts->hdr[ts_hdr].value,
          (char*)rs->hdr[rs_hdr].value))) goto next_sig;

      ts_hdr++;
      rs_hdr++;

    }

    /* Check that the headers forbidden in p0f.fp don't appear in the traffic.
       We first check if they seem to appear in ts->hdr_bloom4, and only if so,
       we do a full check. */

    for (rs_hdr = 0; rs_hdr < rs->miss_cnt; rs_hdr++) {

      u64 miss_bloom4 = bloom4_64(rs->miss[rs_hdr]);

      if ((ts->hdr_bloom4 & miss_bloom4) != miss_bloom4) continue;

      /* Okay, possible instance of a banned header - scan list... */

      for (ts_hdr = 0; ts_hdr < ts->hdr_cnt; ts_hdr++)
        if (rs->miss[rs_hdr] == ts->hdr[ts_hdr].id) goto next_sig;

    }

    /* When doing dupe detection, we want to allow a signature with additional
       banned headers to precede one with fewer, or with a different set. */

    if (dupe_det) {

      if (rs->miss_cnt > ts->miss_cnt) goto next_sig;

      for (rs_hdr = 0; rs_hdr < rs->miss_cnt; rs_hdr++) {

        for (ts_hdr = 0; ts_hdr < ts->miss_cnt; ts_hdr++) 
          if (rs->miss[rs_hdr] == ts->miss[ts_hdr]) break;

        /* One of the reference headers doesn't appear in current sig! */

        if (ts_hdr == ts->miss_cnt) goto next_sig;

      }


    }

    /* Whoa, a match. */

    if (!ref->generic) {

      ts->matched = ref;

      if (rs->sw && ts->sw && !strstr((char*)ts->sw, (char*)rs->sw))
        ts->dishonest = 1;

      return;

    } else if (!gmatch) gmatch = ref;

next_sig:

    ref = ref + 1;

  }

  /* A generic signature is the best we could find. */

  if (!dupe_det && gmatch) {

    ts->matched = gmatch;

    if (gmatch->sig->sw && ts->sw && !strstr((char*)ts->sw,
        (char*)gmatch->sig->sw)) ts->dishonest = 1;

  }

}


/* Register new HTTP signature. */

void http_register_sig(u8 to_srv, u8 generic, s32 sig_class, u32 sig_name,
                       u8* sig_flavor, u32 label_id, u32* sys, u32 sys_cnt,
                       u8* val, u32 line_no) {

  struct http_sig* hsig;
  struct http_sig_record* hrec;

  u8* nxt;

  hsig = DFL_ck_alloc(sizeof(struct http_sig));

  sigs[to_srv] = DFL_ck_realloc(sigs[to_srv], sizeof(struct http_sig_record) *
    (sig_cnt[to_srv] + 1));

  hrec = &sigs[to_srv][sig_cnt[to_srv]];

  if (val[1] != ':') FATAL("Malformed signature in line %u.", line_no);

  /* http_ver */

  switch (*val) {
    case '0': break;
    case '1': hsig->http_ver = 1; break;
    case '*': hsig->http_ver = -1; break;
    default: FATAL("Bad HTTP version in line %u.", line_no);
  }

  val += 2;

  /* horder */

  while (*val != ':') {

    u32 id;
    u8 optional = 0;

    if (hsig->hdr_cnt >= HTTP_MAX_HDRS)
      FATAL("Too many headers listed in line %u.", line_no);

    nxt = val;

    if (*nxt == '?') { optional = 1; val++; nxt++; }

    while (isalnum(*nxt) || *nxt == '-' || *nxt == '_') nxt++;

    if (val == nxt)
      FATAL("Malformed header name in line %u.", line_no);

    id = lookup_hdr(val, nxt - val, 1);

    hsig->hdr[hsig->hdr_cnt].id = id;
    hsig->hdr[hsig->hdr_cnt].optional = optional;

    if (!optional) hsig->hdr_bloom4 |= bloom4_64(id);

    val = nxt;

    if (*val == '=') {

      if (val[1] != '[') 
        FATAL("Missing '[' after '=' in line %u.", line_no);

      val += 2;
      nxt = val;

      while (*nxt && *nxt != ']') nxt++;

      if (val == nxt || !*nxt)
        FATAL("Malformed signature in line %u.", line_no);

      hsig->hdr[hsig->hdr_cnt].value = DFL_ck_memdup_str(val, nxt - val);

      val = nxt + 1;

    }

    hsig->hdr_cnt++;

    if (*val == ',') val++; else if (*val != ':')
      FATAL("Malformed signature in line %u.", line_no);

  }

  val++;

  /* habsent */

  while (*val != ':') {

    u32 id;

    if (hsig->miss_cnt >= HTTP_MAX_HDRS)
      FATAL("Too many headers listed in line %u.", line_no);

    nxt = val;
    while (isalnum(*nxt) || *nxt == '-' || *nxt == '_') nxt++;

    if (val == nxt)
      FATAL("Malformed header name in line %u.", line_no);

    id = lookup_hdr(val, nxt - val, 1);

    hsig->miss[hsig->miss_cnt] = id;

    val = nxt;

    hsig->miss_cnt++;

    if (*val == ',') val++; else if (*val != ':')
      FATAL("Malformed signature in line %u.", line_no);

  }

  val++;

  /* exp_sw */

  if (*val) {

    if (strchr((char*)val, ':'))
      FATAL("Malformed signature in line %u.", line_no);

    hsig->sw = DFL_ck_strdup(val);

  }

  http_find_match(to_srv, hsig, 1);

  if (hsig->matched)
    FATAL("Signature in line %u is already covered by line %u.",
          line_no, hsig->matched->line_no);

  hrec->class_id = sig_class;
  hrec->name_id  = sig_name;
  hrec->flavor   = sig_flavor;
  hrec->label_id = label_id;
  hrec->sys      = sys;
  hrec->sys_cnt  = sys_cnt;
  hrec->line_no  = line_no;
  hrec->generic  = generic;

  hrec->sig      = hsig;

  sig_cnt[to_srv]++;

}


/* Register new HTTP signature. */

void http_parse_ua(u8* val, u32 line_no) {

  u8* nxt;

  while (*val) {

    u32 id;
    u8* name = NULL;

    nxt = val;
    while (*nxt && (isalnum(*nxt) || strchr(NAME_CHARS, *nxt))) nxt++;

    if (val == nxt)
      FATAL("Malformed system name in line %u.", line_no);

    id = lookup_name_id(val, nxt - val);

    val = nxt;

    if (*val == '=') {

      if (val[1] != '[') 
        FATAL("Missing '[' after '=' in line %u.", line_no);

      val += 2;
      nxt = val;

      while (*nxt && *nxt != ']') nxt++;

      if (val == nxt || !*nxt)
        FATAL("Malformed signature in line %u.", line_no);

      name = DFL_ck_memdup_str(val, nxt - val);

      val = nxt + 1;

    }

    ua_map = DFL_ck_realloc(ua_map, (ua_map_cnt + 1) *
                        sizeof(struct ua_map_record));

    ua_map[ua_map_cnt].id = id;
   
    if (!name) ua_map[ua_map_cnt].name = fp_os_names[id];
      else ua_map[ua_map_cnt].name = name;

    ua_map_cnt++;

    if (*val == ',') val++;

  }

}


/* Dump a HTTP signature. */

static u8* dump_sig(u8 to_srv, struct http_sig* hsig) {

  u32 i;
  u8 had_prev = 0;
  struct http_id* list;

  u8 tmp[HTTP_MAX_SHOW + 1];
  u32 tpos;

  static u8* ret;
  u32 rlen = 0;

  u8* val;

#define RETF(_par...) do { \
    s32 _len = snprintf(NULL, 0, _par); \
    if (_len < 0) FATAL("Whoa, snprintf() fails?!"); \
    ret = DFL_ck_realloc_kb(ret, rlen + _len + 1); \
    snprintf((char*)ret + rlen, _len + 1, _par); \
    rlen += _len; \
  } while (0)
    
  RETF("%u:", hsig->http_ver);

  for (i = 0; i < hsig->hdr_cnt; i++) {

    if (hsig->hdr[i].id >= 0) {

      u8 optional = 0;

      /* Check the "optional" list. */

      list = to_srv ? req_optional : resp_optional;

      while (list->name) {
        if (list->id == hsig->hdr[i].id) break;
        list++;
      }

      if (list->name) optional = 1;

      RETF("%s%s%s", had_prev ? "," : "", optional ? "?" : "",
           hdr_names[hsig->hdr[i].id]);
      had_prev = 1;

      if (!(val = hsig->hdr[i].value)) continue;

      /* Next, make sure that the value is not on the ignore list. */

      if (optional) continue;

      list = to_srv ? req_skipval : resp_skipval;

      while (list->name) {
        if (list->id == hsig->hdr[i].id) break;
        list++;
      }

      if (list->name) continue;

      /* Looks like it's not on the list, so let's output a cleaned-up version
         up to HTTP_MAX_SHOW. */

      tpos = 0;

      while (tpos < HTTP_MAX_SHOW && val[tpos] >= 0x20 && val[tpos] < 0x80 &&
             val[tpos] != ']' && val[tpos] != '|') {

        tmp[tpos] = val[tpos];
        tpos++;

      }

      tmp[tpos] = 0;

      if (!tpos) continue;

      RETF("=[%s]", tmp);

    } else {

      RETF("%s%s", had_prev ? "," : "", hsig->hdr[i].name);
      had_prev = 1;

      if (!(val = hsig->hdr[i].value)) continue;

      tpos = 0;

      while (tpos < HTTP_MAX_SHOW && val[tpos] >= 0x20 && val[tpos] < 0x80 &&
             val[tpos] != ']') { tmp[tpos] = val[tpos]; tpos++; }

      tmp[tpos] = 0;

      if (!tpos) continue;

      RETF("=[%s]", tmp);

    }

  }

  RETF(":");

  list = to_srv ? req_common : resp_common;
  had_prev = 0;

  while (list->name) {

    for (i = 0; i < hsig->hdr_cnt; i++) 
      if (hsig->hdr[i].id == list->id) break;

    if (i == hsig->hdr_cnt) {
      RETF("%s%s", had_prev ? "," : "", list->name);
      had_prev = 1;
    }

    list++;

  }

  RETF(":");

  if ((val = hsig->sw)) {

    tpos = 0;

    while (tpos < HTTP_MAX_SHOW &&  val[tpos] >= 0x20 && val[tpos] < 0x80 &&
           val[tpos] != ']') { tmp[tpos] = val[tpos]; tpos++; }

    tmp[tpos] = 0;

    if (tpos) RETF("%s", tmp);

  }

  return ret;


}


/* Dump signature flags. */

static u8* dump_flags(struct http_sig* hsig, struct http_sig_record* m) {

  static u8* ret;
  u32 rlen = 0;

  RETF("");

  if (hsig->dishonest) RETF(" dishonest");
  if (!hsig->sw) RETF(" anonymous");
  if (m && m->generic) RETF(" generic");

#undef RETF

  if (*ret) return ret + 1; else return (u8*)"none";

}


/* Score signature differences. For unknown signatures, the presumption is that
   they identify apps, so the logic is quite different from TCP. */

static void score_nat(u8 to_srv, struct packet_flow* f) {

  struct http_sig_record* m = f->http_tmp.matched;
  struct host_data* hd;
  struct http_sig* ref;

  u8  score = 0, diff_already = 0;
  u16 reason = 0;

  if (to_srv) {

    hd = f->client;
    ref = hd->http_req_os;

  } else {

    hd = f->server;
    ref = hd->http_resp;

    /* If the signature is for a different port, don't read too much into it. */

    if (hd->http_resp_port != f->srv_port) ref = NULL;

  }

  if (!m) {

    /* No match. The user is probably running another app; this is only of
       interest if a server progresses from known to unknown. We can't
       compare two unknown server sigs with that much confidence. */

    if (!to_srv && ref && ref->matched) {

      DEBUG("[#] HTTP server signature changed from known to unknown.\n");
      score  += 4;
      reason |= NAT_TO_UNK;

    }

    goto header_check;

  }

  if (m->class_id == -1) {

    /* Got a match for an application signature. Make sure it runs on the
       OS we have on file... */

    verify_tool_class(to_srv, f, m->sys, m->sys_cnt);

    /* ...and check for inconsistencies in server behavior. */

    if (!to_srv && ref && ref->matched) {

      if (ref->matched->name_id != m->name_id) {

        DEBUG("[#] Name on the matched HTTP server signature changes.\n");
        score  += 8;
        reason |= NAT_APP_LB;

      } else if (ref->matched->label_id != m->label_id) {

        DEBUG("[#] Label on the matched HTTP server signature changes.\n");
        score  += 2;
        reason |= NAT_APP_LB;

      }

    }

  } else {

    /* Ooh, spiffy: a match for an OS signature! There will be about two uses
       for this code, ever. */

    if (ref && ref->matched) {

      if (ref->matched->name_id != m->name_id) {

        DEBUG("[#] Name on the matched HTTP OS signature changes.\n");
        score  += 8;
        reason |= NAT_OS_SIG;
        diff_already = 1;

      } else if (ref->matched->name_id != m->label_id) {

        DEBUG("[#] Label on the matched HTTP OS signature changes.\n");
        score  += 2;
        reason |= NAT_OS_SIG;

      }

    } else if (ref) {

      DEBUG("[#] HTTP OS signature changed from unknown to known.\n");
      score  += 4;
      reason |= NAT_TO_UNK;

    }

    /* If we haven't pointed out anything major yet, also complain if the
       signature doesn't match host data. */

    if (!diff_already && hd->last_name_id != m->name_id) {

      DEBUG("[#] Matched HTTP OS signature different than host data.\n");
      score  += 4;
      reason |= NAT_OS_SIG;

    }

  }

  /* If we have determined that U-A looks legit, but the OS doesn't match,
     that's a clear sign of trouble. */

  if (to_srv && m->class_id == -1 && f->http_tmp.sw && !f->http_tmp.dishonest) {

    u32 i;

    for (i = 0; i < ua_map_cnt; i++)
      if (strstr((char*)f->http_tmp.sw, (char*)ua_map[i].name)) break;

    if (i != ua_map_cnt) {

      if (ua_map[i].id != hd->last_name_id) {

        DEBUG("[#] Otherwise plausible User-Agent points to another OS.\n");
        score  += 4;
        reason |= NAT_APP_UA;

        if (!hd->bad_sw) hd->bad_sw = 1;

      } else {

        DEBUG("[#] User-Agent OS value checks out.\n");
        hd->bad_sw = 0;

      }

    }

  }


header_check:

  /* Okay, some last-resort checks. This is obviously concerning: */

  if (f->http_tmp.via) {
    DEBUG("[#] Explicit use of Via or X-Forwarded-For.\n");
    score  += 8;
    reason |= NAT_APP_VIA;
  }

  /* Last but not least, see what happened to 'Date': */

  if (ref && !to_srv && ref->date && f->http_tmp.date) {

    s64 recv_diff = ((s64)f->http_tmp.recv_date) - ref->recv_date;
    s64 hdr_diff  = ((s64)f->http_tmp.date) - ref->date;

    if (hdr_diff < -HTTP_MAX_DATE_DIFF || 
        hdr_diff > recv_diff + HTTP_MAX_DATE_DIFF) {

      DEBUG("[#] HTTP 'Date' distance too high (%lld in %lld sec).\n",
             hdr_diff, recv_diff);
      score  += 4;
      reason |= NAT_APP_DATE;

    } else {

      DEBUG("[#] HTTP 'Date' distance seems fine (%lld in %lld sec).\n",
             hdr_diff, recv_diff);

    }

  }

  add_nat_score(to_srv, f, reason, score);

}


/* Look up HTTP signature, create an observation. */

static void fingerprint_http(u8 to_srv, struct packet_flow* f) {

  struct http_sig_record* m;
  u8* lang = NULL;

  http_find_match(to_srv, &f->http_tmp, 0);

  start_observation(to_srv ? "http request" : "http response", 4, to_srv, f);

  if ((m = f->http_tmp.matched)) {

    OBSERVF((m->class_id < 0) ? "app" : "os", "%s%s%s",
            fp_os_names[m->name_id], m->flavor ? " " : "",
            m->flavor ? m->flavor : (u8*)"");

  } else add_observation_field("app", NULL);

  if (f->http_tmp.lang && isalpha(f->http_tmp.lang[0]) &&
      isalpha(f->http_tmp.lang[1]) && !isalpha(f->http_tmp.lang[2])) {

    u8 lh = LANG_HASH(f->http_tmp.lang[0], f->http_tmp.lang[1]);
    u8 pos = 0;

    while (languages[lh][pos]) {
      if (f->http_tmp.lang[0] == languages[lh][pos][0] &&
          f->http_tmp.lang[1] == languages[lh][pos][1]) break;
      pos += 2;
    }

    if (!languages[lh][pos]) add_observation_field("lang", NULL);
      else add_observation_field("lang", 
           (lang = (u8*)languages[lh][pos + 1]));

  } else add_observation_field("lang", (u8*)"none");

  add_observation_field("params", dump_flags(&f->http_tmp, m));

  add_observation_field("raw_sig", dump_sig(to_srv, &f->http_tmp));

  score_nat(to_srv, f);

  /* Save observations needed to score future responses. */

  if (!to_srv) {

    /* For server response, always store the signature. */

    ck_free(f->server->http_resp);
    f->server->http_resp = ck_memdup(&f->http_tmp, sizeof(struct http_sig));

    f->server->http_resp->hdr_cnt = 0;
    f->server->http_resp->sw   = NULL;
    f->server->http_resp->lang = NULL;
    f->server->http_resp->via  = NULL;

    f->server->http_resp_port = f->srv_port;

    if (lang) f->server->language = lang;

    if (m) {

      if (m->class_id != -1) {

        /* If this is an OS signature, update host record. */

        f->server->last_class_id = m->class_id;
        f->server->last_name_id  = m->name_id;
        f->server->last_flavor   = m->flavor;
        f->server->last_quality  = (m->generic * P0F_MATCH_GENERIC);

      } else {

        /* Otherwise, record app data. */

        f->server->http_name_id = m->name_id;
        f->server->http_flavor  = m->flavor;

        if (f->http_tmp.dishonest) f->server->bad_sw = 2;

      }

    }

  } else {

    if (lang) f->client->language = lang;

    if (m) {

      if (m->class_id != -1) {

        /* Client request - only OS sig is of any note. */

        ck_free(f->client->http_req_os);
        f->client->http_req_os = ck_memdup(&f->http_tmp,
          sizeof(struct http_sig));

        f->client->http_req_os->hdr_cnt = 0;
        f->client->http_req_os->sw   = NULL;
        f->client->http_req_os->lang = NULL;
        f->client->http_req_os->via  = NULL;

        f->client->last_class_id = m->class_id;
        f->client->last_name_id  = m->name_id;
        f->client->last_flavor   = m->flavor;

        f->client->last_quality  = (m->generic * P0F_MATCH_GENERIC);

      } else {

        /* Record app data for the API. */

        f->client->http_name_id = m->name_id;
        f->client->http_flavor  = m->flavor;

        if (f->http_tmp.dishonest) f->client->bad_sw = 2;

      }
 
    }

  }

}



/* Free up any allocated strings in http_sig. */

void free_sig_hdrs(struct http_sig* h) {

  u32 i;

  for (i = 0; i < h->hdr_cnt; i++) {
    if (h->hdr[i].name) ck_free(h->hdr[i].name);
    if (h->hdr[i].value) ck_free(h->hdr[i].value);
  }

}


/* Parse HTTP date field. */

static u32 parse_date(u8* str) {
  struct tm t;

  if (!strptime((char*)str, "%a, %d %b %Y %H:%M:%S %Z", &t)) {
    DEBUG("[#] Invalid 'Date' field ('%s').\n", str);
    return 0;
  }

  return mktime(&t);

}


/* Parse name=value pairs into a signature. */

static u8 parse_pairs(u8 to_srv, struct packet_flow* f, u8 can_get_more) {

  u32 plen = to_srv ? f->req_len : f->resp_len;

  u32 off;

  /* Try to parse name: value pairs. */

  while ((off = f->http_pos) < plen) {

    u8* pay = to_srv ? f->request : f->response;

    u32 nlen, vlen, vstart;
    s32 hid;
    u32 hcount;

    /* Empty line? Dispatch for fingerprinting! */

    if (pay[off] == '\r' || pay[off] == '\n') {

      f->http_tmp.recv_date = get_unix_time();

      fingerprint_http(to_srv, f);

      /* If this is a request, flush the collected signature and prepare
         for parsing the response. If it's a response, just shut down HTTP
         parsing on this flow. */

      if (to_srv) {

        f->http_req_done = 1;
        f->http_pos = 0;

        free_sig_hdrs(&f->http_tmp);
        memset(&f->http_tmp, 0, sizeof(struct http_sig));

        return 1;

      } else {

        f->in_http = -1;
        return 0;

      }

    }

    /* Looks like we're getting a header value. See if we have room for it. */

    if ((hcount = f->http_tmp.hdr_cnt) >= HTTP_MAX_HDRS) {

      DEBUG("[#] Too many HTTP headers in a %s.\n", to_srv ? "request" :
            "response");

      f->in_http = -1;
      return 0;

    }

    /* Try to extract header name. */
      
    nlen = 0;

    while ((isalnum(pay[off]) || pay[off] == '-' || pay[off] == '_') &&
           off < plen && nlen <= HTTP_MAX_HDR_NAME) {

      off++;
      nlen++;

    }

    if (off == plen) {

      if (!can_get_more) {

        DEBUG("[#] End of HTTP %s before end of headers.\n",
              to_srv ? "request" : "response");
        f->in_http = -1;

      }

      return can_get_more;

    }

    /* Empty, excessively long, or non-':'-followed header name? */

    if (!nlen || pay[off] != ':' || nlen > HTTP_MAX_HDR_NAME) {

      DEBUG("[#] Invalid HTTP header encountered (len = %u, char = 0x%02x).\n",
            nlen, pay[off]);

      f->in_http = -1;
      return 0;

    }

    /* At this point, header name starts at f->http_pos, and has nlen bytes.
       Skip ':' and a subsequent whitespace next. */

    off++;

    if (off < plen && isblank(pay[off])) off++;

    vstart = off;
    vlen = 0;

    /* Find the next \n. */

    while (off < plen && vlen <= HTTP_MAX_HDR_VAL && pay[off] != '\n') {

      off++;
      vlen++;

    }

    if (vlen > HTTP_MAX_HDR_VAL) {
      DEBUG("[#] HTTP %s header value length exceeded.\n",
            to_srv ? "request" : "response");
      f->in_http = -1;
      return -1;
    }

    if (off == plen) {

      if (!can_get_more) {
        DEBUG("[#] End of HTTP %s before end of headers.\n",
              to_srv ? "request" : "response");
        f->in_http = -1;
      }

      return can_get_more;

    }

    /* If party is using \r\n terminators, go back one char. */

    if (pay[off - 1] == '\r') vlen--;
 
    /* Header value starts at vstart, and has vlen bytes (may be zero). Record
       this in the signature. */

    hid = lookup_hdr(pay + f->http_pos, nlen, 0);

    f->http_tmp.hdr[hcount].id = hid;

    if (hid < 0) {

      /* Header ID not found, store literal value. */

      f->http_tmp.hdr[hcount].name = ck_memdup_str(pay + f->http_pos, nlen);

    } else {

      /* Found - update Bloom filter. */

      f->http_tmp.hdr_bloom4 |= bloom4_64(hid);

    }

    /* If there's a value, store that too. For U-A and Server, also update
       'sw'; and for requests, collect Accept-Language. */

    if (vlen) {

      u8* val = ck_memdup_str(pay + vstart, vlen);

      f->http_tmp.hdr[hcount].value = val;

      if (to_srv) {

        switch (hid) {
          case HDR_UA: f->http_tmp.sw = val; break;
          case HDR_AL: f->http_tmp.lang = val; break;
          case HDR_VIA:
          case HDR_XFF: f->http_tmp.via = val; break;
        }

      } else {

        switch (hid) {

          case HDR_SRV: f->http_tmp.sw = val; break;
          case HDR_DAT: f->http_tmp.date = parse_date(val); break;
          case HDR_VIA:
          case HDR_XFF: f->http_tmp.via = val; break;

        }

      }

    }

    /* Moving on... */

    f->http_tmp.hdr_cnt++;
    f->http_pos = off + 1; 

  }

  if (!can_get_more) {
    DEBUG("[#] End of HTTP %s before end of headers.\n",
          to_srv ? "request" : "response");
    f->in_http = -1;
  }

  return can_get_more;

}


/* Examine request or response; returns 1 if more data needed and plausibly can
   be read. Note that the buffer is always NUL-terminated. */

u8 process_http(u8 to_srv, struct packet_flow* f) {

  /* Already decided this flow is not worth tracking? */

  if (f->in_http < 0) return 0;

  if (to_srv) {

    u8* pay = f->request;
    u8 can_get_more = (f->req_len < MAX_FLOW_DATA);
    u32 off;

    /* Request done, but pending response? */

    if (f->http_req_done) return 1;

    if (!f->in_http) {

      u8 chr;
      u8* sig_at;

      /* Ooh, new flow! */

      if (f->req_len < 15) return can_get_more;

      /* Scan until \n, or until binary data spotted. */

      off = f->http_pos;

      /* We only care about GET and HEAD requests at this point. */

      if (!off && strncmp((char*)pay, "GET /", 5) &&
          strncmp((char*)pay, "HEAD /", 6)) {
        DEBUG("[#] Does not seem like a GET / HEAD request.\n");
        f->in_http = -1;
        return 0;
      }

      while (off < f->req_len && off < HTTP_MAX_URL &&
             (chr = pay[off]) != '\n') {

        if (chr != '\r' && (chr < 0x20 || chr > 0x7f)) {

          DEBUG("[#] Not HTTP - character 0x%02x encountered.\n", chr);

          f->in_http = -1;
          return 0;
        }

        off++;
  
      }

      /* Newline too far or too close? */

      if (off == HTTP_MAX_URL || off < 14) {

        DEBUG("[#] Not HTTP - newline offset %u.\n", off);

        f->in_http = -1;
        return 0;

      }

      /* Not enough data yet? */

      if (off == f->req_len) {

        f->http_pos = off;

        if (!can_get_more) {

          DEBUG("[#] Not HTTP - no opening line found.\n");
          f->in_http = -1;

        }

        return can_get_more;

      }

      sig_at = pay + off - 8;
      if (pay[off - 1] == '\r') sig_at--;

      /* Bad HTTP/1.x signature? */

      if (strncmp((char*)sig_at, "HTTP/1.", 7)) {

        DEBUG("[#] Not HTTP - bad signature.\n");

        f->in_http = -1;
        return 0;

      }

      f->http_tmp.http_ver = (sig_at[7] == '1');

      f->in_http  = 1;
      f->http_pos = off + 1;

      DEBUG("[#] HTTP detected.\n");

    }

    return parse_pairs(1, f, can_get_more);

  } else {

    u8* pay = f->response;
    u8 can_get_more = (f->resp_len < MAX_FLOW_DATA);
    u32 off;

    /* Response before request? Bail out. */

    if (!f->in_http || !f->http_req_done) {
      f->in_http = -1;
      return 0;
    }

    if (!f->http_gotresp1) {

      u8 chr;

      if (f->resp_len < 13) return can_get_more;

      /* Scan until \n, or until binary data spotted. */

      off = f->http_pos;

      while (off < f->resp_len && off < HTTP_MAX_URL &&
             (chr = pay[off]) != '\n') {

        if (chr != '\r' && (chr < 0x20 || chr > 0x7f)) {

          DEBUG("[#] Invalid HTTP response - character 0x%02x encountered.\n",
                chr);
          f->in_http = -1;
          return 0;

        }

        off++;
  
      }

      /* Newline too far or too close? */

      if (off == HTTP_MAX_URL || off < 13) {

        DEBUG("[#] Invalid HTTP response - newline offset %u.\n", off);

        f->in_http = -1;
        return 0;

      }

      /* Not enough data yet? */

      if (off == f->resp_len) {

        f->http_pos = off;

        if (!can_get_more) {

          DEBUG("[#] Invalid HTTP response - no opening line found.\n");
          f->in_http = -1;

        }

        return can_get_more;

      }

      /* Bad HTTP/1.x signature? */

      if (strncmp((char*)pay, "HTTP/1.", 7)) {

        DEBUG("[#] Invalid HTTP response - bad signature.\n");

        f->in_http = -1;
        return 0;

      }

      f->http_tmp.http_ver = (pay[7] == '1');

      f->http_pos = off + 1;

      DEBUG("[#] HTTP response starts correctly.\n");


    }

    return parse_pairs(0, f, can_get_more);

  }

}