Merge branch 'master' into release/0.1

Author: pjenvey

.gitignore

@@ -34,3 +34,4 @@ service-account.json
 
 
 config/local.toml
+venv

Makefile

@@ -1,2 +1,2 @@
-run-local:
+run_local:
 	RUST_LOG=debug RUST_BACKTRACE=1 cargo run -- --config config/local.toml

README.md

@@ -29,14 +29,17 @@ Mozilla Sync Storage built with [Rust](https://rust-lang.org).
 - MySQL 5.7 (or compatible)
   -\* libmysqlclient (`brew install mysql` on macOS, `apt-get install libmysqlclient-dev` on Ubuntu)
 - [Go](https://golang.org/doc/install)
+- Cmake
+- Pkg-config
+- Openssl
 
 Depending on your OS, you may also need to install `libgrpcdev`, and `protobuf-compiler-grpc`.
 
 ## Local Setup
 
 1. Follow the instructions below to use either MySQL or Spanner as your DB.
 2. Now `cp config/local.example.toml config/local.toml`. Open `config/local.toml` and make sure you have the desired settings configured. For a complete list of available configuration options, check out [docs/config.md](docs/config.md).
-3. `make run-local` starts the server in debug mode, using your new `local.toml` file for config options. Or, simply `cargo run` with your own config options provided as env vars.
+3. `make run_local` starts the server in debug mode, using your new `local.toml` file for config options. Or, simply `cargo run` with your own config options provided as env vars.
 4. Visit `http://localhost:8000/__heartbeat__` to make sure the server is running.
 
 ### MySQL

config/local.example.toml

@@ -1,4 +1,4 @@
 database_url = "mysql://sample_user:sample_password@localhost/syncstorage_rs"
-master_secret = "some_secret_here"
+master_secret = "INSERT_SECRET_KEY_HERE"
 # removing this line will default to moz_json formatted logs (which is preferred for production envs)
 human_logs = 1 

shell.nix

@@ -0,0 +1,20 @@
+# Nix is a powerful package manager for Linux and other Unix systems that makes
+# package management reliable and reproducible: https://nixos.org/nix/.
+# This file is intended to be used with `nix-shell`
+# (https://nixos.org/nix/manual/#sec-nix-shell) to setup a fully-functional
+# syncstorage-rs build environment by installing all required dependencies.
+with import <nixpkgs> {};
+stdenv.mkDerivation {
+  name = "syncstorage-rs";
+  buildInputs = [
+    rustc
+    cargo
+    libmysqlclient
+    pkgconfig
+    openssl
+    cmake
+    protobuf
+    go
+  ];
+  NIX_LDFLAGS = "-L${libmysqlclient}/lib/mysql";
+}

spanner-2019-10-01.ddl

@@ -37,7 +37,7 @@ CREATE TABLE bsos (
 INTERLEAVE IN user_collections;
 
     CREATE INDEX BsoExpiry
-        ON bsos(expiry),
+        ON bsos(fxa_uid, fxa_kid, collection_id, expiry),
 INTERLEAVE IN user_collections;
 
 CREATE TABLE collections (

src/server/test.rs

@@ -371,6 +371,17 @@ fn put_bso() {
     assert!(result >= start);
 }
 
+#[test]
+fn put_meta_storage() {
+    let start = SyncTimestamp::default();
+    // test that "collection" is accepted, even if ignored
+    let body = json!({"id": "global", "collection": "meta", "payload": "SomePayload"});
+    let bytes = test_endpoint_with_body(http::Method::PUT, "/1.5/42/storage/meta/global", body);
+    let result: PutBso = serde_json::from_slice(&bytes).unwrap();
+
+    assert!(result >= start);
+}
+
 #[test]
 fn invalid_content_type() {
     let path = "/1.5/42/storage/bookmarks/wibble";

src/web/extractors.rs

@@ -8,7 +8,7 @@ use actix_web::{
     dev::{ConnectionInfo, Extensions, Payload},
     error::ErrorInternalServerError,
     http::{
-        header::{qitem, Accept, ContentType, Header, HeaderMap, HeaderValue},
+        header::{qitem, Accept, ContentType, Header, HeaderMap},
         Uri,
     },
     web::{Json, Query},
@@ -313,9 +313,11 @@ pub struct BsoBody {
     pub payload: Option<String>,
     #[validate(custom = "validate_body_bso_ttl")]
     pub ttl: Option<u32>,
-    /// Any client-supplied value for this field is ignored
+    /// Any client-supplied value for these fields are ignored
     #[serde(rename(deserialize = "modified"), skip_serializing)]
     pub _ignored_modified: Option<IgnoredAny>,
+    #[serde(rename(deserialize = "collection"), skip_serializing)]
+    pub _ignored_collection: Option<IgnoredAny>,
 }
 
 impl FromRequest for BsoBody {
@@ -367,26 +369,6 @@ impl FromRequest for BsoBody {
 
         let max_payload_size = state.limits.max_record_payload_bytes as usize;
 
-        // The `meta/global` BSO is special in that it's used as a command to
-        // reset the timestamp. A possible client bug is sending an empty
-        // body. Handle it, but issue an info so we can track the event.
-        if req.path().ends_with("/meta/global")
-            && req
-                .headers()
-                .get("content-length")
-                .unwrap_or(&HeaderValue::from(0))
-                == HeaderValue::from(0)
-        {
-            info!("⚠️ Got an empty meta BSO");
-            return Box::new(future::ok(BsoBody {
-                id: None,
-                sortindex: None,
-                payload: None,
-                ttl: None,
-                _ignored_modified: None,
-            }));
-        }
-
         let fut = <Json<BsoBody>>::from_request(&req, payload)
             .map_err(|e| {
                 warn!("⚠️ Could not parse BSO Body: {:?}", e);

tools/examples/README.md

@@ -0,0 +1,3 @@
+# Example scripts for working with syncstorage-rs
+
+* `put.bash` - First `chmod +x put.bash` then `./put.bash` to simulate a PUT.

tools/poster/post.bash tools/examples/put.bashtools/poster/post.bash renamed to tools/examples/put.bash

@@ -1,9 +1,11 @@
 #!/bin/bash
 NODE="http://localhost:8000"
 URI="/1.5/1/storage/col2/DEADBEEF"
-AUTH=`../hawk/venv/bin/python ../hawk/make_hawk_token.py --node $NODE --uri $URI --as_header`
+METHOD="PUT"
+SYNC_MASTER_SECRET="INSERT_SECRET_KEY_HERE"
+AUTH=`../hawk/venv/bin/python ../hawk/make_hawk_token.py --node $NODE --uri $URI --method $METHOD --secret=$SYNC_MASTER_SECRET --as_header`
 curl -vv -X PUT "$NODE$URI" \
-    -H "Authorization: $AUTH" \
+    -H "$AUTH" \
     -H 'Content-Type: application/json' \
     -H 'Accept: application/json' \
     -d '{"id": "womble", "payload": "mary had a little lamb with a nice mint jelly", "sortindex": 0, "ttl": 86400}'