Skip to content

Commit 0dc33b5

Browse files
zookzookzookzook
committed
updated documentation
1 parent 68a3149 commit 0dc33b5

File tree

3 files changed

+223
-18
lines changed

3 files changed

+223
-18
lines changed

‎lib/mongo/bulk_ops.ex

Lines changed: 42 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -31,9 +31,9 @@ defmodule Mongo.BulkOps do
3131
@doc """
3232
Returns an `insert_one` operation tupel for appending to a bulk. Used to perform stream bulk writes.
3333
34-
## Example
34+
Example
3535
```
36-
BulkOps.get_insert_one(%{name: "Waldo"})
36+
Mongo.BulkOps.get_insert_one(%{name: "Waldo"})
3737
3838
{:insert, %{name: "Waldo"}}
3939
```
@@ -44,26 +44,43 @@ defmodule Mongo.BulkOps do
4444
@doc """
4545
Returns an `delete_one` operation tupel for appending to a bulk. Used to perform stream bulk writes.
4646
47-
## Example
47+
Example
4848
4949
```
50-
BulkOps.get_delete_one(%{name: "Waldo"})
50+
Mongo.BulkOps.get_delete_one(%{name: "Waldo"})
5151
5252
{:delete, {%{name: "Waldo"}, [limit: 1]}}
5353
```
5454
"""
55-
@spec get_delete_one(BSON.document, Keyword.t) :: bulk_op
56-
def get_delete_one(doc, opts \\ []), do: {:delete, {doc, Keyword.put(opts, :limit, 1)}}
55+
@spec get_delete_one(BSON.document) :: bulk_op
56+
def get_delete_one(doc), do: {:delete, {doc, [limit: 1]}}
5757

5858

5959
@doc """
6060
Returns an `delete_many` operation for appending to a bulk. Used to perform stream bulk writes.
61+
62+
Example
63+
64+
```
65+
Mongo.BulkOps.get_delete_many(%{name: "Waldo"})
66+
67+
{:delete, {%{name: "Waldo"}, [limit: 0]}}
68+
```
6169
"""
62-
@spec get_delete_many(BSON.document, Keyword.t) :: bulk_op
63-
def get_delete_many(doc, opts \\ []), do: {:delete, {doc, Keyword.put(opts, :limit, 0)}}
70+
@spec get_delete_many(BSON.document) :: bulk_op
71+
def get_delete_many(doc), do: {:delete, {doc, [limit: 0]}}
6472

6573
@doc """
6674
Returns an `update_one` operation for appending to a bulk. Used to perform stream bulk writes.
75+
76+
Example
77+
78+
```
79+
Mongo.BulkOps.get_update_one(%{name: "Waldo"}, %{"$set" : %{name: "Greta", kind: "dog"}})
80+
81+
{:update,
82+
{%{name: "Waldo"}, %{"$set": %{kind: "dog", name: "Greta"}}, [multi: false]}}
83+
```
6784
"""
6885
@spec get_update_one(BSON.document, BSON.document, Keyword.t) :: bulk_op
6986
def get_update_one(filter, update, opts \\ []) do
@@ -73,6 +90,15 @@ defmodule Mongo.BulkOps do
7390

7491
@doc """
7592
Returns an `update_many` operation for appending to a bulk. Used to perform stream bulk writes.
93+
94+
Example
95+
96+
```
97+
Mongo.BulkOps.get_update_many(%{name: "Waldo"}, %{"$set" : %{name: "Greta", kind: "dog"}})
98+
99+
{:update,
100+
{%{name: "Waldo"}, %{"$set": %{kind: "dog", name: "Greta"}}, [multi: true]}}
101+
```
76102
"""
77103
@spec get_update_many(BSON.document, BSON.document, Keyword.t) :: bulk_op
78104
def get_update_many(filter, update, opts \\ []) do
@@ -82,6 +108,14 @@ defmodule Mongo.BulkOps do
82108

83109
@doc """
84110
Returns an `replace_one` operation for appending to a bulk. Used to perform stream bulk writes.
111+
112+
Example
113+
114+
```
115+
Mongo.BulkOps.get_replace_one(%{name: "Waldo"}, %{name: "Greta", kind: "dog"})
116+
117+
{:update, {%{name: "Waldo"}, %{kind: "dog", name: "Greta"}, [multi: false]}}
118+
```
85119
"""
86120
@spec get_replace_one(BSON.document, BSON.document, Keyword.t) :: bulk_op
87121
def get_replace_one(filter, replacement, opts \\ []) do

‎lib/mongo/ordered_bulk.ex

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,11 @@ defmodule Mongo.OrderedBulk do
1010

1111
import Mongo.BulkOps
1212

13+
@type t :: %__MODULE__{
14+
coll: String.t,
15+
ops: [BulkOps.bulk_op]
16+
}
17+
1318
defstruct coll: nil, ops: []
1419

1520
def new(coll) do
@@ -24,12 +29,12 @@ defmodule Mongo.OrderedBulk do
2429
get_insert_one(doc) |> push(bulk)
2530
end
2631

27-
def delete_one(%OrderedBulk{} = bulk, doc, opts \\ []) do
28-
get_delete_one(doc, opts) |> push(bulk)
32+
def delete_one(%OrderedBulk{} = bulk, doc) do
33+
get_delete_one(doc) |> push(bulk)
2934
end
3035

31-
def delete_many(%OrderedBulk{} = bulk, doc, opts \\ []) do
32-
get_delete_many(doc, opts) |> push(bulk)
36+
def delete_many(%OrderedBulk{} = bulk, doc) do
37+
get_delete_many(doc) |> push(bulk)
3338
end
3439

3540
def replace_one(%OrderedBulk{} = bulk, filter, replacement, opts \\ []) do

‎lib/mongo/unordered_bulk.ex

Lines changed: 172 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,48 @@
11
defmodule Mongo.UnorderedBulk do
2-
@moduledoc """
2+
@moduledoc"""
33
4-
The maxWriteBatchSize limit of a database, which indicates the maximum number of write operations permitted in a write batch, raises from 1,000 to 100,000.
4+
An unordered bulk is filled in the store with the bulk operations. These are divided into three lists (inserts, updates, deletes)
5+
added. If the unordered bulk is sent to the database, the groups are written in the following order:
56
7+
1. inserts
8+
2. updates
9+
3. deletes
10+
11+
The order within the group is undefined.
12+
13+
## Example
14+
15+
```
16+
bulk = "bulk"
17+
|> new()
18+
|> insert_one(%{name: "Greta"})
19+
|> insert_one(%{name: "Tom"})
20+
|> insert_one(%{name: "Waldo"})
21+
|> update_one(%{name: "Greta"}, %{"$set": %{kind: "dog"}})
22+
|> update_one(%{name: "Tom"}, %{"$set": %{kind: "dog"}})
23+
|> update_one(%{name: "Waldo"}, %{"$set": %{kind: "dog"}})
24+
|> delete_one(%{kind: "dog"})
25+
|> delete_one(%{kind: "dog"})
26+
|> delete_one(%{kind: "dog"})
27+
28+
result = BulkWrite.write(:mongo, bulk, w: 1)
29+
```
30+
31+
To reduce the memory usage the unordered bulk can be used with streams.
32+
33+
## Example
34+
35+
```
36+
1..1_000_000
37+
|> Stream.map(fn i -> BulkOps.get_insert_one(%{number: i}) end)
38+
|> UnorderedBulk.write(:mongo, "bulk", 1_000)
39+
|> Stream.run()
40+
```
41+
42+
This example first generates the bulk operation by calling `get_insert_one\1`. The operation is used as a parameter in the `write\3` function.
43+
The unordered bulk was created with a buffer of 1000 operations. After 1000 operations, the
44+
unordered bulk is written to the database. Depending on the selected size you can control the speed and memory consumption. The higher the
45+
value, the faster the processing and the greater the memory consumption.
646
"""
747

848
alias Mongo.UnorderedBulk
@@ -20,10 +60,26 @@ defmodule Mongo.UnorderedBulk do
2060

2161
defstruct coll: nil, inserts: [], updates: [], deletes: []
2262

63+
@doc """
64+
Creates an empty unordered bulk for a collection.
65+
66+
Example:
67+
68+
```
69+
Mongo.UnorderedBulk.new("bulk")
70+
%Mongo.UnorderedBulk{coll: "bulk", deletes: [], inserts: [], updates: []}
71+
```
72+
"""
73+
@spec new(String.t) :: UnorderedBulk.t
2374
def new(coll) do
2475
%UnorderedBulk{coll: coll}
2576
end
2677

78+
@doc """
79+
Appends a bulk operation to the unordered bulk. One of the field (inserts, updates or deletes)
80+
will be updated.
81+
"""
82+
@spec push(BulkOps.bulk_op, UnorderedBulk.t) :: UnorderedBulk.t
2783
def push({:insert, doc}, %UnorderedBulk{inserts: rest} = bulk) do
2884
%UnorderedBulk{bulk | inserts: [doc | rest] }
2985
end
@@ -34,30 +90,138 @@ defmodule Mongo.UnorderedBulk do
3490
%UnorderedBulk{bulk | deletes: [doc | rest] }
3591
end
3692

93+
94+
@doc """
95+
Appends an insert operation.
96+
97+
Example:
98+
99+
```
100+
Mongo.UnorderedBulk.insert_one(bulk, %{name: "Waldo"})
101+
%Mongo.UnorderedBulk{
102+
coll: "bulk",
103+
deletes: [],
104+
inserts: [%{name: "Waldo"}],
105+
updates: []
106+
}
107+
```
108+
"""
109+
@spec insert_one(UnorderedBulk.t, BulkOps.bulk_op) :: UnorderedBulk.t
37110
def insert_one(%UnorderedBulk{} = bulk, doc) do
38111
get_insert_one(doc) |> push(bulk)
39112
end
40113

41-
def delete_one(%UnorderedBulk{} = bulk, doc, opts \\ []) do
42-
get_delete_one(doc, opts) |> push(bulk)
114+
@doc """
115+
Appends a delete operation with `:limit = 1`.
116+
117+
Example:
118+
119+
```
120+
Mongo.UnorderedBulk.delete_one(bulk, %{name: "Waldo"})
121+
%Mongo.UnorderedBulk{
122+
coll: "bulk",
123+
deletes: [{%{name: "Waldo"}, [limit: 1]}],
124+
inserts: [],
125+
updates: []
126+
}
127+
```
128+
"""
129+
@spec delete_one(UnorderedBulk.t, BulkOps.bulk_op) :: UnorderedBulk.t
130+
def delete_one(%UnorderedBulk{} = bulk, doc) do
131+
get_delete_one(doc) |> push(bulk)
43132
end
44133

45-
def delete_many(%UnorderedBulk{} = bulk, doc, opts \\ []) do
46-
get_delete_many(doc, opts) |> push(bulk)
134+
@doc """
135+
Appends a delete operation with `:limit = 0`.
136+
137+
Example:
138+
139+
```
140+
Mongo.UnorderedBulk.delete_many(bulk, %{name: "Waldo"})
141+
%Mongo.UnorderedBulk{
142+
coll: "bulk",
143+
deletes: [{%{name: "Waldo"}, [limit: 0]}],
144+
inserts: [],
145+
updates: []
146+
}
147+
```
148+
"""
149+
@spec delete_many(UnorderedBulk.t, BulkOps.bulk_op) :: UnorderedBulk.t
150+
def delete_many(%UnorderedBulk{} = bulk, doc) do
151+
get_delete_many(doc) |> push(bulk)
47152
end
48153

154+
@doc """
155+
Appends a replace operation with `:multi = false`.
156+
157+
Example:
158+
159+
```
160+
Mongo.UnorderedBulk.replace_one(bulk, %{name: "Waldo"}, %{name: "Greta", kind: "dog"})
161+
%Mongo.UnorderedBulk{
162+
coll: "bulk",
163+
deletes: [],
164+
inserts: [],
165+
updates: [{%{name: "Waldo"}, %{kind: "dog", name: "Greta"}, [multi: false]}]
166+
}
167+
```
168+
"""
169+
@spec replace_one(UnorderedBulk.t, BSON.document, BSON.document, Keyword.t) :: UnorderedBulk.t
49170
def replace_one(%UnorderedBulk{} = bulk, filter, replacement, opts \\ []) do
50171
get_replace_one(filter, replacement, opts) |> push(bulk)
51172
end
52173

174+
@doc """
175+
Appends a update operation with `:multi = false`.
176+
177+
Example:
178+
179+
```
180+
Mongo.UnorderedBulk.update_one(bulk, %{name: "Waldo"}, %{"$set": %{name: "Greta", kind: "dog"}})
181+
%Mongo.UnorderedBulk{
182+
coll: "bulk",
183+
deletes: [],
184+
inserts: [],
185+
updates: [
186+
{%{name: "Waldo"}, %{"$set": %{kind: "dog", name: "Greta"}}, [multi: false]}
187+
]
188+
}
189+
```
190+
"""
191+
@spec update_one(UnorderedBulk.t, BSON.document, BSON.document, Keyword.t) :: UnorderedBulk.t
53192
def update_one(%UnorderedBulk{} = bulk, filter, update, opts \\ []) do
54193
get_update_one(filter, update, opts) |> push(bulk)
55194
end
56195

196+
@doc """
197+
Appends a update operation with `:multi = true`.
198+
199+
Example:
200+
201+
```
202+
Mongo.UnorderedBulk.update_many(bulk, %{name: "Waldo"}, %{"$set": %{name: "Greta", kind: "dog"}})
203+
%Mongo.UnorderedBulk{
204+
coll: "bulk",
205+
deletes: [],
206+
inserts: [],
207+
updates: [
208+
{%{name: "Waldo"}, %{"$set": %{kind: "dog", name: "Greta"}}, [multi: true]}
209+
]
210+
}
211+
```
212+
"""
213+
@spec update_many(UnorderedBulk.t, BSON.document, BSON.document, Keyword.t) :: UnorderedBulk.t
57214
def update_many(%UnorderedBulk{} = bulk, filter, update, opts \\ []) do
58215
get_update_many(filter, update, opts) |> push(bulk)
59216
end
60217

218+
@doc """
219+
Returns a stream chunk function that can be used with streams. The `limit` specifies the number
220+
of operation hold in the memory while processing the stream inputs.
221+
222+
The inputs of the stream should be `Mongo.BulkOps.bulk_op`. See `Mongo.BulkOps`
223+
"""
224+
@spec write(Enumerable.t(), GenServer.server, String.t, non_neg_integer, Keyword.t ) :: Enumerable.t()
61225
def write(enum, top, coll, limit \\ 1000, opts \\ []) when limit > 1 do
62226
Stream.chunk_while(enum,
63227
{new(coll), limit - 1},
@@ -71,6 +235,8 @@ defmodule Mongo.UnorderedBulk do
71235
end)
72236
# todo reduce to one
73237
end
238+
## todo limit == 0 => write direct
239+
## Stream.map(fn op -> BulkWrite.write(conn, op, opts)
74240

75241
def test(top) do
76242

0 commit comments

Comments
 (0)