Skip to content

Commit ea3b6a0

Browse files
eecksteineeckstein
committed
stdlib: rewrite Array/ContiguousArray logic for reserving capacity and creating a new buffer.
Share more code and avoid the large generic functions of ArrayProtocol. The result is a significant code size win.
1 parent 033ddff commit ea3b6a0

File tree

5 files changed

+155
-47
lines changed

5 files changed

+155
-47
lines changed

‎stdlib/public/core/Array.swift

Lines changed: 66 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -346,7 +346,8 @@ extension Array {
346346
@_semantics("array.make_mutable")
347347
internal mutating func _makeMutableAndUnique() {
348348
if _slowPath(!_buffer.isMutableAndUniquelyReferenced()) {
349-
_buffer = _Buffer(copying: _buffer)
349+
_createNewBuffer(bufferIsUnique: false, minimumCapacity: count,
350+
growForAppend: false)
350351
}
351352
}
352353

@@ -1023,19 +1024,65 @@ extension Array: RangeReplaceableCollection {
10231024
@inlinable
10241025
@_semantics("array.mutate_unknown")
10251026
public mutating func reserveCapacity(_ minimumCapacity: Int) {
1026-
if _buffer.requestUniqueMutableBackingBuffer(
1027-
minimumCapacity: minimumCapacity) == nil {
1027+
_reserveCapacityImpl(minimumCapacity: minimumCapacity,
1028+
growForAppend: false)
1029+
}
10281030

1029-
let newBuffer = _ContiguousArrayBuffer<Element>(
1030-
_uninitializedCount: count, minimumCapacity: minimumCapacity)
1031+
/// Reserves enough space to store `minimumCapacity` elements.
1032+
/// If a new buffer needs to be allocated and `growForAppend` is true,
1033+
/// the new capacity is calculated using `_growArrayCapacity`, but at least
1034+
/// kept at `minimumCapacity`.
1035+
@_alwaysEmitIntoClient
1036+
internal mutating func _reserveCapacityImpl(
1037+
minimumCapacity: Int, growForAppend: Bool
1038+
) {
1039+
let isUnique = _buffer.isUniquelyReferenced()
1040+
if _slowPath(!isUnique || _getCapacity() < minimumCapacity) {
1041+
_createNewBuffer(bufferIsUnique: isUnique,
1042+
minimumCapacity: Swift.max(minimumCapacity, count),
1043+
growForAppend: growForAppend)
1044+
}
1045+
_internalInvariant(capacity >= minimumCapacity)
1046+
_internalInvariant(capacity == 0 || _buffer.isUniquelyReferenced())
1047+
}
10311048

1049+
/// Creates a new buffer, replacing the current buffer.
1050+
///
1051+
/// If `bufferIsUnique` is true, the buffer is assumed to be uniquely
1052+
/// referenced by this array and the elements are moved - instead of copied -
1053+
/// to the new buffer.
1054+
/// The `minimumCapacity` is the lower bound for the new capacity.
1055+
/// If `growForAppend` is true, the new capacity is calculated using
1056+
/// `_growArrayCapacity`, but at least kept at `minimumCapacity`.
1057+
@_alwaysEmitIntoClient
1058+
@inline(never)
1059+
internal mutating func _createNewBuffer(
1060+
bufferIsUnique: Bool, minimumCapacity: Int, growForAppend: Bool
1061+
) {
1062+
let newCapacity = _growArrayCapacity(oldCapacity: _getCapacity(),
1063+
minimumCapacity: minimumCapacity,
1064+
growForAppend: growForAppend)
1065+
let count = _getCount()
1066+
_internalInvariant(newCapacity >= count)
1067+
1068+
let newBuffer = _ContiguousArrayBuffer<Element>(
1069+
_uninitializedCount: count, minimumCapacity: newCapacity)
1070+
1071+
if bufferIsUnique {
1072+
_internalInvariant(_buffer.isUniquelyReferenced())
1073+
1074+
// As an optimization, if the original buffer is unique, we can just move
1075+
// the elements instead of copying.
1076+
let dest = newBuffer.firstElementAddress
1077+
dest.moveInitialize(from: _buffer.firstElementAddress,
1078+
count: count)
1079+
_buffer.count = 0
1080+
} else {
10321081
_buffer._copyContents(
1033-
subRange: _buffer.indices,
1082+
subRange: 0..<count,
10341083
initializing: newBuffer.firstElementAddress)
1035-
_buffer = _Buffer(
1036-
_buffer: newBuffer, shiftedToStartIndex: _buffer.startIndex)
10371084
}
1038-
_internalInvariant(capacity >= minimumCapacity)
1085+
_buffer = _Buffer(_buffer: newBuffer, shiftedToStartIndex: 0)
10391086
}
10401087

10411088
/// Copy the contents of the current buffer to a new unique mutable buffer.
@@ -1054,7 +1101,9 @@ extension Array: RangeReplaceableCollection {
10541101
@_semantics("array.make_mutable")
10551102
internal mutating func _makeUniqueAndReserveCapacityIfNotUnique() {
10561103
if _slowPath(!_buffer.isMutableAndUniquelyReferenced()) {
1057-
_copyToNewBuffer(oldCount: _buffer.count)
1104+
_createNewBuffer(bufferIsUnique: false,
1105+
minimumCapacity: count + 1,
1106+
growForAppend: true)
10581107
}
10591108
}
10601109

@@ -1083,7 +1132,9 @@ extension Array: RangeReplaceableCollection {
10831132
_buffer.isMutableAndUniquelyReferenced())
10841133

10851134
if _slowPath(oldCount + 1 > _buffer.capacity) {
1086-
_copyToNewBuffer(oldCount: oldCount)
1135+
_createNewBuffer(bufferIsUnique: true,
1136+
minimumCapacity: oldCount + 1,
1137+
growForAppend: true)
10871138
}
10881139
}
10891140

@@ -1124,6 +1175,8 @@ extension Array: RangeReplaceableCollection {
11241175
@inlinable
11251176
@_semantics("array.append_element")
11261177
public mutating func append(_ newElement: __owned Element) {
1178+
// Separating uniqueness check and capacity check allows hoisting the
1179+
// uniqueness check out of a loop.
11271180
_makeUniqueAndReserveCapacityIfNotUnique()
11281181
let oldCount = _getCount()
11291182
_reserveCapacityAssumingUniqueBuffer(oldCount: oldCount)
@@ -1183,16 +1236,10 @@ extension Array: RangeReplaceableCollection {
11831236
@inlinable
11841237
@_semantics("array.reserve_capacity_for_append")
11851238
internal mutating func reserveCapacityForAppend(newElementsCount: Int) {
1186-
let oldCount = self.count
1187-
let oldCapacity = self.capacity
1188-
let newCount = oldCount + newElementsCount
1189-
11901239
// Ensure uniqueness, mutability, and sufficient storage. Note that
11911240
// for consistency, we need unique self even if newElements is empty.
1192-
self.reserveCapacity(
1193-
newCount > oldCapacity ?
1194-
Swift.max(newCount, _growArrayCapacity(oldCapacity))
1195-
: newCount)
1241+
_reserveCapacityImpl(minimumCapacity: self.count + newElementsCount,
1242+
growForAppend: true)
11961243
}
11971244

11981245
@inlinable

‎stdlib/public/core/ArrayShared.swift

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -138,6 +138,23 @@ internal func _growArrayCapacity(_ capacity: Int) -> Int {
138138
return capacity * 2
139139
}
140140

141+
@_alwaysEmitIntoClient
142+
internal func _growArrayCapacity(
143+
oldCapacity: Int, minimumCapacity: Int, growForAppend: Bool
144+
) -> Int {
145+
if growForAppend {
146+
if oldCapacity < minimumCapacity {
147+
// When appending to an array, grow exponentially.
148+
return Swift.max(minimumCapacity, _growArrayCapacity(oldCapacity))
149+
}
150+
return oldCapacity
151+
}
152+
// If not for append, just use the specified capacity, ignoring oldCapacity.
153+
// This means that we "shrink" the buffer in case minimumCapacity is less
154+
// than oldCapacity.
155+
return minimumCapacity
156+
}
157+
141158
//===--- generic helpers --------------------------------------------------===//
142159

143160
extension _ArrayBufferProtocol {

‎stdlib/public/core/ContiguousArray.swift

Lines changed: 66 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,8 @@ extension ContiguousArray {
6767
@_semantics("array.make_mutable")
6868
internal mutating func _makeMutableAndUnique() {
6969
if _slowPath(!_buffer.isMutableAndUniquelyReferenced()) {
70-
_buffer = _Buffer(copying: _buffer)
70+
_createNewBuffer(bufferIsUnique: false, minimumCapacity: count,
71+
growForAppend: false)
7172
}
7273
}
7374

@@ -655,19 +656,64 @@ extension ContiguousArray: RangeReplaceableCollection {
655656
@inlinable
656657
@_semantics("array.mutate_unknown")
657658
public mutating func reserveCapacity(_ minimumCapacity: Int) {
658-
if _buffer.requestUniqueMutableBackingBuffer(
659-
minimumCapacity: minimumCapacity) == nil {
659+
_reserveCapacityImpl(minimumCapacity: minimumCapacity,
660+
growForAppend: false)
661+
}
662+
663+
/// Reserves enough space to store `minimumCapacity` elements.
664+
/// If a new buffer needs to be allocated and `growForAppend` is true,
665+
/// the new capacity is calculated using `_growArrayCapacity`.
666+
@_alwaysEmitIntoClient
667+
internal mutating func _reserveCapacityImpl(
668+
minimumCapacity: Int, growForAppend: Bool
669+
) {
670+
let isUnique = _buffer.isUniquelyReferenced()
671+
if _slowPath(!isUnique || _getCapacity() < minimumCapacity) {
672+
_createNewBuffer(bufferIsUnique: isUnique,
673+
minimumCapacity: Swift.max(minimumCapacity, count),
674+
growForAppend: growForAppend)
675+
}
676+
_internalInvariant(capacity >= minimumCapacity)
677+
_internalInvariant(capacity == 0 || _buffer.isUniquelyReferenced())
678+
}
679+
680+
/// Creates a new buffer, replacing the current buffer.
681+
///
682+
/// If `bufferIsUnique` is true, the buffer is assumed to be uniquely
683+
/// referenced by this array and the elements are moved - instead of copied -
684+
/// to the new buffer.
685+
/// The `minimumCapacity` is the lower bound for the new capacity.
686+
/// If `growForAppend` is true, the new capacity is calculated using
687+
/// `_growArrayCapacity`.
688+
@_alwaysEmitIntoClient
689+
@inline(never)
690+
internal mutating func _createNewBuffer(
691+
bufferIsUnique: Bool, minimumCapacity: Int, growForAppend: Bool
692+
) {
693+
let newCapacity = _growArrayCapacity(oldCapacity: _getCapacity(),
694+
minimumCapacity: minimumCapacity,
695+
growForAppend: growForAppend)
696+
let count = _getCount()
697+
_internalInvariant(newCapacity >= count)
660698

661-
let newBuffer = _ContiguousArrayBuffer<Element>(
662-
_uninitializedCount: count, minimumCapacity: minimumCapacity)
699+
let newBuffer = _ContiguousArrayBuffer<Element>(
700+
_uninitializedCount: count, minimumCapacity: newCapacity)
663701

702+
if bufferIsUnique {
703+
_internalInvariant(_buffer.isUniquelyReferenced())
704+
705+
// As an optimization, if the original buffer is unique, we can just move
706+
// the elements instead of copying.
707+
let dest = newBuffer.firstElementAddress
708+
dest.moveInitialize(from: _buffer.firstElementAddress,
709+
count: count)
710+
_buffer.count = 0
711+
} else {
664712
_buffer._copyContents(
665-
subRange: _buffer.indices,
713+
subRange: 0..<count,
666714
initializing: newBuffer.firstElementAddress)
667-
_buffer = _Buffer(
668-
_buffer: newBuffer, shiftedToStartIndex: _buffer.startIndex)
669715
}
670-
_internalInvariant(capacity >= minimumCapacity)
716+
_buffer = _Buffer(_buffer: newBuffer, shiftedToStartIndex: 0)
671717
}
672718

673719
/// Copy the contents of the current buffer to a new unique mutable buffer.
@@ -687,7 +733,9 @@ extension ContiguousArray: RangeReplaceableCollection {
687733
@_semantics("array.make_mutable")
688734
internal mutating func _makeUniqueAndReserveCapacityIfNotUnique() {
689735
if _slowPath(!_buffer.isMutableAndUniquelyReferenced()) {
690-
_copyToNewBuffer(oldCount: _buffer.count)
736+
_createNewBuffer(bufferIsUnique: false,
737+
minimumCapacity: count + 1,
738+
growForAppend: true)
691739
}
692740
}
693741

@@ -716,7 +764,9 @@ extension ContiguousArray: RangeReplaceableCollection {
716764
_buffer.isMutableAndUniquelyReferenced())
717765

718766
if _slowPath(oldCount + 1 > _buffer.capacity) {
719-
_copyToNewBuffer(oldCount: oldCount)
767+
_createNewBuffer(bufferIsUnique: true,
768+
minimumCapacity: oldCount + 1,
769+
growForAppend: true)
720770
}
721771
}
722772

@@ -757,6 +807,8 @@ extension ContiguousArray: RangeReplaceableCollection {
757807
@inlinable
758808
@_semantics("array.append_element")
759809
public mutating func append(_ newElement: __owned Element) {
810+
// Separating uniqueness check and capacity check allows hoisting the
811+
// uniqueness check out of a loop.
760812
_makeUniqueAndReserveCapacityIfNotUnique()
761813
let oldCount = _getCount()
762814
_reserveCapacityAssumingUniqueBuffer(oldCount: oldCount)
@@ -816,23 +868,17 @@ extension ContiguousArray: RangeReplaceableCollection {
816868
@inlinable
817869
@_semantics("array.reserve_capacity_for_append")
818870
internal mutating func reserveCapacityForAppend(newElementsCount: Int) {
819-
let oldCount = self.count
820-
let oldCapacity = self.capacity
821-
let newCount = oldCount + newElementsCount
822-
823871
// Ensure uniqueness, mutability, and sufficient storage. Note that
824872
// for consistency, we need unique self even if newElements is empty.
825-
self.reserveCapacity(
826-
newCount > oldCapacity ?
827-
Swift.max(newCount, _growArrayCapacity(oldCapacity))
828-
: newCount)
873+
_reserveCapacityImpl(minimumCapacity: self.count + newElementsCount,
874+
growForAppend: true)
829875
}
830876

831877
@inlinable
832878
public mutating func _customRemoveLast() -> Element? {
879+
_makeMutableAndUnique()
833880
let newCount = _getCount() - 1
834881
_precondition(newCount >= 0, "Can't removeLast from an empty ContiguousArray")
835-
_makeUniqueAndReserveCapacityIfNotUnique()
836882
let pointer = (_buffer.firstElementAddress + newCount)
837883
let element = pointer.move()
838884
_buffer.count = newCount

‎test/SILOptimizer/array_contentof_opt.swift

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,10 @@
1-
// RUN: %target-swift-frontend -O -sil-verify-all -emit-sil -enforce-exclusivity=unchecked %s | %FileCheck %s
1+
// RUN: %target-swift-frontend -O -sil-verify-all -emit-sil %s | %FileCheck %s
22
// REQUIRES: swift_stdlib_no_asserts,optimized_stdlib
33

4+
// Temporary disabled on linux.
5+
// TODO: need to adapt some CHECK-lines for linux.
6+
// REQUIRES: OS=macosx
7+
48
// This is an end-to-end test of the array(contentsOf) -> array(Element) optimization
59

610
// CHECK-LABEL: sil @{{.*}}testInt
@@ -15,12 +19,9 @@ public func testInt(_ a: inout [Int]) {
1519
}
1620

1721
// CHECK-LABEL: sil @{{.*}}testThreeInt
18-
// CHECK-NOT: apply
19-
// CHECK: [[FR:%[0-9]+]] = function_ref @$sSa15reserveCapacityyySiFSi_Tg5
22+
// CHECK: [[FR:%[0-9]+]] = function_ref @${{(sSa15reserveCapacityyySiFSi_Tg5|sSa16_createNewBuffer)}}
2023
// CHECK-NEXT: apply [[FR]]
21-
// CHECK-NOT: apply
2224
// CHECK: [[F:%[0-9]+]] = function_ref @$sSa6appendyyxnFSi_Tg5
23-
// CHECK-NOT: apply
2425
// CHECK: apply [[F]]
2526
// CHECK-NEXT: apply [[F]]
2627
// CHECK-NEXT: apply [[F]]

‎test/SILOptimizer/merge_exclusivity.swift

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -372,13 +372,10 @@ private struct EscapedTransforme<T>: WriteProt {
372372
// TESTSIL-NEXT: [[B1:%.*]] = begin_access [modify] [dynamic] [no_nested_conflict] [[REFADDR]]
373373
// TESTSIL: end_access [[B1]]
374374
// TESTSIL: [[BCONF:%.*]] = begin_access [modify] [dynamic] [[REFADDR]]
375-
// TESTSIL: apply {{.*}} : $@convention(method) (Int, @inout Array<UInt8>) -> ()
376375
// TESTSIL: end_access [[BCONF]]
377376
// TESTSIL: [[BCONF:%.*]] = begin_access [modify] [dynamic] [[REFADDR]]
378-
// TESTSIL: apply {{.*}} : $@convention(method) (Int, @inout Array<UInt8>) -> ()
379377
// TESTSIL: end_access [[BCONF]]
380378
// TESTSIL: [[BCONF:%.*]] = begin_access [modify] [dynamic] [[REFADDR]]
381-
// TESTSIL: apply {{.*}} : $@convention(method) (Int, @inout Array<UInt8>) -> ()
382379
// TESTSIL: end_access [[BCONF]]
383380
// TESTSIL-LABEL: } // end sil function '$s17merge_exclusivity14run_MergeTest9yySiF'
384381
@inline(never)

0 commit comments

Comments
 (0)