Skip to content

Commit cb257d3

Browse files
mattford-amazonmattford-amazon
committed
Refactor inventory gathering in preparation for new source of application inventory
1 parent e55692f commit cb257d3

File tree

10 files changed

+196
-74
lines changed

10 files changed

+196
-74
lines changed

‎agent/plugins/inventory/datauploader/uploader_test.go‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
package datauploader
1616

1717
import (
18+
"encoding/json"
1819
"testing"
1920

2021
"github.com/aws/amazon-ssm-agent/agent/context"
@@ -51,6 +52,26 @@ func FakeInventoryItems(count int) (items []model.Item) {
5152
return
5253
}
5354

55+
func ApplicationInventoryItem() (items []model.Item) {
56+
// Omit Version which is not omitempty
57+
// Omit InstalledTime and URL which are omitempty
58+
// Include CompType which should be omitted in all cases
59+
items = append(items, model.Item{
60+
Name: "RandomInventoryItem",
61+
Content: model.ApplicationData{
62+
Name: "Test1",
63+
Publisher: "Pub1",
64+
ApplicationType: "Foo",
65+
Architecture: "Brutalism",
66+
CompType: model.AWSComponent,
67+
},
68+
SchemaVersion: "1.0",
69+
CaptureTime: "time",
70+
})
71+
72+
return
73+
}
74+
5475
//TODO: add unit tests for ShouldUpdate scenario once content hash is implemented
5576

5677
func TestConvertToSsmInventoryItems(t *testing.T) {
@@ -81,3 +102,27 @@ func TestConvertToSsmInventoryItems(t *testing.T) {
81102
inventoryItems, _, err = u.ConvertToSsmInventoryItems(c, items)
82103
assert.NotNil(t, err, "Error should be thrown for unsupported Item.Content")
83104
}
105+
106+
func TestConvertExcludedAndEmptyToSsmInventoryItems(t *testing.T) {
107+
108+
var items []model.Item
109+
var inventoryItems []*ssm.InventoryItem
110+
var err error
111+
112+
c := context.NewMockDefault()
113+
u := MockInventoryUploader()
114+
115+
//testing positive scenario
116+
117+
//setting up inventory.Item
118+
items = append(items, ApplicationInventoryItem()...)
119+
inventoryItems, _, err = u.ConvertToSsmInventoryItems(c, items)
120+
121+
assert.Nil(t, err, "Error shouldn't be thrown for application inventory item")
122+
assert.Equal(t, len(items), len(inventoryItems), "Count of inventory items should be equal to input")
123+
124+
bytes, err := json.Marshal(items[0].Content)
125+
assert.Nil(t, err, "Error shouldn't be thrown when marshalling content")
126+
// CompType not present even though it has value. Version should be present even though it doesn't. InstallTime and Url should not be present because they have no value.
127+
assert.Equal(t, "{\"Name\":\"Test1\",\"Publisher\":\"Pub1\",\"Version\":\"\",\"ApplicationType\":\"Foo\",\"Architecture\":\"Brutalism\"}", string(bytes[:]))
128+
}

‎agent/plugins/inventory/gatherers/application/dataProvider.go‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
package application
2+
3+
import (
4+
"strings"
5+
6+
"github.com/aws/amazon-ssm-agent/agent/context"
7+
"github.com/aws/amazon-ssm-agent/agent/plugins/inventory/model"
8+
)
9+
10+
const (
11+
amazonPublisherName = "amazon"
12+
amazonSsmAgentLinux = "amazon-ssm-agent"
13+
amazonSsmAgentWin = "amazon ssm agent"
14+
awsToolsWindows = "aws tools for windows"
15+
ec2ConfigService = "ec2configservice"
16+
awsCfnBootstrap = "aws-cfn-bootstrap"
17+
awsPVDrivers = "aws pv drivers"
18+
awsAPIToolsPrefix = "aws-apitools-"
19+
awsAMIToolsPrefix = "aws-amitools-"
20+
)
21+
22+
var selectAwsApps map[string]string
23+
24+
func init() {
25+
//NOTE:
26+
// For V1 - to filter out aws components from aws applications - we are using a list of all aws components that
27+
// have been identified in various OS - amazon linux, ubuntu, windows etc.
28+
// This is also useful for amazon linux ami - where all packages have Amazon.com as publisher.
29+
selectAwsApps = make(map[string]string)
30+
selectAwsApps[amazonSsmAgentLinux] = amazonPublisherName
31+
selectAwsApps[amazonSsmAgentWin] = amazonPublisherName
32+
selectAwsApps[awsToolsWindows] = amazonPublisherName
33+
selectAwsApps[ec2ConfigService] = amazonPublisherName
34+
selectAwsApps[awsCfnBootstrap] = amazonPublisherName
35+
selectAwsApps[awsPVDrivers] = amazonPublisherName
36+
}
37+
38+
func componentType(applicationName string) model.ComponentType {
39+
formattedName := strings.TrimSpace(applicationName)
40+
formattedName = strings.ToLower(formattedName)
41+
42+
var compType model.ComponentType
43+
44+
//check if application is a known aws component or part of aws-apitool- or aws-amitools- tool set.
45+
if _, found := selectAwsApps[formattedName]; found || strings.Contains(formattedName, awsAPIToolsPrefix) || strings.Contains(formattedName, awsAMIToolsPrefix) {
46+
compType |= model.AWSComponent
47+
}
48+
49+
return compType
50+
}
51+
52+
func CollectApplicationData(context context.T) (appData []model.ApplicationData) {
53+
return collectPlatformDependentApplicationData(context)
54+
}

‎agent/plugins/inventory/gatherers/application/dataProvider_test.go‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
// Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License"). You may not
4+
// use this file except in compliance with the License. A copy of the
5+
// License is located at
6+
//
7+
// http://aws.amazon.com/apache2.0/
8+
//
9+
// or in the "license" file accompanying this file. This file is distributed
10+
// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
11+
// either express or implied. See the License for the specific language governing
12+
// permissions and limitations under the License.
13+
14+
// Package application contains a application gatherer.
15+
package application
16+
17+
import (
18+
"testing"
19+
20+
"github.com/aws/amazon-ssm-agent/agent/plugins/inventory/model"
21+
"github.com/stretchr/testify/assert"
22+
)
23+
24+
func TestComponentType(t *testing.T) {
25+
awsComponents := []string{"amazon-ssm-agent", "aws-apitools-mon", "aws-amitools-ec2", "AWS Tools for Windows", "AWS PV Drivers"}
26+
nonawsComponents := []string{"Notepad++", "Google Update Helper", "accountsservice", "pcre", "kbd-misc"}
27+
28+
for _, name := range awsComponents {
29+
assert.Equal(t, model.AWSComponent, componentType(name))
30+
}
31+
32+
for _, name := range nonawsComponents {
33+
assert.Equal(t, model.ComponentType(0), componentType(name))
34+
}
35+
}

‎agent/plugins/inventory/gatherers/application/dataProvider_unix.go‎

Lines changed: 15 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -61,8 +61,8 @@ func platformInfoProvider(log log.T) (name string, err error) {
6161
return platform.PlatformName(log)
6262
}
6363

64-
// CollectApplicationData collects all application data from the system using rpm or dpkg query.
65-
func CollectApplicationData(context context.T) (appData []model.ApplicationData) {
64+
// collectPlatformDependentApplicationData collects all application data from the system using rpm or dpkg query.
65+
func collectPlatformDependentApplicationData(context context.T) (appData []model.ApplicationData) {
6666

6767
var err error
6868
log := context.Log()
@@ -71,11 +71,11 @@ func CollectApplicationData(context context.T) (appData []model.ApplicationData)
7171
cmd := dpkgCmd
7272

7373
// try dpkg first, if any error occurs, use rpm
74-
if appData, err = GetApplicationData(context, cmd, args); err != nil {
74+
if appData, err = getApplicationData(context, cmd, args); err != nil {
7575
log.Info("Getting applications information using dpkg failed, trying rpm now")
7676
cmd = rpmCmd
7777
args = []string{rpmCmdArgToGetAllApplications, rpmQueryFormat, rpmQueryFormatArgs}
78-
if appData, err = GetApplicationData(context, cmd, args); err != nil {
78+
if appData, err = getApplicationData(context, cmd, args); err != nil {
7979
log.Errorf("Unable to detect package manager - hence no inventory data for %v", GathererName)
8080
}
8181
}
@@ -86,8 +86,8 @@ func CollectApplicationData(context context.T) (appData []model.ApplicationData)
8686
return
8787
}
8888

89-
// GetApplicationData runs a shell command and gets information about all packages/applications
90-
func GetApplicationData(context context.T, command string, args []string) (data []model.ApplicationData, err error) {
89+
// getApplicationData runs a shell command and gets information about all packages/applications
90+
func getApplicationData(context context.T, command string, args []string) (data []model.ApplicationData, err error) {
9191

9292
/*
9393
Note: Following are samples of how rpm & dpkg stores package information.
@@ -190,7 +190,7 @@ func GetApplicationData(context context.T, command string, args []string) (data
190190
cmdOutput := string(output)
191191
log.Debugf("Command output: %v", cmdOutput)
192192

193-
if data, err = ConvertToApplicationData(cmdOutput); err != nil {
193+
if data, err = convertToApplicationData(cmdOutput); err != nil {
194194
err = fmt.Errorf("Unable to convert query output to ApplicationData - %v", err.Error())
195195
} else {
196196
log.Infof("Number of applications detected - %v", len(data))
@@ -200,8 +200,8 @@ func GetApplicationData(context context.T, command string, args []string) (data
200200
return
201201
}
202202

203-
// ConvertToApplicationData converts query output into json string so that it can be deserialized easily
204-
func ConvertToApplicationData(input string) (data []model.ApplicationData, err error) {
203+
// convertToApplicationData converts query output into json string so that it can be deserialized easily
204+
func convertToApplicationData(input string) (data []model.ApplicationData, err error) {
205205

206206
//This implementation is closely tied to the kind of rpm/dpkg query. A change in query MUST be accompanied
207207
//with a change in transform logic or else json formatting will be impacted.
@@ -236,15 +236,17 @@ func ConvertToApplicationData(input string) (data []model.ApplicationData, err e
236236
if err = json.Unmarshal([]byte(str), &data); err == nil {
237237

238238
//transform the date - by iterating over all elements
239-
for j, item := range data {
239+
for i, item := range data {
240240
if item.InstalledTime != "" {
241-
if i, err := strconv.ParseInt(item.InstalledTime, 10, 64); err == nil {
241+
if sec, err := strconv.ParseInt(item.InstalledTime, 10, 64); err == nil {
242242
//InstalledTime must comply with format: 2016-07-30T18:15:37Z to provide better search experience for customers
243-
tm := time.Unix(i, 0).UTC()
244-
data[j].InstalledTime = tm.Format(time.RFC3339)
243+
tm := time.Unix(sec, 0).UTC()
244+
item.InstalledTime = tm.Format(time.RFC3339)
245245
}
246246
//ignore the date transformation if error is encountered
247247
}
248+
item.CompType = componentType(item.Name)
249+
data[i] = item
248250
}
249251
}
250252

‎agent/plugins/inventory/gatherers/application/dataProvider_unix_test.go‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ func MockTestExecutorWithAndWithoutError(command string, args ...string) ([]byte
4949
}
5050

5151
func TestConvertToApplicationData(t *testing.T) {
52-
data, err := ConvertToApplicationData(sampleData)
52+
data, err := convertToApplicationData(sampleData)
5353

5454
assert.Nil(t, err, "Check conversion logic - since sample data in unit test is tied to implementation")
5555
assert.Equal(t, 2, len(data), "Given sample data must return 2 entries of application data")
@@ -71,15 +71,15 @@ func TestGetApplicationData(t *testing.T) {
7171
//testing with error
7272
cmdExecutor = MockTestExecutorWithError
7373

74-
data, err = GetApplicationData(mockContext, mockCommand, mockArgs)
74+
data, err = getApplicationData(mockContext, mockCommand, mockArgs)
7575

7676
assert.NotNil(t, err, "Error must be thrown when command execution fails")
7777
assert.Equal(t, 0, len(data), "When command execution fails - application dataset must be empty")
7878

7979
//testing without error
8080
cmdExecutor = MockTestExecutorWithoutError
8181

82-
data, err = GetApplicationData(mockContext, mockCommand, mockArgs)
82+
data, err = getApplicationData(mockContext, mockCommand, mockArgs)
8383

8484
assert.Nil(t, err, "Error must not be thrown with MockTestExecutorWithoutError")
8585
assert.Equal(t, 2, len(data), "Given sample data must return 2 entries of application data")
@@ -90,16 +90,16 @@ func TestCollectApplicationData(t *testing.T) {
9090

9191
// both dpkg and rpm return result without error
9292
cmdExecutor = MockTestExecutorWithoutError
93-
data := CollectApplicationData(mockContext)
93+
data := collectPlatformDependentApplicationData(mockContext)
9494
assert.Equal(t, 2, len(data), "Given sample data must return 2 entries of application data")
9595

9696
// both dpkg and rpm return errors
9797
cmdExecutor = MockTestExecutorWithError
98-
data = CollectApplicationData(mockContext)
98+
data = collectPlatformDependentApplicationData(mockContext)
9999
assert.Equal(t, 0, len(data), "When command execution fails - application dataset must be empty")
100100

101101
// dpkg returns error and rpm return some result
102102
cmdExecutor = MockTestExecutorWithAndWithoutError
103-
data = CollectApplicationData(mockContext)
103+
data = collectPlatformDependentApplicationData(mockContext)
104104
assert.Equal(t, 2, len(data), "Given sample data must return 2 entries of application data")
105105
}

‎agent/plugins/inventory/gatherers/application/dataProvider_windows.go‎

Lines changed: 13 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -44,8 +44,8 @@ func executeCommand(command string, args ...string) ([]byte, error) {
4444
return exec.Command(command, args...).CombinedOutput()
4545
}
4646

47-
// CollectApplicationData collects application data for windows platform
48-
func CollectApplicationData(context context.T) []model.ApplicationData {
47+
// collectPlatformDependentApplicationData collects application data for windows platform
48+
func collectPlatformDependentApplicationData(context context.T) []model.ApplicationData {
4949
/*
5050
Note:
5151
@@ -69,11 +69,11 @@ func CollectApplicationData(context context.T) []model.ApplicationData {
6969
var data, apps []model.ApplicationData
7070

7171
//getting all 64 bit applications
72-
apps = ExecutePowershellCommands(context, PowershellCmd, ArgsFor64BitApplications, Arch64Bit)
72+
apps = executePowershellCommands(context, PowershellCmd, ArgsFor64BitApplications, Arch64Bit)
7373
data = append(data, apps...)
7474

7575
//getting all 32 bit applications
76-
apps = ExecutePowershellCommands(context, PowershellCmd, ArgsFor32BitApplications, Arch32Bit)
76+
apps = executePowershellCommands(context, PowershellCmd, ArgsFor32BitApplications, Arch32Bit)
7777
data = append(data, apps...)
7878

7979
//sorts the data based on application-name
@@ -82,8 +82,8 @@ func CollectApplicationData(context context.T) []model.ApplicationData {
8282
return data
8383
}
8484

85-
// ExecutePowershellCommands executes commands in powershell to get all windows applications installed.
86-
func ExecutePowershellCommands(context context.T, command, args, arch string) (data []model.ApplicationData) {
85+
// executePowershellCommands executes commands in powershell to get all windows applications installed.
86+
func executePowershellCommands(context context.T, command, args, arch string) (data []model.ApplicationData) {
8787

8888
var output []byte
8989
var err error
@@ -105,7 +105,7 @@ func ExecutePowershellCommands(context context.T, command, args, arch string) (d
105105
cmdOutput := string(output)
106106
log.Debugf("Command output: %v", cmdOutput)
107107

108-
if data, err = ConvertToApplicationData(cmdOutput, arch); err != nil {
108+
if data, err = convertToApplicationData(cmdOutput, arch); err != nil {
109109
err = fmt.Errorf("Unable to convert query output to ApplicationData - %v", err.Error())
110110
log.Error(err.Error())
111111
log.Infof("No application data to return")
@@ -120,8 +120,8 @@ func ExecutePowershellCommands(context context.T, command, args, arch string) (d
120120
return
121121
}
122122

123-
// ConvertToApplicationData converts powershell command output to an array of model.ApplicationData
124-
func ConvertToApplicationData(cmdOutput, architecture string) (data []model.ApplicationData, err error) {
123+
// convertToApplicationData converts powershell command output to an array of model.ApplicationData
124+
func convertToApplicationData(cmdOutput, architecture string) (data []model.ApplicationData, err error) {
125125
//This implementation is closely tied to the kind of powershell command we run in windows. A change in command
126126
//MUST be accompanied with a change in json conversion logic as well.
127127

@@ -163,10 +163,11 @@ func ConvertToApplicationData(cmdOutput, architecture string) (data []model.Appl
163163
if err = json.Unmarshal([]byte(str), &data); err == nil {
164164

165165
//iterate over all entries and add default value of architecture as given input
166-
for i, v := range data {
166+
for i, item := range data {
167167
//set architecture to given input
168-
v.Architecture = architecture
169-
data[i] = v
168+
item.Architecture = architecture
169+
item.CompType = componentType(item.Name)
170+
data[i] = item
170171
}
171172
}
172173

‎agent/plugins/inventory/gatherers/application/dataProvider_windows_test.go‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ func TestConvertToApplicationData(t *testing.T) {
5050
var data []model.ApplicationData
5151
var err error
5252

53-
data, err = ConvertToApplicationData(sampleData, mockArch)
53+
data, err = convertToApplicationData(sampleData, mockArch)
5454

5555
assert.Nil(t, err, "Error is not expected for processing sample data - %v", sampleData)
5656
assert.Equal(t, 3, len(data))
@@ -66,19 +66,19 @@ func TestExecutePowershellCommands(t *testing.T) {
6666

6767
//testing command executor without errors
6868
cmdExecutor = MockTestExecutorWithoutError
69-
data = ExecutePowershellCommands(c, mockCmd, mockArgs, mockArch)
69+
data = executePowershellCommands(c, mockCmd, mockArgs, mockArch)
7070

7171
assert.Equal(t, 3, len(data), "There must be 3 applications for given sample data - %v", sampleData)
7272

7373
//testing command executor with errors
7474
cmdExecutor = MockTestExecutorWithError
75-
data = ExecutePowershellCommands(c, mockCmd, mockArgs, mockArch)
75+
data = executePowershellCommands(c, mockCmd, mockArgs, mockArch)
7676

7777
assert.Equal(t, 0, len(data), "On encountering error - application dataset must be empty")
7878

7979
//testing command executor with ConvertToApplicationData throwing errors
8080
cmdExecutor = MockTestExecutorWithConvertToApplicationDataReturningRandomString
81-
data = ExecutePowershellCommands(c, mockCmd, mockArgs, mockArch)
81+
data = executePowershellCommands(c, mockCmd, mockArgs, mockArch)
8282

8383
assert.Equal(t, 0, len(data), "On encountering error during json conversion - application dataset must be empty")
8484
}
@@ -90,13 +90,13 @@ func TestCollectApplicationData(t *testing.T) {
9090

9191
//testing command executor without errors
9292
cmdExecutor = MockTestExecutorWithoutError
93-
data = CollectApplicationData(c)
93+
data = collectPlatformDependentApplicationData(c)
9494

9595
assert.Equal(t, 6, data, "MockExecutor will be called 2 times hence total entries must be 6")
9696

9797
//testing command executor with errors
9898
cmdExecutor = MockTestExecutorWithError
99-
data = CollectApplicationData(c)
99+
data = collectPlatformDependentApplicationData(c)
100100

101101
assert.Equal(t, 0, data, "If MockExecutor throws error, application dataset must be empty")
102102
}

0 commit comments

Comments
 (0)