Skip to content

Commit 92b870a

Browse files
authored
Track RunCommand process (#10)
1 parent 7370b32 commit 92b870a

File tree

6 files changed

+158
-9
lines changed

6 files changed

+158
-9
lines changed

integration-test/test/handler-commands.bats

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ teardown(){
2121
echo "Enable count=$enable_count"
2222
[ "$enable_count" -eq 1 ]
2323
[[ "$output" == *"this script configuration is already processed, will not run again"* ]] # not processed again
24-
[[ "$output" == *"transitioning status detected"* ]]
24+
[[ "$output" == *"Previous runcommand process is still running"* ]]
2525
}
2626

2727
@test "handler command: install - creates the data dir" {

main/cmds.go

Lines changed: 16 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -130,13 +130,17 @@ func enablePre(ctx *log.Context, hEnv HandlerEnvironment, seqNum int) error {
130130
return errors.Wrap(err, "failed to process seqnum")
131131
} else if shouldExit {
132132
ctx.Log("event", "exit", "message", "this script configuration is already processed, will not run again")
133-
statusType, err := readStatus(ctx, hEnv, seqNum)
134-
if err == nil && statusType == StatusTransitioning {
135-
// Make sure status is not stuck in transitioning state
136-
// If VM was restarted or process didn't finish its possible that seqNum ihas been marked as processed
137-
// but last status file is still is "transitioning"
138-
ctx.Log("event", "check status", "message", "transitioning status detected - set to error status")
139-
reportStatus(ctx, hEnv, seqNum, StatusError, cmd{enable, "Enable", true, enablePre, 3}, "Last script execution didn't finish.")
133+
if IsExtensionStillRunning(pidFilePath) {
134+
ctx.Log("event", "check status", "message", "Previous runcommand process is still running")
135+
} else {
136+
statusType, err := readStatus(ctx, hEnv, seqNum)
137+
if err == nil && statusType == StatusTransitioning {
138+
// Make sure status is not stuck in transitioning state when previous extension process is not running
139+
// If VM was restarted or process didn't finish its possible that seqNum ihas been marked as processed
140+
// but last status file is still is "transitioning"
141+
ctx.Log("event", "check status", "message", "transitioning status detected but no process to handle it - set to success status")
142+
reportStatus(ctx, hEnv, seqNum, StatusSuccess, cmd{enable, "Enable", true, enablePre, 3}, "Last script execution didn't finish.")
143+
}
140144
}
141145
os.Exit(0)
142146
}
@@ -278,6 +282,11 @@ func runCmd(ctx log.Logger, dir string, cfg handlerSettings) (err error) {
278282
scenario = fmt.Sprintf("protected-script;%s", scenarioInfo)
279283
}
280284

285+
// Store the active process id and start time in case second instance was started by the agent
286+
// If process exited successfully the pid file is deleted
287+
SaveCurrentPidAndStartTime(pidFilePath)
288+
defer DeleteCurrentPidAndStartTime(pidFilePath)
289+
281290
begin := time.Now()
282291
err = ExecCmdInDir(cmd, dir)
283292
elapsed := time.Now().Sub(begin)

main/main.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,9 @@ var (
2323
// incorrect. The correct way is mrseq. This file is auto-preserved by the agent.
2424
mostRecentSequence = "mrseq"
2525

26+
// Filename where active process keeps track of process id and process start time
27+
pidFilePath = "pidstart"
28+
2629
// downloadDir is where we store the downloaded files in the "{downloadDir}/{seqnum}/file"
2730
// format and the logs as "{downloadDir}/{seqnum}/std(out|err)". Stored under dataDir
2831
downloadDir = "download"

main/pid.go

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
package main
2+
3+
import (
4+
"fmt"
5+
"io/ioutil"
6+
"os"
7+
"os/exec"
8+
"strconv"
9+
"strings"
10+
11+
"github.com/pkg/errors"
12+
)
13+
14+
// GetProcessStartTime returns the start time of the active process if still active
15+
func GetProcessStartTime(pid int) (string, error) {
16+
pidString := fmt.Sprintf("%d", pid)
17+
startTime, err := exec.Command("bash", "-c", "ps -o lstart= -p "+pidString).Output()
18+
if err != nil {
19+
return "", errors.Wrap(err, "failed to execute bash ps command")
20+
}
21+
return string(startTime), nil
22+
}
23+
24+
// SaveCurrentPidAndStartTime stores current process id with start date in file extName.pid
25+
// Example: 325 Tue Dec 8 15:54:04 2020
26+
func SaveCurrentPidAndStartTime(path string) error {
27+
pid := os.Getpid()
28+
pidString := fmt.Sprintf("%d", pid)
29+
startTime, err := GetProcessStartTime(pid)
30+
if err != nil {
31+
return errors.Wrap(err, "failed to execute bash ps command")
32+
}
33+
34+
b := []byte(fmt.Sprintf("%s\t%s", pidString, startTime))
35+
return errors.Wrap(ioutil.WriteFile(path, b, chmod), "extName.pid: failed to write")
36+
}
37+
38+
// DeleteCurrentPidAndStartTime delete the file created by SaveCurrentPidAndStartTime
39+
func DeleteCurrentPidAndStartTime(path string) error {
40+
return errors.Wrap(os.Remove(path), "failed to delete "+path)
41+
}
42+
43+
// ReadPidAndStartTime reads the stored pid and process start time from a file extName.pid
44+
// Returns 0 and "" if path not found
45+
func ReadPidAndStartTime(path string) (int, string, error) {
46+
b, err := ioutil.ReadFile(path)
47+
if err != nil {
48+
if os.IsNotExist(err) {
49+
return 0, "", nil
50+
}
51+
return 0, "", errors.Wrap(err, "extName.pid: failed to read:"+path)
52+
}
53+
data := strings.Split(string(b), "\t")
54+
if len(data) != 2 {
55+
return 0, "", errors.Wrap(err, "unexpected format in extName.pid:"+string(b))
56+
}
57+
58+
pid, err := strconv.Atoi(data[0])
59+
if err != nil {
60+
return 0, "", errors.Wrap(err, "failed to convert pid:"+data[0])
61+
}
62+
return pid, data[1], nil
63+
}
64+
65+
// IsExtensionStillRunning checks if there is active process for the same extension name
66+
func IsExtensionStillRunning(path string) bool {
67+
// Check if we have a file record for previous process
68+
previousPid, previousStartTime, err := ReadPidAndStartTime(path)
69+
if err != nil || previousPid == 0 || previousStartTime == "" {
70+
return false
71+
}
72+
73+
// Try to get previous process start time
74+
startTime, err := GetProcessStartTime(previousPid)
75+
if err != nil || startTime == "" {
76+
return false
77+
}
78+
79+
return startTime == previousStartTime
80+
}

main/pid_test.go

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
package main
2+
3+
import (
4+
"fmt"
5+
"io/ioutil"
6+
"os"
7+
"os/exec"
8+
"path/filepath"
9+
"testing"
10+
11+
"github.com/stretchr/testify/require"
12+
)
13+
14+
func Test_SaveAndReadPid(t *testing.T) {
15+
tmpDir, err := ioutil.TempDir("", "")
16+
require.Nil(t, err)
17+
defer os.RemoveAll(tmpDir)
18+
19+
// Verify Save pid operation
20+
path := filepath.Join(tmpDir, "extName.pid")
21+
require.Nil(t, SaveCurrentPidAndStartTime(path))
22+
23+
pid, date, err := ReadPidAndStartTime(path)
24+
require.Nil(t, err, "ReadPidAndStartTime failed")
25+
26+
expectedPid := os.Getpid()
27+
pidString := fmt.Sprintf("%d", pid)
28+
expectedStartTime, err := exec.Command("bash", "-c", "ps -o lstart= -p "+pidString).Output()
29+
require.Equal(t, expectedPid, pid)
30+
require.Equal(t, string(expectedStartTime), date)
31+
}
32+
33+
func Test_IsExtensionStillRunning(t *testing.T) {
34+
tmpDir, err := ioutil.TempDir("", "")
35+
require.Nil(t, err)
36+
defer os.RemoveAll(tmpDir)
37+
38+
path := filepath.Join(tmpDir, "extName.pid")
39+
require.Nil(t, SaveCurrentPidAndStartTime(path))
40+
41+
running := IsExtensionStillRunning(path)
42+
require.Equal(t, true, running)
43+
44+
running = IsExtensionStillRunning(path + "notexist")
45+
require.Equal(t, false, running)
46+
}
47+
48+
func Test_GetProcessStartTime(t *testing.T) {
49+
startTime, err := GetProcessStartTime(os.Getpid())
50+
require.NotEmpty(t, startTime)
51+
require.Nil(t, err)
52+
53+
startTime, err = GetProcessStartTime(-1)
54+
require.Empty(t, startTime)
55+
require.NotNil(t, err)
56+
require.Contains(t, err.Error(), "failed to execute bash ps command")
57+
}

misc/manifest.xml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
<ExtensionImage xmlns="http://schemas.microsoft.com/windowsazure">
33
<ProviderNameSpace>Microsoft.CPlat.Core</ProviderNameSpace>
44
<Type>RunCommandLinux</Type>
5-
<Version>1.0.2</Version>
5+
<Version>1.0.3</Version>
66
<Label>Microsoft Azure Run Command Extension for Linux Virtual Machines</Label>
77
<HostingResources>VmRole</HostingResources>
88
<MediaLink></MediaLink>

0 commit comments

Comments
 (0)