Refactor AskQuery function to accept image bytes; implement command history storage in SQLite; add Co Pilot feature for enhanced user assistance

Author: 4nkitd

Readme.md

@@ -87,6 +87,24 @@ Or customize the prompt:
 gema commit --prompt "Write a detailed commit message explaining the following changes:"
 ```
 
+### Co Pilot
+
+Get assistance with anything on your screen:
+
+```bash
+gema assist "explain this error message"
+gema a "explain this error message" # alias
+gema copilot "suggest improvements for this code" # alias
+gema assistant "what does this output mean?" # alias
+```
+
+The Co Pilot feature analyzes text from your terminal and provides helpful explanations, suggestions, or guidance based on the content.
+
+Example:
+```bash
+gema assist "What's wrong with this command: grep -l 'function' | xargs sed 's/old/new/g'"
+```
+
 ## Contributing
 
 Contributions are welcome! Please feel free to submit a Pull Request.

cli.go

@@ -57,7 +57,7 @@ func waitForResponse(m model) model {
 	}()
 
 	time.Sleep(3 * time.Second)
-	genaiResponse := AskQuery(m.query)
+	genaiResponse := AskQuery(m.query, nil)
 	done <- true
 	fmt.Print("\r") // Clear the loading line
 

copilot.go

@@ -0,0 +1,126 @@
+package main
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/fatih/color"
+	"github.com/getlantern/systray"
+	"github.com/getlantern/systray/example/icon"
+	"github.com/ncruces/zenity"
+	"github.com/spf13/cobra"
+)
+
+// WriterCmd represents the writer command
+var CoPilotCmd = &cobra.Command{
+	Use:     "assist [text]",
+	Aliases: []string{"a", "copilot", "assistant"},
+	Short:   "Help user with anything on his/her screen.",
+	Long:    `Help user with anything on his/her screen.`,
+	RunE: func(cmd *cobra.Command, args []string) error {
+
+		// Indicate processing
+		processingMsg := color.New(color.FgYellow).PrintFunc()
+		processingMsg("Starting Gema tray application...\n")
+
+		systray.Run(onReady, onExit)
+
+		return nil
+	},
+}
+
+func onReady() {
+	fmt.Println("Initializing system tray...")
+	systray.SetIcon(icon.Data)
+	systray.SetTitle("Gema")
+	mHelp := systray.AddMenuItem("Help", "Take Screensot and and audio.")
+	fmt.Println("Gema is now running in your system tray")
+
+	// Listen for clicks on the help button
+	go func() {
+		for {
+			<-mHelp.ClickedCh
+			fmt.Println("Help button clicked, launching Copilot...")
+			RunCopilot()
+		}
+	}()
+}
+
+func onExit() {
+	fmt.Println("Exiting Gema...")
+	// clean up here
+}
+
+func RunCopilot() {
+	fmt.Println("--------------------------------------------------")
+	fmt.Printf("[%s] Starting Gema Assistant\n", time.Now().Format("15:04:05"))
+
+	fmt.Println("[INFO] Displaying input dialog...")
+	// Ask the user for their question with a centered dialog
+	userQuery, err := zenity.Entry(
+		"What would you like help with?",
+		zenity.Title("Gema Assistant"),
+		zenity.Width(400),
+	)
+	if err != nil {
+		// User canceled or there was an error
+		fmt.Printf("[ERROR] Dialog error: %v\n", err)
+		return
+	}
+
+	// If the user didn't enter anything, return early
+	if userQuery == "" {
+		fmt.Println("[INFO] No query provided, canceling request")
+		return
+	}
+
+	fmt.Println("[INFO] Taking screenshot...")
+	imgBytes, err := Screenshot()
+	if err != nil {
+		fmt.Printf("[ERROR] Failed to capture screenshot: %v\n", err)
+	} else {
+		fmt.Printf("[INFO] Screenshot captured successfully (%d bytes)\n", len(imgBytes))
+	}
+
+	fmt.Println("[INFO] Sending query to AI service...")
+	aiResp := AskQuery(userQuery, [][]byte{imgBytes})
+
+	fmt.Println("====================================")
+	fmt.Printf("[%s] QUERY: %s\n", time.Now().Format("15:04:05"), userQuery)
+	fmt.Println("====================================")
+	fmt.Println("[RESPONSE]")
+	fmt.Println(aiResp.Response)
+	fmt.Println("====================================")
+
+	fmt.Println("[INFO] Copying response to clipboard...")
+	err = PutTextOnClipboard(aiResp.Response)
+	if err != nil {
+		fmt.Printf("[ERROR] Failed to copy to clipboard: %v\n", err)
+	} else {
+		fmt.Println("[INFO] Response copied to clipboard")
+	}
+
+	fmt.Println("[INFO] Converting response to speech...")
+	// Create a channel to signal speech to stop
+	stopSpeech := make(chan struct{})
+
+	// Start speech in a goroutine
+	go func() {
+		SpeakMessage(aiResp.Response, stopSpeech)
+	}()
+
+	// Monitor CLI input in parallel
+	go func() {
+		fmt.Println("[INFO] Press any key to stop speech...")
+		var input string
+		fmt.Scanln(&input) // Wait for Enter, but any input will work
+		close(stopSpeech)  // Signal to stop the speech
+		fmt.Println("[INFO] Speech stopped by user input")
+	}()
+
+	// Wait for speech to complete or be stopped
+	<-stopSpeech
+
+	fmt.Printf("[%s] Gema Assistant completed\n", time.Now().Format("15:04:05"))
+	fmt.Println("--------------------------------------------------")
+}

gemini.go

@@ -17,7 +17,7 @@ type AiResponse struct {
 	Command  string `json:"command"`
 }
 
-func AskQuery(query string) AiResponse {
+func AskQuery(query string, imageBytes [][]byte) AiResponse {
 
 	apiKey := os.Getenv("GENAI_API_KEY")
 	if apiKey == "" {
@@ -47,7 +47,23 @@ func AskQuery(query string) AiResponse {
 
 	finalPrompt := sysInfo + SystemInstruction
 
-	model.SystemInstruction = &genai.Content{Parts: []genai.Part{genai.Text(finalPrompt)}}
+	SystemInstruction := []genai.Part{
+		genai.Text(finalPrompt),
+	}
+
+	attachMents := []genai.Part{}
+
+	attachMents = append(attachMents, genai.Text(query))
+
+	for _, imgBytes := range imageBytes {
+		if imageBytes != nil {
+
+			attachMents = append(attachMents, genai.ImageData("image/jpeg", imgBytes))
+		}
+	}
+
+	model.SystemInstruction = &genai.Content{Parts: SystemInstruction}
+
 	model.ResponseMIMEType = "application/json"
 	model.ResponseSchema = &genai.Schema{
 		Type: genai.TypeObject,
@@ -62,7 +78,7 @@ func AskQuery(query string) AiResponse {
 		Required: []string{"response"},
 	}
 
-	resp, err := model.GenerateContent(ctx, genai.Text(query))
+	resp, err := model.GenerateContent(ctx, attachMents...)
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -82,6 +98,8 @@ func AskQuery(query string) AiResponse {
 		log.Fatal("Response field is missing or not a string")
 	}
 
+	StoreCommandHistory(query, result.Response)
+
 	return result
 }
 

git.go

@@ -128,7 +128,7 @@ func GenerateCommitMessage(path, systemPrompt string) (string, []string) {
 		limitDiffSize(string(diffOutput), 4000)) // Limit diff size to avoid token limits
 
 	// Use AskQuery from gemini.go
-	result := AskQuery(query)
+	result := AskQuery(query, nil)
 	return result.Response, changedFiles
 }
 

go.mod

@@ -5,7 +5,11 @@ go 1.23.4
 require (
 	github.com/elastic/go-sysinfo v1.15.0
 	github.com/fatih/color v1.18.0
+	github.com/getlantern/systray v1.2.2
 	github.com/google/generative-ai-go v0.19.0
+	github.com/kbinani/screenshot v0.0.0-20250118074034-a3924b7bbc8c
+	github.com/mattn/go-sqlite3 v1.14.24
+	github.com/ncruces/zenity v0.10.14
 	github.com/spf13/cobra v1.8.1
 	google.golang.org/api v0.186.0
 )
@@ -17,34 +21,54 @@ require (
 	cloud.google.com/go/auth/oauth2adapt v0.2.2 // indirect
 	cloud.google.com/go/compute/metadata v0.3.0 // indirect
 	cloud.google.com/go/longrunning v0.5.7 // indirect
+	github.com/akavel/rsrc v0.10.2 // indirect
+	github.com/dchest/jsmin v0.0.0-20220218165748-59f39799265f // indirect
 	github.com/elastic/go-windows v1.0.0 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
-	github.com/go-logr/logr v1.4.1 // indirect
+	github.com/gen2brain/shm v0.1.0 // indirect
+	github.com/getlantern/context v0.0.0-20220418194847-3d5e7a086201 // indirect
+	github.com/getlantern/errors v1.0.4 // indirect
+	github.com/getlantern/golog v0.0.0-20230503153817-8e72de7e0a65 // indirect
+	github.com/getlantern/hex v0.0.0-20220104173244-ad7e4b9194dc // indirect
+	github.com/getlantern/hidden v0.0.0-20220104173330-f221c5a24770 // indirect
+	github.com/getlantern/ops v0.0.0-20231025133620-f368ab734534 // indirect
+	github.com/go-logr/logr v1.4.2 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/go-stack/stack v1.8.1 // indirect
+	github.com/godbus/dbus/v5 v5.1.0 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/google/s2a-go v0.1.7 // indirect
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
 	github.com/googleapis/gax-go/v2 v2.12.5 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/jezek/xgb v1.1.1 // indirect
+	github.com/josephspurrier/goversioninfo v1.4.1 // indirect
+	github.com/lxn/win v0.0.0-20210218163916-a377121e959e // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/prometheus/procfs v0.15.1 // indirect
+	github.com/randall77/makefat v0.0.0-20210315173500-7ddd0e42c844 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	go.opencensus.io v0.24.0 // indirect
+	go.opentelemetry.io/auto/sdk v1.1.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.51.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.51.0 // indirect
-	go.opentelemetry.io/otel v1.26.0 // indirect
-	go.opentelemetry.io/otel/metric v1.26.0 // indirect
-	go.opentelemetry.io/otel/trace v1.26.0 // indirect
+	go.opentelemetry.io/otel v1.34.0 // indirect
+	go.opentelemetry.io/otel/metric v1.34.0 // indirect
+	go.opentelemetry.io/otel/trace v1.34.0 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
+	go.uber.org/zap v1.27.0 // indirect
 	golang.org/x/crypto v0.24.0 // indirect
+	golang.org/x/image v0.20.0 // indirect
 	golang.org/x/net v0.26.0 // indirect
 	golang.org/x/oauth2 v0.21.0 // indirect
 	golang.org/x/sync v0.9.0 // indirect
-	golang.org/x/sys v0.27.0 // indirect
-	golang.org/x/text v0.16.0 // indirect
+	golang.org/x/sys v0.30.0 // indirect
+	golang.org/x/text v0.18.0 // indirect
 	golang.org/x/time v0.5.0 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20240617180043-68d350f18fd4 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20240617180043-68d350f18fd4 // indirect